From 89fe2ef53ce6bc0099646dd1c47d4e3d8e152a86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Wed, 25 Oct 2023 12:50:28 +0200 Subject: [PATCH 01/54] Update gems --- Gemfile | 16 ++++----- Gemfile.lock | 82 +++++++++++++++++++------------------------ lib/3scale/backend.rb | 2 +- 3 files changed, 44 insertions(+), 56 deletions(-) diff --git a/Gemfile b/Gemfile index 0d8aa41f..4c671813 100644 --- a/Gemfile +++ b/Gemfile @@ -9,7 +9,7 @@ gemspec # implementations (ie. pure Ruby, java, etc). # platform :ruby do - gem 'hiredis', '~> 0.6.1' + gem 'hiredis-client' gem 'yajl-ruby', '~> 1.4.3', require: 'yajl' gem 'pry-byebug', '~> 3.5.1', groups: [:development] end @@ -24,7 +24,7 @@ group :test do gem 'mocha', '~> 1.3' gem 'nokogiri', '~> 1.14.3' gem 'pkg-config', '~> 1.1.7' - gem 'resque_unit', '~> 0.4.4', source: 'https://rubygems.org' + gem 'resque_unit', '~> 0.4.4' gem 'test-unit', '~> 3.5' gem 'resque_spec', '~> 0.17.0' gem 'timecop', '~> 0.9.1' @@ -55,8 +55,8 @@ gem 'daemons', '= 1.2.4' # Production gems gem 'rake', '~> 13.0' gem 'builder', '= 3.2.3' -# Use a patched resque to allow reusing their Airbrake Failure class -gem 'resque', git: 'https://github.com/3scale/resque', branch: '3scale' +gem 'redis', '~> 5' +gem 'resque', '~> 2.6.0' gem 'redis-namespace', '~>1.8' gem 'rack', '~> 2.2.6' gem 'sinatra', '~> 2.2.4' @@ -64,12 +64,8 @@ gem 'sinatra-contrib', '~> 2.2.4' # Optional external error logging services gem 'bugsnag', '~> 6', require: nil gem 'yabeda-prometheus', '~> 0.5.0' -gem 'async-redis', '~> 0.7.0' -gem 'async-pool', '~> 0.3.12' +gem 'async-redis', '~> 0.8' +gem 'async-pool', '~> 0.4' gem 'falcon', '~> 0.35' gem 'webrick', '~> 1.8' -# Use a patched redis-rb that fixes an issue when trying to connect with -# sentinels and avoids retrying calls when there's a timeout to prevent -# duplicated commands. It's based on version 4.1.3. -gem 'redis', git: 'https://github.com/3scale/redis-rb', branch: 'apisonator' diff --git a/Gemfile.lock b/Gemfile.lock index fab1ea59..c912a6ec 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -6,25 +6,6 @@ GIT puma (4.3.9) nio4r (~> 2.0) -GIT - remote: https://github.com/3scale/redis-rb - revision: 7210a9d6cf733fe5a1ad0dd20f5f613167743810 - branch: apisonator - specs: - redis (4.1.3) - -GIT - remote: https://github.com/3scale/resque - revision: db327e389cf2fc572a503c47b19871ed899356d1 - branch: 3scale - specs: - resque (1.27.4) - mono_logger (~> 1.0) - multi_json (~> 1.0) - redis-namespace (~> 1.3) - sinatra (>= 0.9.2) - vegas (~> 0.1.2) - GIT remote: https://github.com/3scale/source2swagger revision: 9a787007577fc58b5822b55720e977cc063057fd @@ -63,16 +44,16 @@ GEM traces (>= 0.8.0) async-http-cache (0.4.3) async-http (~> 0.56) - async-io (1.34.3) + async-io (1.36.0) async - async-pool (0.3.12) + async-pool (0.4.0) async (>= 1.25) - async-redis (0.7.0) + async-redis (0.8.0) async (>= 1.8, < 3.0) async-io (~> 1.10) async-pool (~> 0.2) - protocol-redis (~> 0.6.0) - async-rspec (1.13.0) + protocol-redis (~> 0.8.0) + async-rspec (1.17.0) rspec (~> 3.0) rspec-files (~> 1.0) rspec-memory (~> 1.0) @@ -86,10 +67,12 @@ GEM simplecov (>= 0.7.1, < 1.0.0) coderay (1.1.3) concurrent-ruby (1.1.6) - console (1.16.2) + connection_pool (2.4.1) + console (1.23.2) + fiber-annotation fiber-local daemons (1.2.4) - diff-lcs (1.3) + diff-lcs (1.5.0) docile (1.1.5) dry-initializer (3.0.3) falcon (0.42.3) @@ -105,12 +88,14 @@ GEM process-metrics (~> 0.2.0) protocol-rack (~> 0.1) samovar (~> 2.1) + fiber-annotation (0.2.0) fiber-local (1.0.0) gli (2.16.1) - hiredis (0.6.3) + hiredis-client (0.17.0) + redis-client (= 0.17.0) i18n (1.8.2) concurrent-ruby (~> 1.0) - json (2.3.1) + json (2.6.3) license_finder (7.1.0) bundler rubyzip (>= 1, < 3) @@ -126,7 +111,7 @@ GEM minitest (5.18.0) mocha (1.3.0) metaclass (~> 0.0.1) - mono_logger (1.1.0) + mono_logger (1.1.2) multi_json (1.15.0) mustache (1.0.5) mustermann (2.0.2) @@ -155,7 +140,7 @@ GEM protocol-rack (0.2.4) protocol-http (~> 0.23) rack (>= 1.0) - protocol-redis (0.6.1) + protocol-redis (0.8.0) pry (0.14.0) coderay (~> 1.1) method_source (~> 1.0) @@ -166,14 +151,23 @@ GEM pry (~> 0.11) yard (~> 0.9.11) racc (1.6.2) - rack (2.2.6.4) + rack (2.2.8) rack-protection (2.2.4) rack rack-test (0.8.2) rack (>= 1.0, < 3) rake (13.0.1) - redis-namespace (1.10.0) + redis (5.0.7) + redis-client (>= 0.9.0) + redis-client (0.17.0) + connection_pool + redis-namespace (1.11.0) redis (>= 4) + resque (2.6.0) + mono_logger (~> 1.0) + multi_json (~> 1.0) + redis-namespace (~> 1.6) + sinatra (>= 0.9.2) resque_spec (0.17.0) resque (>= 1.19.0) rspec-core (>= 3.0.0) @@ -186,19 +180,19 @@ GEM rspec-core (~> 3.7.0) rspec-expectations (~> 3.7.0) rspec-mocks (~> 3.7.0) - rspec-core (3.7.0) + rspec-core (3.7.1) rspec-support (~> 3.7.0) rspec-expectations (3.7.0) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.7.0) - rspec-files (1.0.1) + rspec-files (1.1.3) rspec (~> 3.0) - rspec-memory (1.0.1) + rspec-memory (1.0.4) rspec (~> 3.0) rspec-mocks (3.7.0) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.7.0) - rspec-support (3.7.0) + rspec-support (3.7.1) rspec_api_documentation (5.1.0) activesupport (>= 3.0.0) mustache (~> 1.0, >= 0.99.4) @@ -231,15 +225,13 @@ GEM power_assert thor (1.2.1) thread_safe (0.3.6) - tilt (2.1.0) + tilt (2.3.0) timecop (0.9.1) timers (4.3.5) tomlrb (2.0.3) traces (0.9.1) tzinfo (1.2.7) thread_safe (~> 0.1) - vegas (0.1.11) - rack (>= 1.0.0) webrick (1.8.1) with_env (1.1.0) xml-simple (1.1.9) @@ -258,8 +250,8 @@ PLATFORMS DEPENDENCIES apisonator! - async-pool (~> 0.3.12) - async-redis (~> 0.7.0) + async-pool (~> 0.4) + async-redis (~> 0.8) async-rspec benchmark-ips (~> 2.7.2) bugsnag (~> 6) @@ -268,7 +260,7 @@ DEPENDENCIES daemons (= 1.2.4) falcon (~> 0.35) gli (~> 2.16.1) - hiredis (~> 0.6.1) + hiredis-client license_finder (~> 7.0) mocha (~> 1.3) nokogiri (~> 1.14.3) @@ -280,11 +272,11 @@ DEPENDENCIES rack (~> 2.2.6) rack-test (= 0.8.2) rake (~> 13.0) - redis! + redis (~> 5) redis-namespace (~> 1.8) - resque! + resque (~> 2.6.0) resque_spec (~> 0.17.0) - resque_unit (~> 0.4.4)! + resque_unit (~> 0.4.4) rspec (~> 3.7.0) rspec_api_documentation (~> 5.0) sinatra (~> 2.2.4) diff --git a/lib/3scale/backend.rb b/lib/3scale/backend.rb index f92b88f9..2524b0ea 100644 --- a/lib/3scale/backend.rb +++ b/lib/3scale/backend.rb @@ -2,7 +2,7 @@ require_relative 'bundler_shim' require 'builder' -require 'hiredis' +require 'hiredis-client' require 'redis' From bde5ea15fee380ce701ab9d0742bdaa269092cbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Wed, 25 Oct 2023 13:01:08 +0200 Subject: [PATCH 02/54] Use Boolifyed API Some redis methods that used to return boolean in the old version now return whatever the redis server returns. To keep using the boolifyed versions we need to call the same methods but with a `?` suffix. Also, now the redis client only accepts strings as parameters --- lib/3scale/backend/alert_limit.rb | 4 ++-- lib/3scale/backend/application.rb | 6 +++--- lib/3scale/backend/application_events.rb | 2 +- lib/3scale/backend/service.rb | 4 ++-- lib/3scale/backend/service_token.rb | 2 +- lib/3scale/backend/worker.rb | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/3scale/backend/alert_limit.rb b/lib/3scale/backend/alert_limit.rb index 2132e32a..5079153b 100644 --- a/lib/3scale/backend/alert_limit.rb +++ b/lib/3scale/backend/alert_limit.rb @@ -9,7 +9,7 @@ class AlertLimit attr_accessor :service_id, :value def save - storage.sadd(key_allowed_set(service_id), value.to_i) if valid? + storage.sadd?(key_allowed_set(service_id), value.to_i) if valid? end def to_hash @@ -32,7 +32,7 @@ def self.save(service_id, value) end def self.delete(service_id, value) - storage.srem(key_allowed_set(service_id), value.to_i) if valid_value?(value) + storage.srem?(key_allowed_set(service_id), value.to_i) if valid_value?(value) end def self.valid_value?(value) diff --git a/lib/3scale/backend/application.rb b/lib/3scale/backend/application.rb index e19244da..a556a85f 100644 --- a/lib/3scale/backend/application.rb +++ b/lib/3scale/backend/application.rb @@ -91,7 +91,7 @@ def extract_id!(service_id, app_id, user_key) end def exists?(service_id, id) - storage.exists(storage_key(service_id, id, :state)) + storage.exists?(storage_key(service_id, id, :state)) end memoize :exists? @@ -260,7 +260,7 @@ def create_key(value = nil) def delete_key(value) db_key = storage_key(:keys) invalidate_cache([:smembers, :scard, :sismember], db_key) - storage.srem(db_key, value) + storage.srem?(db_key, value) end def has_keys? @@ -279,7 +279,7 @@ def has_key?(value) db_key = storage_key(:keys) key = Memoizer.build_key(self.class, :sismember, db_key, value) Memoizer.memoize_block(key) do - storage.sismember(db_key, value) + storage.sismember(db_key, value.to_s) end end diff --git a/lib/3scale/backend/application_events.rb b/lib/3scale/backend/application_events.rb index 210cf566..6b7cd3cd 100644 --- a/lib/3scale/backend/application_events.rb +++ b/lib/3scale/backend/application_events.rb @@ -40,7 +40,7 @@ def self.generate(applications) private def self.first_traffic(service_id, application_id) - if storage.sadd(Stats::Keys.set_of_apps_with_traffic(service_id), + if storage.sadd?(Stats::Keys.set_of_apps_with_traffic(service_id), encode_key(application_id)) EventStorage.store(:first_traffic, { service_id: service_id, diff --git a/lib/3scale/backend/service.rb b/lib/3scale/backend/service.rb index 225737b9..1a5877ad 100644 --- a/lib/3scale/backend/service.rb +++ b/lib/3scale/backend/service.rb @@ -80,7 +80,7 @@ def delete_by_id(service_id) end def exists?(service_id) - storage.exists(storage_key(service_id, 'provider_key')) + storage.exists?(storage_key(service_id, 'provider_key')) end def get_service(id) @@ -277,7 +277,7 @@ def persist_attribute(attribute, value, ignore_nils = false) def persist_sets storage.sadd storage_key_by_provider(:ids), id storage.sadd encode_key("services_set"), id - storage.sadd encode_key("provider_keys_set"), provider_key + storage.sadd encode_key("provider_keys_set"), provider_key unless provider_key.nil? end end diff --git a/lib/3scale/backend/service_token.rb b/lib/3scale/backend/service_token.rb index d0ee5c79..4d4761be 100644 --- a/lib/3scale/backend/service_token.rb +++ b/lib/3scale/backend/service_token.rb @@ -60,7 +60,7 @@ def delete(service_token, service_id) end def exists?(service_token, service_id) - storage.exists(key(service_token, service_id)) + storage.exists?(key(service_token, service_id)) end memoize :exists? diff --git a/lib/3scale/backend/worker.rb b/lib/3scale/backend/worker.rb index 5b322346..473cdba8 100644 --- a/lib/3scale/backend/worker.rb +++ b/lib/3scale/backend/worker.rb @@ -72,11 +72,11 @@ def perform(job) end def register_worker - redis.sadd(:workers, self) + redis.sadd(:workers, self.class.name) end def unregister_worker - redis.srem(:workers, self) + redis.srem(:workers, self.class.name) end def hostname From 06776385f872b008d1982d3598b79156a7a67150 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Wed, 25 Oct 2023 13:07:51 +0200 Subject: [PATCH 03/54] Use the new pipeline API Check https://github.com/redis/redis-rb/blob/master/CHANGELOG.md#460 --- lib/3scale/backend/alerts.rb | 12 +++---- lib/3scale/backend/application.rb | 34 +++++++++---------- lib/3scale/backend/event_storage.rb | 10 +++--- lib/3scale/backend/metric.rb | 28 +++++++-------- lib/3scale/backend/stats/aggregator.rb | 8 ++--- lib/3scale/backend/stats/aggregators/base.rb | 13 +++---- .../stats/aggregators/response_code.rb | 4 +-- lib/3scale/backend/stats/aggregators/usage.rb | 4 +-- lib/3scale/backend/stats/cleaner.rb | 4 +-- .../storage_async/resque_extensions.rb | 6 ++-- .../backend/transactor/notify_batcher.rb | 12 +++---- lib/3scale/backend/usage_limit.rb | 4 +-- 12 files changed, 70 insertions(+), 69 deletions(-) diff --git a/lib/3scale/backend/alerts.rb b/lib/3scale/backend/alerts.rb index c817edda..d99a7cbf 100644 --- a/lib/3scale/backend/alerts.rb +++ b/lib/3scale/backend/alerts.rb @@ -85,15 +85,15 @@ def update_utilization(service_id, app_id, max_utilization, max_record, timestam keys = alert_keys(service_id, app_id, discrete) - already_alerted, allowed = storage.pipelined do - storage.get(keys[:already_notified]) - storage.sismember(keys[:allowed], discrete) + already_alerted, allowed = storage.pipelined do |pipeline| + pipeline.get(keys[:already_notified]) + pipeline.sismember(keys[:allowed], discrete) end if already_alerted.nil? && allowed && discrete.to_i > 0 - next_id, _ = storage.pipelined do - storage.incr(keys[:current_id]) - storage.setex(keys[:already_notified], ALERT_TTL, "1") + next_id, _ = storage.pipelined do |pipeline| + pipeline.incr(keys[:current_id]) + pipeline.setex(keys[:already_notified], ALERT_TTL, "1") end alert = { :id => next_id, diff --git a/lib/3scale/backend/application.rb b/lib/3scale/backend/application.rb index a556a85f..05987bfa 100644 --- a/lib/3scale/backend/application.rb +++ b/lib/3scale/backend/application.rb @@ -102,9 +102,9 @@ def delete(service_id, id) end def delete_data(service_id, id) - storage.pipelined do - delete_set(service_id, id) - delete_attributes(service_id, id) + storage.pipelined do |pipeline| + delete_set(pipeline, service_id, id) + delete_attributes(pipeline, service_id, id) end end @@ -137,12 +137,12 @@ def id_by_key_storage_key(service_id, key) encode_key("application/service_id:#{service_id}/key:#{key}/id") end - def delete_set(service_id, id) - storage.srem(applications_set_key(service_id), id) + def delete_set(client, service_id, id) + client.srem(applications_set_key(service_id), id) end - def delete_attributes(service_id, id) - storage.del( + def delete_attributes(client, service_id, id) + client.del( ATTRIBUTES.map do |f| storage_key(service_id, id, f) end @@ -166,9 +166,9 @@ def with_app_id_from_params(service_id, app_id, user_key) def save raise ApplicationHasNoState.new(id) if !state - storage.pipelined do - persist_attributes - persist_set + storage.pipelined do |pipeline| + persist_attributes(pipeline) + persist_set(pipeline) end self.class.clear_cache(service_id, id) @@ -319,15 +319,15 @@ def has_referrer_filters? private - def persist_attributes - storage.set(storage_key(:state), state.to_s) if state - storage.set(storage_key(:plan_id), plan_id) if plan_id - storage.set(storage_key(:plan_name), plan_name) if plan_name - storage.set(storage_key(:redirect_url), redirect_url) if redirect_url + def persist_attributes(client) + client.set(storage_key(:state), state.to_s) if state + client.set(storage_key(:plan_id), plan_id) if plan_id + client.set(storage_key(:plan_name), plan_name) if plan_name + client.set(storage_key(:redirect_url), redirect_url) if redirect_url end - def persist_set - storage.sadd(applications_set_key(service_id), id) + def persist_set(client) + client.sadd(applications_set_key(service_id), id) end end end diff --git a/lib/3scale/backend/event_storage.rb b/lib/3scale/backend/event_storage.rb index b5db42ea..31c96d23 100644 --- a/lib/3scale/backend/event_storage.rb +++ b/lib/3scale/backend/event_storage.rb @@ -38,8 +38,8 @@ def delete(id) (id > 0) ? storage.zremrangebyscore(events_queue_key, id, id) : 0 end - def size - storage.zcard(events_queue_key) + def size(strg = storage) + strg.zcard(events_queue_key) end def ping_if_not_empty @@ -90,9 +90,9 @@ def events_hook_uri def pending_ping? ## the queue is not empty and more than timeout has passed ## since the front-end was notified - events_set_size, can_ping = storage.pipelined do - size - storage.set(events_ping_key, '1'.freeze, ex: PING_TTL, nx: true) + events_set_size, can_ping = storage.pipelined do |pipeline| + size(pipeline) + pipeline.set(events_ping_key, '1'.freeze, ex: PING_TTL, nx: true) end can_ping && events_set_size > 0 diff --git a/lib/3scale/backend/metric.rb b/lib/3scale/backend/metric.rb index 0153540e..d31f5b49 100644 --- a/lib/3scale/backend/metric.rb +++ b/lib/3scale/backend/metric.rb @@ -27,9 +27,9 @@ def id_set_key(service_id) def save old_name = self.class.load_name(service_id, id) - storage.pipelined do - save_attributes - save_to_list + storage.pipelined do |pipeline| + save_attributes(pipeline) + save_to_list(pipeline) remove_reverse_mapping(service_id, old_name) if old_name != name end @@ -196,10 +196,10 @@ def delete(service_id, id) return false unless name and not name.empty? clear_cache(service_id, id, name) - storage.pipelined do - storage.srem(id_set_key(service_id), id) + storage.pipelined do |pipeline| + pipeline.srem(id_set_key(service_id), id) - storage.del(key(service_id, id, :name), + pipeline.del(key(service_id, id, :name), key(service_id, id, :parent_id), id_key(service_id, name)) end @@ -223,9 +223,9 @@ def hierarchy_ids(service_id) ids = load_all_ids(service_id) parent_ids_keys = ids.map { |id| key(service_id, id, :parent_id) } - parent_ids = storage.pipelined do + parent_ids = storage.pipelined do |pipeline| parent_ids_keys.each_slice(PIPELINED_SLICE_SIZE).map do |slice| - storage.mget(slice) + pipeline.mget(slice) end end.flatten @@ -247,14 +247,14 @@ def remove_reverse_mapping(service_id, name) storage.del id_key(service_id, name) end - def save_attributes - storage.set(id_key(service_id, name), id) - storage.set(key(service_id, id, :name), name) - storage.set(key(service_id, id, :parent_id), parent_id) if parent_id + def save_attributes(client) + client.set(id_key(service_id, name), id) + client.set(key(service_id, id, :name), name) + client.set(key(service_id, id, :parent_id), parent_id) if parent_id end - def save_to_list - storage.sadd(id_set_key(service_id), id) + def save_to_list(client) + client.sadd(id_set_key(service_id), id) end def save_children diff --git a/lib/3scale/backend/stats/aggregator.rb b/lib/3scale/backend/stats/aggregator.rb index 987da4a4..3781a260 100644 --- a/lib/3scale/backend/stats/aggregator.rb +++ b/lib/3scale/backend/stats/aggregator.rb @@ -46,9 +46,9 @@ def aggregate(transactions) touched_apps = {} transactions.each_slice(PIPELINED_SLICE_SIZE) do |slice| - storage.pipelined do + storage.pipelined do |pipeline| slice.each do |transaction| - aggregate_all(transaction) + aggregate_all(transaction, pipeline) touched_apps.merge!(touched_relation(transaction)) end end @@ -57,9 +57,9 @@ def aggregate(transactions) touched_apps end - def aggregate_all(transaction) + def aggregate_all(transaction, client) [Aggregators::ResponseCode, Aggregators::Usage].each do |aggregator| - aggregator.aggregate(transaction) + aggregator.aggregate(transaction, client) end end diff --git a/lib/3scale/backend/stats/aggregators/base.rb b/lib/3scale/backend/stats/aggregators/base.rb index 572c590d..d7d2ac00 100644 --- a/lib/3scale/backend/stats/aggregators/base.rb +++ b/lib/3scale/backend/stats/aggregators/base.rb @@ -11,7 +11,8 @@ module Base # @param [Time] timestamp # @param [Array] keys array of {(service|application|user) => "key"} # @param [Symbol] cmd - def aggregate_values(value, timestamp, keys, cmd) + # # @param [Redis] client + def aggregate_values(value, timestamp, keys, cmd, client = storage) keys_for_bucket = [] keys.each do |metric_type, prefix_key| @@ -23,9 +24,9 @@ def aggregate_values(value, timestamp, keys, cmd) # memory because for rate-limiting and stats, a key of set to 0 # is equivalent to a key that does not exist. if cmd == :set && value == 0 - storage.del(key) + client.del(key) else - store_key(cmd, key, value, expire_time) + store_key(client, cmd, key, value, expire_time) end unless Stats::PeriodCommons::EXCLUDED_FOR_BUCKETS.include?(granularity) @@ -55,9 +56,9 @@ def granularities(metric_type) metric_type == :service ? Stats::PeriodCommons::SERVICE_GRANULARITIES : Stats::PeriodCommons::EXPANDED_GRANULARITIES end - def store_key(cmd, key, value, expire_time = nil) - storage.send(cmd, key, value) - storage.expire(key, expire_time) if expire_time + def store_key(client, cmd, key, value, expire_time = nil) + client.send(cmd, key, value) + client.expire(key, expire_time) if expire_time end end end diff --git a/lib/3scale/backend/stats/aggregators/response_code.rb b/lib/3scale/backend/stats/aggregators/response_code.rb index 865f483d..f9d82290 100644 --- a/lib/3scale/backend/stats/aggregators/response_code.rb +++ b/lib/3scale/backend/stats/aggregators/response_code.rb @@ -11,12 +11,12 @@ class << self include Keys include Base - def aggregate(transaction) + def aggregate(transaction, client = storage) keys_for_multiple_codes = keys_for_response_code(transaction) timestamp = transaction.timestamp keys_for_multiple_codes.each do |keys| - aggregate_values(1, timestamp, keys, :incrby) + aggregate_values(1, timestamp, keys, :incrby, client) end end diff --git a/lib/3scale/backend/stats/aggregators/usage.rb b/lib/3scale/backend/stats/aggregators/usage.rb index 6e986f70..b01065ba 100644 --- a/lib/3scale/backend/stats/aggregators/usage.rb +++ b/lib/3scale/backend/stats/aggregators/usage.rb @@ -14,13 +14,13 @@ class << self # Aggregates the usage of a transaction. # # @param [Transaction] transaction - def aggregate(transaction) + def aggregate(transaction, client = storage) transaction.usage.each do |metric_id, raw_value| metric_keys = Keys.transaction_keys(transaction, :metric, metric_id) cmd = storage_cmd(raw_value) value = Backend::Usage.get_from raw_value - aggregate_values(value, transaction.timestamp, metric_keys, cmd) + aggregate_values(value, transaction.timestamp, metric_keys, cmd, client) end end diff --git a/lib/3scale/backend/stats/cleaner.rb b/lib/3scale/backend/stats/cleaner.rb index 49c9a961..63b31d9e 100644 --- a/lib/3scale/backend/stats/cleaner.rb +++ b/lib/3scale/backend/stats/cleaner.rb @@ -199,9 +199,9 @@ def delete_keys(redis_conn, services, log_deleted_keys) end def remove_services_from_delete_set(services) - storage.pipelined do + storage.pipelined do |pipeline| services.each do |service| - storage.srem(KEY_SERVICES_TO_DELETE, service) + pipeline.srem(KEY_SERVICES_TO_DELETE, service) end end end diff --git a/lib/3scale/backend/storage_async/resque_extensions.rb b/lib/3scale/backend/storage_async/resque_extensions.rb index 1eba2457..752ba3a1 100644 --- a/lib/3scale/backend/storage_async/resque_extensions.rb +++ b/lib/3scale/backend/storage_async/resque_extensions.rb @@ -16,9 +16,9 @@ def enqueue(klass, *args) # We need to add the "resque" namespace in the keys for all the # commands. - async_client.pipelined do - async_client.sadd('resque:queues', queue.to_s) - async_client.rpush( + async_client.pipelined do |pipeline| + pipeline.sadd('resque:queues', queue.to_s) + pipeline.rpush( "resque:queue:#{queue}", Resque.encode(:class => klass.to_s, :args => args) ) end diff --git a/lib/3scale/backend/transactor/notify_batcher.rb b/lib/3scale/backend/transactor/notify_batcher.rb index 130f4891..906588b2 100644 --- a/lib/3scale/backend/transactor/notify_batcher.rb +++ b/lib/3scale/backend/transactor/notify_batcher.rb @@ -60,9 +60,9 @@ def notify_batch(provider_key, usage) end def get_batch(num_elements) - storage.pipelined do - storage.lrange(key_for_notifications_batch, 0, num_elements - 1) - storage.ltrim(key_for_notifications_batch, num_elements, -1) + storage.pipelined do |pipeline| + pipeline.lrange(key_for_notifications_batch, 0, num_elements - 1) + pipeline.ltrim(key_for_notifications_batch, num_elements, -1) end.first end @@ -122,9 +122,9 @@ def enqueue_notify_job(provider_key, usage, timestamp, enqueue_ts) if ThreeScale::Backend.test? module Test def get_full_batch - storage.pipelined do - storage.lrange(key_for_notifications_batch, 0, -1) - storage.del(key_for_notifications_batch) + storage.pipelined do |pipeline| + pipeline.lrange(key_for_notifications_batch, 0, -1) + pipeline.del(key_for_notifications_batch) end.first end diff --git a/lib/3scale/backend/usage_limit.rb b/lib/3scale/backend/usage_limit.rb index 5523b653..717b53db 100644 --- a/lib/3scale/backend/usage_limit.rb +++ b/lib/3scale/backend/usage_limit.rb @@ -40,10 +40,10 @@ def save(attributes) service_id = attributes[:service_id] plan_id = attributes[:plan_id] prefix = key_prefix(service_id, plan_id, attributes[:metric_id]) - storage.pipelined do + storage.pipelined do |pipeline| PERIODS.each do |period| p_val = attributes[period.to_sym] - p_val and storage.set(key_for_period(prefix, period), p_val) + p_val and pipeline.set(key_for_period(prefix, period), p_val) end end clear_cache(service_id, plan_id) From 08e5a3b47761d47c0d7ea588ce2d310ab6029753 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Wed, 25 Oct 2023 13:10:04 +0200 Subject: [PATCH 04/54] Use new pipeline API for async mode --- lib/3scale/backend/storage_async.rb | 1 - .../backend/storage_async/async_redis.rb | 39 ------------------- lib/3scale/backend/storage_async/client.rb | 22 ++++++----- lib/3scale/backend/storage_async/pipeline.rb | 33 +++++++++++++++- 4 files changed, 45 insertions(+), 50 deletions(-) delete mode 100644 lib/3scale/backend/storage_async/async_redis.rb diff --git a/lib/3scale/backend/storage_async.rb b/lib/3scale/backend/storage_async.rb index 109200fe..0bde5e3f 100644 --- a/lib/3scale/backend/storage_async.rb +++ b/lib/3scale/backend/storage_async.rb @@ -1,4 +1,3 @@ require '3scale/backend/storage_async/client' require '3scale/backend/storage_async/pipeline' -require '3scale/backend/storage_async/async_redis' require '3scale/backend/storage_async/resque_extensions' diff --git a/lib/3scale/backend/storage_async/async_redis.rb b/lib/3scale/backend/storage_async/async_redis.rb deleted file mode 100644 index 22ed690d..00000000 --- a/lib/3scale/backend/storage_async/async_redis.rb +++ /dev/null @@ -1,39 +0,0 @@ -# Monkey-patches the async-redis lib to provide a 'call_pipeline' method that -# sends multiple commands at once and returns an array of the responses for -# each of them. - -module Async - module Redis - class Client - def call_pipeline(commands) - @pool.acquire do |connection| - commands.each do |command| - connection.write_request(command) - end - - connection.flush - - # Redis returns an answer for each of the commands sent in the - # pipeline. But in order to keep compatibility with redis-rb, here, if - # there is an error in any of the commands of the pipeline we will - # raise an error (the first one that occurred). - - first_err = nil - - res = commands.size.times.map do - begin - connection.read_response - rescue ::Protocol::Redis::ServerError => e - first_err ||= e - nil - end - end - - raise first_err if first_err - - res - end - end - end - end -end diff --git a/lib/3scale/backend/storage_async/client.rb b/lib/3scale/backend/storage_async/client.rb index 95df1e52..73045bb8 100644 --- a/lib/3scale/backend/storage_async/client.rb +++ b/lib/3scale/backend/storage_async/client.rb @@ -73,6 +73,7 @@ def initialize(opts) METHODS_TO_BE_CALLED_DIRECTLY = [ :del, + :exists, :expire, :expireat, :flushdb, @@ -90,9 +91,11 @@ def initialize(opts) :mget, :ping, :rpush, + :sadd, :scard, :setex, :smembers, + :srem, :sunion, :ttl, :zcard, @@ -109,17 +112,18 @@ def initialize(opts) end METHODS_TO_BOOLIFY = [ - :exists, + :exists?, :sismember, - :sadd, - :srem, + :sadd?, + :srem?, :zadd ].freeze private_constant :METHODS_TO_BOOLIFY METHODS_TO_BOOLIFY.each do |method| + command = method.to_s.delete('?') define_method(method) do |*args| - @redis_async.call(method, *args.flatten) > 0 + @redis_async.call(command, *args.flatten) > 0 end end @@ -163,9 +167,9 @@ def scan(cursor, opts = {}) end # This method allows us to send pipelines like this: - # storage.pipelined do - # storage.get('a') - # storage.get('b') + # storage.pipelined do |pipeline| + # pipeline.get('a') + # pipeline.get('b') # end def pipelined(&block) # This replaces the client with a Pipeline that accumulates the Redis @@ -177,7 +181,7 @@ def pipelined(&block) # When running a nested pipeline, we just need to continue # accumulating commands. if @building_pipeline - block.call + block.call self return end @@ -188,7 +192,7 @@ def pipelined(&block) @redis_async = pipeline begin - block.call + block.call self ensure @redis_async = original @building_pipeline = false diff --git a/lib/3scale/backend/storage_async/pipeline.rb b/lib/3scale/backend/storage_async/pipeline.rb index e1b2d50f..5df5e9e9 100644 --- a/lib/3scale/backend/storage_async/pipeline.rb +++ b/lib/3scale/backend/storage_async/pipeline.rb @@ -59,7 +59,7 @@ def call(*args) # Returns an array with the result for each command in the same order # that they added with .call(). def run(redis_async_client) - responses = redis_async_client.call_pipeline(@commands) + responses = collect_responses(redis_async_client) responses.zip(@commands).map do |resp, cmd| command_name = cmd.first.to_s.upcase @@ -73,6 +73,37 @@ def run(redis_async_client) end end end + + private + + def collect_responses(redis_async_client) + async_pipe = redis_async_client.pipeline + @commands.each do |command| + async_pipe.write_request(*command) + end + + # Redis returns an answer for each of the commands sent in the + # pipeline. But in order to keep compatibility with redis-rb, here, if + # there is an error in any of the commands of the pipeline we will + # raise an error (the first one that occurred). + + first_err = nil + + res = @commands.size.times.map do + begin + async_pipe.read_response + rescue ::Protocol::Redis::ServerError => e + first_err ||= e + nil + end + end + + raise first_err if first_err + + res + ensure + async_pipe.close + end end end end From be9c1e6bc780c37d78fb85baf25eeceee6376b19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Wed, 25 Oct 2023 13:11:09 +0200 Subject: [PATCH 05/54] Fix tests --- spec/unit/queue_storage_spec.rb | 6 ++-- spec/unit/stats/cleaner_spec.rb | 16 +++++----- test/integration/authrep/basic_test.rb | 2 +- test/integration/authrep/set_usage_test.rb | 4 +-- test/integration/report_test.rb | 6 ++-- test/test_helpers/storage.rb | 8 ++--- test/unit/storage_sync_test.rb | 37 ++++++++++++---------- 7 files changed, 41 insertions(+), 38 deletions(-) diff --git a/spec/unit/queue_storage_spec.rb b/spec/unit/queue_storage_spec.rb index 5543e4d7..d08cb51b 100644 --- a/spec/unit/queue_storage_spec.rb +++ b/spec/unit/queue_storage_spec.rb @@ -52,11 +52,11 @@ module Backend private def is_sentinel?(connection) - connector = connection.instance_variable_get(:@inner) + config = connection.instance_variable_get(:@inner) .instance_variable_get(:@client) - .instance_variable_get(:@connector) + .instance_variable_get(:@config) - connector.instance_of?(Redis::Client::Connector::Sentinel) + !!config&.sentinel? end end end diff --git a/spec/unit/stats/cleaner_spec.rb b/spec/unit/stats/cleaner_spec.rb index 89d3ee49..d600a7d4 100644 --- a/spec/unit/stats/cleaner_spec.rb +++ b/spec/unit/stats/cleaner_spec.rb @@ -76,10 +76,10 @@ module Stats it 'deletes only the stats of services marked to be deleted' do Cleaner.delete!(non_proxied_instances) - expect(keys_not_to_be_deleted.keys.all? { |key| storage.exists(key) }) + expect(keys_not_to_be_deleted.keys.all? { |key| storage.exists?(key) }) .to be true - expect(keys_to_be_deleted.keys.none? { |key| storage.exists(key) }) + expect(keys_to_be_deleted.keys.none? { |key| storage.exists?(key) }) .to be true end @@ -94,7 +94,7 @@ module Stats before { storage.del(redis_set_marked_to_be_deleted) } it 'does not delete any keys' do - expect(all_keys.keys.all? {|key| storage.exists(key) }) + expect(all_keys.keys.all? {|key| storage.exists?(key) }) end end @@ -120,10 +120,10 @@ module Stats it 'deletes only the stats of services marked to be deleted' do Cleaner.delete!(non_proxied_instances, log_deleted_keys: log_to) - expect(keys_not_to_be_deleted.keys.all? { |key| storage.exists(key) }) + expect(keys_not_to_be_deleted.keys.all? { |key| storage.exists?(key) }) .to be true - expect(keys_to_be_deleted.keys.none? { |key| storage.exists(key) }) + expect(keys_to_be_deleted.keys.none? { |key| storage.exists?(key) }) .to be true end @@ -180,17 +180,17 @@ module Stats it 'deletes the stats with usage 0' do Cleaner.delete_stats_keys_set_to_0(non_proxied_instances) - expect(stats_with_usage_0.keys.none? { |k| storage.exists(k) }).to be true + expect(stats_with_usage_0.keys.none? { |k| storage.exists?(k) }).to be true end it 'does not delete the stats with usage != 0' do Cleaner.delete_stats_keys_set_to_0(non_proxied_instances) - expect(stats_with_non_zero_usage.keys.all? { |k| storage.exists(k) }). to be true + expect(stats_with_non_zero_usage.keys.all? { |k| storage.exists?(k) }). to be true end it 'does not delete non-stats keys' do Cleaner.delete_stats_keys_set_to_0(non_proxied_instances) - expect(non_stats_keys.keys.all? { |k| storage.exists(k) }).to be true + expect(non_stats_keys.keys.all? { |k| storage.exists?(k) }).to be true end context 'with the option to log deleted keys enabled' do diff --git a/test/integration/authrep/basic_test.rb b/test/integration/authrep/basic_test.rb index e0422e52..a625a4f7 100644 --- a/test/integration/authrep/basic_test.rb +++ b/test/integration/authrep/basic_test.rb @@ -1003,7 +1003,7 @@ def setup assert_equal 200, last_response.status # 'Hits' was 0, so there shouldn't be a stats key for it. - hits_key_created = @storage.exists( + hits_key_created = @storage.exists?( application_key( @service_id, @application.id, diff --git a/test/integration/authrep/set_usage_test.rb b/test/integration/authrep/set_usage_test.rb index 95ccbc7e..a138913d 100644 --- a/test/integration/authrep/set_usage_test.rb +++ b/test/integration/authrep/set_usage_test.rb @@ -276,7 +276,7 @@ def setup end stats_keys = app_keys_for_all_periods(@service_id, @application.id, hits_id, current_time) - stats_keys_created = stats_keys.any? { |key| @storage.exists(key) } + stats_keys_created = stats_keys.any? { |key| @storage.exists?(key) } assert_false stats_keys_created end @@ -304,7 +304,7 @@ def setup end stats_keys = app_keys_for_all_periods(@service_id, @application.id, hits_id, current_time) - stats_keys_created = stats_keys.any? { |key| @storage.exists(key) } + stats_keys_created = stats_keys.any? { |key| @storage.exists?(key) } assert_false stats_keys_created end end diff --git a/test/integration/report_test.rb b/test/integration/report_test.rb index 6a582573..d865a59f 100644 --- a/test/integration/report_test.rb +++ b/test/integration/report_test.rb @@ -286,7 +286,7 @@ def setup assert_equal 202, last_response.status # 'Hits' was 0, so there shouldn't be a stats key for it. - hits_key_created = @storage.exists( + hits_key_created = @storage.exists?( application_key( @service_id, @application.id, @@ -331,7 +331,7 @@ def setup assert_equal 202, last_response.status stats_keys = app_keys_for_all_periods(@service_id, @application.id, hits_id, current_time) - stats_keys_created = stats_keys.any? { |key| @storage.exists(key) } + stats_keys_created = stats_keys.any? { |key| @storage.exists?(key) } assert_false stats_keys_created end @@ -371,7 +371,7 @@ def setup assert_equal 202, last_response.status stats_keys = app_keys_for_all_periods(@service_id, @application.id, hits_id, current_time) - stats_keys_created = stats_keys.any? { |key| @storage.exists(key) } + stats_keys_created = stats_keys.any? { |key| @storage.exists?(key) } assert_false stats_keys_created end diff --git a/test/test_helpers/storage.rb b/test/test_helpers/storage.rb index 43f91231..a52ca9d8 100644 --- a/test/test_helpers/storage.rb +++ b/test/test_helpers/storage.rb @@ -88,12 +88,12 @@ def flushall end end - def method_missing(m, *args, &blk) + def method_missing(m, *args, **kwargs, &blk) # define and delegate the missing method - self.class.send(:define_method, m) do |*a, &b| - inner.send(m, *a, &b) + self.class.send(:define_method, m) do |*a, **kwa, &b| + inner.send(m, *a, **kwa, &b) end - inner.send(m, *args, &blk) + inner.send(m, *args, **kwargs, &blk) end def respond_to_missing?(m) diff --git a/test/unit/storage_sync_test.rb b/test/unit/storage_sync_test.rb index 8eb9a192..5c9cf055 100644 --- a/test/unit/storage_sync_test.rb +++ b/test/unit/storage_sync_test.rb @@ -49,7 +49,7 @@ def test_sentinels_connection_string } conn = StorageSync.send :new, Storage::Helpers.config_with(config_obj) - assert_sentinel_connector(conn) + assert_sentinel_config(conn) assert_client_config(conn, url: config_obj[:url], sentinels: [{ host: '127.0.0.1', port: 26_379 }, { host: '127.0.0.1', port: 36_379 }]) @@ -62,7 +62,7 @@ def test_sentinels_connection_string_escaped } conn = StorageSync.send :new, Storage::Helpers.config_with(config_obj) - assert_sentinel_connector(conn) + assert_sentinel_config(conn) assert_client_config(conn, url: config_obj[:url], sentinels: [{ host: '127.0.0.1', port: 26_379, password: 'passw,ord' }, { host: '127.0.0.1', port: 36_379 }]) @@ -75,7 +75,7 @@ def test_sentinels_connection_array_strings } conn = StorageSync.send :new, Storage::Helpers.config_with(config_obj) - assert_sentinel_connector(conn) + assert_sentinel_config(conn) assert_client_config(conn, url: config_obj[:url], sentinels: [{ host: '127.0.0.1', port: 26_379 }, { host: '127.0.0.1', port: 36_379 }]) @@ -91,7 +91,7 @@ def test_sentinels_connection_array_hashes } conn = StorageSync.send :new, Storage::Helpers.config_with(config_obj) - assert_sentinel_connector(conn) + assert_sentinel_config(conn) assert_client_config(conn, url: config_obj[:url], sentinels: config_obj[:sentinels].compact.reject(&:empty?)) end @@ -113,7 +113,7 @@ def test_sentinels_simple_url } conn = StorageSync.send :new, Storage::Helpers.config_with(config_obj) - assert_sentinel_connector(conn) + assert_sentinel_config(conn) assert_client_config(conn, url: "redis://#{config_obj[:url]}", sentinels: [{ host: '127.0.0.1', port: 26_379 }]) end @@ -128,7 +128,7 @@ def test_sentinels_array_hashes_default_port } conn = StorageSync.send :new, Storage::Helpers.config_with(config_obj) - assert_sentinel_connector(conn) + assert_sentinel_config(conn) assert_client_config(conn, url: config_obj[:url], sentinels: [{ host: '127.0.0.1', port: default_sentinel_port }, { host: '192.168.1.1', port: default_sentinel_port }, @@ -146,7 +146,7 @@ def test_sentinels_array_strings_default_port } conn = StorageSync.send :new, Storage::Helpers.config_with(config_obj) - assert_sentinel_connector(conn) + assert_sentinel_config(conn) assert_client_config(conn, url: config_obj[:url], sentinels: [{ host: '127.0.0.2', port: default_sentinel_port }, { host: '127.0.0.1', port: default_sentinel_port }, @@ -164,7 +164,7 @@ def test_sentinels_array_hashes_password } conn = StorageSync.send :new, Storage::Helpers.config_with(config_obj) - assert_sentinel_connector(conn) + assert_sentinel_config(conn) assert_client_config(conn, url: config_obj[:url], sentinels: [{ host: '192.168.1.1', port: 3333, password: 'abc' }, { host: '192.168.1.2', port: 4444 }, @@ -180,7 +180,7 @@ def test_sentinels_array_strings_password } conn = StorageSync.send :new, Storage::Helpers.config_with(config_obj) - assert_sentinel_connector(conn) + assert_sentinel_config(conn) assert_client_config(conn, url: config_obj[:url], sentinels: [{ host: '192.168.1.1', port: 3333, password: 'abc' }, { host: '192.168.1.2', port: 4444 }, @@ -196,7 +196,7 @@ def test_sentinels_correct_role } conn = StorageSync.send :new, Storage::Helpers.config_with(config_obj) - assert_sentinel_connector(conn) + assert_sentinel_config(conn) assert_client_config(conn, url: config_obj[:url], sentinels: [{ host: '127.0.0.1', port: 26_379 }], role: role) @@ -255,16 +255,19 @@ def assert_connection(client) assert_equal 'bar', client.get('foo') end - def assert_sentinel_connector(client) - connector = client.instance_variable_get(:@inner).instance_variable_get(:@client).instance_variable_get(:@connector) - assert_instance_of Redis::Client::Connector::Sentinel, connector + def assert_sentinel_config(client) + config = client.instance_variable_get(:@inner).instance_variable_get(:@client).instance_variable_get(:@config) + assert config.sentinel? end def assert_client_config(conn, url:, **conf) - client = conn.instance_variable_get(:@inner).instance_variable_get(:@client) - assert_equal client.options[:url], url - conf.each do |k, v| - assert_equal v, client.options[k] + config = conn.instance_variable_get(:@inner).instance_variable_get(:@client).instance_variable_get(:@config) + assert_equal URI(url).host, config.name + assert_equal conf[:role] || :master, config.instance_variable_get(:@role) + conf[:sentinels].each_with_index do |s, i| + assert_equal s[:host], config.sentinels[i].host + assert_equal s[:port], config.sentinels[i].port + assert_equal s[:password], config.sentinels[i].password end end From b99e3075ca98f84c1aa0b68b556e27f45c688ccc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Wed, 25 Oct 2023 13:10:59 +0200 Subject: [PATCH 06/54] Accept new parameters and ENV variables --- docs/configuration.md | 70 +++++++++++++++++++++++++++ lib/3scale/backend/configuration.rb | 7 ++- lib/3scale/backend/storage_helpers.rb | 37 ++++++++++++-- openshift/3scale_backend.conf | 14 ++++++ 4 files changed, 121 insertions(+), 7 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 0ea87ad9..0da6da88 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -26,6 +26,41 @@ variables. - Applies to: listener, worker, cron. - Format: string. +### CONFIG_REDIS_USERNAME + +- Redis ACL user name +- Optional. Defaults to empty. +- Applies to: listener, worker, cron. +- Format: string. + +### CONFIG_REDIS_PASSWORD + +- Redis ACL password +- Optional. Defaults to empty. +- Applies to: listener, worker, cron. +- Format: string. + +### CONFIG_REDIS_CA_FILE + +- Certification authority to validate Redis server TLS connections with +- Optional. Defaults to empty. +- Applies to: listener, worker, cron. +- Format: path to file as string. + +### CONFIG_REDIS_CERT + +- User certificate to connect to Redis through TLS +- Optional. Defaults to empty. +- Applies to: listener, worker, cron. +- Format: path to file as string. + +### CONFIG_REDIS_PRIVATE_KEY + +- User key to connect to Redis through TLS +- Optional. Defaults to empty. +- Applies to: listener, worker, cron. +- Format: path to file as string. + ### CONFIG_REDIS_SENTINEL_HOSTS - URL of Redis sentinels. @@ -80,6 +115,41 @@ sentinels. - Applies to: listener, worker, cron. - Format: string. +### CONFIG_QUEUES_USERNAME + +- Redis ACL user name +- Optional. Defaults to empty. +- Applies to: listener, worker, cron. +- Format: string. + +### CONFIG_QUEUES_PASSWORD + +- Redis ACL password +- Optional. Defaults to empty. +- Applies to: listener, worker, cron. +- Format: string. + +### CONFIG_QUEUES_CA_FILE + +- Certification authority certificate Redis should trust to accept TLS connections +- Optional. Defaults to empty. +- Applies to: listener, worker, cron. +- Format: path to file as string. + +### CONFIG_QUEUES_CERT + +- User certificate to connect to Redis through TLS +- Optional. Defaults to empty. +- Applies to: listener, worker, cron. +- Format: path to file as string. + +### CONFIG_QUEUES_PRIVATE_KEY + +- User key to connect to Redis through TLS +- Optional. Defaults to empty. +- Applies to: listener, worker, cron. +- Format: path to file as string. + ### CONFIG_QUEUES_SENTINEL_HOSTS - URL of Redis sentinels. diff --git a/lib/3scale/backend/configuration.rb b/lib/3scale/backend/configuration.rb index 3b89ef55..a33250dc 100644 --- a/lib/3scale/backend/configuration.rb +++ b/lib/3scale/backend/configuration.rb @@ -38,11 +38,10 @@ def parse_int(value, default) config.workers_logger_formatter = :text # Add configuration sections - config.add_section(:queues, :master_name, :sentinels, :role, + config.add_section(:queues, :master_name, :username, :password, :ssl_params, :sentinels, :role, :connect_timeout, :read_timeout, :write_timeout, :max_connections) - config.add_section(:redis, :url, :proxy, :sentinels, :role, - :connect_timeout, :read_timeout, :write_timeout, :max_connections, - :async) + config.add_section(:redis, :url, :proxy, :username, :password, :ssl_params, :sentinels, :role, + :connect_timeout, :read_timeout, :write_timeout, :max_connections, :async) config.add_section(:hoptoad, :service, :api_key) config.add_section(:internal_api, :user, :password) config.add_section(:master, :metrics) diff --git a/lib/3scale/backend/storage_helpers.rb b/lib/3scale/backend/storage_helpers.rb index 02d0e0ed..d7c93ac8 100644 --- a/lib/3scale/backend/storage_helpers.rb +++ b/lib/3scale/backend/storage_helpers.rb @@ -62,7 +62,7 @@ class << self # CONN_WHITELIST - Connection options that can be specified in config # Note: we don't expose reconnect_attempts until the bug above is fixed CONN_WHITELIST = [ - :connect_timeout, :read_timeout, :write_timeout, :max_connections + :connect_timeout, :read_timeout, :write_timeout, :max_connections, :username, :password, :ssl_params ].freeze private_constant :CONN_WHITELIST @@ -99,8 +99,7 @@ def config_with(config, end.merge(options) cfg_with_sentinels = cfg_sentinels_handler cfg - - defaults.merge(ensure_url_param(cfg_with_sentinels)) + cfg_defaults_handler cfg_with_sentinels, defaults end private @@ -241,6 +240,38 @@ def cfg_sentinels_handler(options) options end + # The new Redis client accepts either `:url` or `:path`, but not both. + # In the case of a path, Redis expects it to not include the `unix://` prefix. + # On the other hand, Apisonator accepts only `:url`, for both Sockets and TCP connections. + # For paths, Apisonator expects it to be given as a URL using the `unix://` scheme. + # + # This method handles the conversion. + def cfg_unix_path_handler(options) + if options.key? :path + options.delete(:url) + return options + end + + if options[:url].start_with? "unix://" + options[:path] = options.delete(:url).delete_prefix("unix://") + end + + options + end + + # This ensures some default values are valid for the redis client. + # In particular: + # + # - The :url key is always present + # - Except when connecting to a unix socket + # - :max_connections is only present for async mode + def cfg_defaults_handler(options, defaults) + cfg_with_defaults = defaults.merge(ensure_url_param(options)) + cfg_with_defaults = cfg_unix_path_handler(cfg_with_defaults) + cfg_with_defaults.delete(:max_connections) unless options[:async] + cfg_with_defaults + end + # helper to convert a sentinel object to a Hash def sentinel_to_hash(sentinel) return if sentinel.nil? diff --git a/openshift/3scale_backend.conf b/openshift/3scale_backend.conf index 1e5de940..37eb41b8 100644 --- a/openshift/3scale_backend.conf +++ b/openshift/3scale_backend.conf @@ -22,6 +22,13 @@ ThreeScale::Backend.configure do |config| config.internal_api.user = "#{ENV['CONFIG_INTERNAL_API_USER']}" config.internal_api.password = "#{ENV['CONFIG_INTERNAL_API_PASSWORD']}" config.queues.master_name = "#{ENV['CONFIG_QUEUES_MASTER_NAME']}" + config.queues.username = "#{ENV['CONFIG_QUEUES_USERNAME']}" + config.queues.password = "#{ENV['CONFIG_QUEUES_PASSWORD']}" + config.queues.ssl_params = { + ca_file: "#{ENV['CONFIG_QUEUES_CA_FILE]}", + cert: "#{ENV['CONFIG_QUEUES_CERT]}", + key: "#{ENV['CONFIG_QUEUES_PRIVATE_KEY]}" + } config.queues.sentinels = "#{ENV['CONFIG_QUEUES_SENTINEL_HOSTS'] && !ENV['CONFIG_QUEUES_SENTINEL_HOSTS'].empty? ? ENV['CONFIG_QUEUES_SENTINEL_HOSTS'] : ENV['SENTINEL_HOSTS']}" config.queues.role = "#{ENV['CONFIG_QUEUES_SENTINEL_ROLE']}".to_sym config.queues.connect_timeout = parse_int_env('CONFIG_QUEUES_CONNECT_TIMEOUT') @@ -29,6 +36,13 @@ ThreeScale::Backend.configure do |config| config.queues.write_timeout = parse_int_env('CONFIG_QUEUES_WRITE_TIMEOUT') config.queues.max_connections = parse_int_env('CONFIG_QUEUES_MAX_CONNS') config.redis.proxy = "#{ENV['CONFIG_REDIS_PROXY']}" + config.redis.username = "#{ENV['CONFIG_REDIS_USERNAME']}" + config.redis.password = "#{ENV['CONFIG_REDIS_PASSWORD']}" + config.redis.ssl_params = { + ca_file: "#{ENV['CONFIG_REDIS_CA_FILE]}", + cert: "#{ENV['CONFIG_REDIS_CERT]}", + key: "#{ENV['CONFIG_REDIS_PRIVATE_KEY]}" + } config.redis.sentinels = "#{ENV['CONFIG_REDIS_SENTINEL_HOSTS']}" config.redis.role = "#{ENV['CONFIG_REDIS_SENTINEL_ROLE']}".to_sym config.redis.connect_timeout = parse_int_env('CONFIG_REDIS_CONNECT_TIMEOUT') From f0418d72ba277f7635097a5ada19e1fafae4aa26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Mon, 20 Nov 2023 15:06:44 +0100 Subject: [PATCH 07/54] Update ENV vars descriptions --- docs/configuration.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 0da6da88..24fdbe70 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -49,14 +49,14 @@ variables. ### CONFIG_REDIS_CERT -- User certificate to connect to Redis through TLS +- The path to the client SSL certificate - Optional. Defaults to empty. - Applies to: listener, worker, cron. - Format: path to file as string. ### CONFIG_REDIS_PRIVATE_KEY -- User key to connect to Redis through TLS +- The path to the client SSL private key - Optional. Defaults to empty. - Applies to: listener, worker, cron. - Format: path to file as string. From b7394621eabc82462688f9b5fcf1abbb42ef6821 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Mon, 20 Nov 2023 15:08:06 +0100 Subject: [PATCH 08/54] Gemfile: require redis ~> 5.0 --- Gemfile | 2 +- Gemfile.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile b/Gemfile index 4c671813..024ba242 100644 --- a/Gemfile +++ b/Gemfile @@ -55,7 +55,7 @@ gem 'daemons', '= 1.2.4' # Production gems gem 'rake', '~> 13.0' gem 'builder', '= 3.2.3' -gem 'redis', '~> 5' +gem 'redis', '~> 5.0' gem 'resque', '~> 2.6.0' gem 'redis-namespace', '~>1.8' gem 'rack', '~> 2.2.6' diff --git a/Gemfile.lock b/Gemfile.lock index c912a6ec..dc15048f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -272,7 +272,7 @@ DEPENDENCIES rack (~> 2.2.6) rack-test (= 0.8.2) rake (~> 13.0) - redis (~> 5) + redis (~> 5.0) redis-namespace (~> 1.8) resque (~> 2.6.0) resque_spec (~> 0.17.0) From c91b00aeaaf10f5786a3022be7ad83d8a63fca64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Tue, 21 Nov 2023 17:28:58 +0100 Subject: [PATCH 09/54] Create a separate context for pipelines --- lib/3scale/backend/storage_async.rb | 1 + lib/3scale/backend/storage_async/client.rb | 136 +------------------ lib/3scale/backend/storage_async/methods.rb | 123 +++++++++++++++++ lib/3scale/backend/storage_async/pipeline.rb | 7 + 4 files changed, 135 insertions(+), 132 deletions(-) create mode 100644 lib/3scale/backend/storage_async/methods.rb diff --git a/lib/3scale/backend/storage_async.rb b/lib/3scale/backend/storage_async.rb index 0bde5e3f..54ab82a7 100644 --- a/lib/3scale/backend/storage_async.rb +++ b/lib/3scale/backend/storage_async.rb @@ -1,3 +1,4 @@ +require '3scale/backend/storage_async/methods' require '3scale/backend/storage_async/client' require '3scale/backend/storage_async/pipeline' require '3scale/backend/storage_async/resque_extensions' diff --git a/lib/3scale/backend/storage_async/client.rb b/lib/3scale/backend/storage_async/client.rb index 73045bb8..209ad6cb 100644 --- a/lib/3scale/backend/storage_async/client.rb +++ b/lib/3scale/backend/storage_async/client.rb @@ -13,6 +13,7 @@ module StorageAsync # the Storage instance behaves likes the redis-rb client. class Client include Configurable + include Methods DEFAULT_HOST = 'localhost'.freeze private_constant :DEFAULT_HOST @@ -49,120 +50,9 @@ def initialize(opts) @redis_async = Async::Redis::Client.new( endpoint, limit: opts[:max_connections] ) - @building_pipeline = false end - # Now we are going to define the methods to run redis commands - # following the interface of the redis-rb lib. - # - # These are the different cases: - # 1) Methods that can be called directly. For example SET: - # @redis_async.call('SET', some_key) - # 2) Methods that need to be "boolified". These are methods for which - # redis-rb returns a boolean, but redis just returns an integer. - # For example, Redis returns 0 or 1 for the EXISTS command, but - # redis-rb transforms that into a boolean. - # 3) There are a few methods that need to be treated differently and - # do not fit in any of the previous categories. For example, SSCAN - # which accepts a hash of options in redis-rb. - # - # All of this might be simplified a bit in the future using the - # "methods" in async-redis - # https://github.com/socketry/async-redis/tree/master/lib/async/redis/methods - # but there are some commands missing, so for now, that's not an option. - - METHODS_TO_BE_CALLED_DIRECTLY = [ - :del, - :exists, - :expire, - :expireat, - :flushdb, - :get, - :hset, - :hmget, - :incr, - :incrby, - :keys, - :llen, - :lpop, - :lpush, - :lrange, - :ltrim, - :mget, - :ping, - :rpush, - :sadd, - :scard, - :setex, - :smembers, - :srem, - :sunion, - :ttl, - :zcard, - :zrangebyscore, - :zremrangebyscore, - :zrevrange - ].freeze - private_constant :METHODS_TO_BE_CALLED_DIRECTLY - - METHODS_TO_BE_CALLED_DIRECTLY.each do |method| - define_method(method) do |*args| - @redis_async.call(method, *args.flatten) - end - end - - METHODS_TO_BOOLIFY = [ - :exists?, - :sismember, - :sadd?, - :srem?, - :zadd - ].freeze - private_constant :METHODS_TO_BOOLIFY - - METHODS_TO_BOOLIFY.each do |method| - command = method.to_s.delete('?') - define_method(method) do |*args| - @redis_async.call(command, *args.flatten) > 0 - end - end - - def blpop(*args) - call_args = ['BLPOP'] + args - - # redis-rb accepts a Hash as last arg that can contain :timeout. - if call_args.last.is_a? Hash - timeout = call_args.pop[:timeout] - call_args << timeout - end - - @redis_async.call(*call_args.flatten) - end - - def set(key, val, opts = {}) - args = ['SET', key, val] - - args += ['EX', opts[:ex]] if opts[:ex] - args << 'NX' if opts[:nx] - - @redis_async.call(*args) - end - - def sscan(key, cursor, opts = {}) - args = ['SSCAN', key, cursor] - - args += ['MATCH', opts[:match]] if opts[:match] - args += ['COUNT', opts[:count]] if opts[:count] - - @redis_async.call(*args) - end - - def scan(cursor, opts = {}) - args = ['SCAN', cursor] - - args += ['MATCH', opts[:match]] if opts[:match] - args += ['COUNT', opts[:count]] if opts[:count] - + def call(*args) @redis_async.call(*args) end @@ -178,27 +68,9 @@ def pipelined(&block) # There's an important limitation: this assumes that the fiber will # not yield in the block. - # When running a nested pipeline, we just need to continue - # accumulating commands. - if @building_pipeline - block.call self - return - end - - @building_pipeline = true - - original = @redis_async pipeline = Pipeline.new - @redis_async = pipeline - - begin - block.call self - ensure - @redis_async = original - @building_pipeline = false - end - - pipeline.run(original) + block.call pipeline + pipeline.run(@redis_async) end def close diff --git a/lib/3scale/backend/storage_async/methods.rb b/lib/3scale/backend/storage_async/methods.rb new file mode 100644 index 00000000..d07ba3df --- /dev/null +++ b/lib/3scale/backend/storage_async/methods.rb @@ -0,0 +1,123 @@ +# frozen_string_literal: true + +module ThreeScale + module Backend + module StorageAsync + module Methods + # Now we are going to define the methods to run redis commands + # following the interface of the redis-rb lib. + # + # These are the different cases: + # 1) Methods that can be called directly. For example SET: + # @redis_async.call('SET', some_key) + # 2) Methods that need to be "boolified". These are methods for which + # redis-rb returns a boolean, but redis just returns an integer. + # For example, Redis returns 0 or 1 for the EXISTS command, but + # redis-rb transforms that into a boolean. + # 3) There are a few methods that need to be treated differently and + # do not fit in any of the previous categories. For example, SSCAN + # which accepts a hash of options in redis-rb. + # + # All of this might be simplified a bit in the future using the + # "methods" in async-redis + # https://github.com/socketry/async-redis/tree/master/lib/async/redis/methods + # but there are some commands missing, so for now, that's not an option. + + METHODS_TO_BE_CALLED_DIRECTLY = [ + :del, + :exists, + :expire, + :expireat, + :flushdb, + :get, + :hset, + :hmget, + :incr, + :incrby, + :keys, + :llen, + :lpop, + :lpush, + :lrange, + :ltrim, + :mget, + :ping, + :rpush, + :sadd, + :scard, + :setex, + :smembers, + :srem, + :sunion, + :ttl, + :zcard, + :zrangebyscore, + :zremrangebyscore, + :zrevrange + ].freeze + private_constant :METHODS_TO_BE_CALLED_DIRECTLY + + METHODS_TO_BE_CALLED_DIRECTLY.each do |method| + define_method(method) do |*args| + call(method, *args.flatten) + end + end + + METHODS_TO_BOOLIFY = [ + :exists?, + :sismember, + :sadd?, + :srem?, + :zadd + ].freeze + private_constant :METHODS_TO_BOOLIFY + + METHODS_TO_BOOLIFY.each do |method| + command = method.to_s.delete('?') + define_method(method) do |*args| + call(command, *args.flatten) > 0 + end + end + + def blpop(*args) + call_args = ['BLPOP'] + args + + # redis-rb accepts a Hash as last arg that can contain :timeout. + if call_args.last.is_a? Hash + timeout = call_args.pop[:timeout] + call_args << timeout + end + + call(*call_args.flatten) + end + + def set(key, val, opts = {}) + args = ['SET', key, val] + + args += ['EX', opts[:ex]] if opts[:ex] + args << 'NX' if opts[:nx] + + call(*args) + end + + def sscan(key, cursor, opts = {}) + args = ['SSCAN', key, cursor] + + args += ['MATCH', opts[:match]] if opts[:match] + args += ['COUNT', opts[:count]] if opts[:count] + + call(*args) + end + + def scan(cursor, opts = {}) + args = ['SCAN', cursor] + + args += ['MATCH', opts[:match]] if opts[:match] + args += ['COUNT', opts[:count]] if opts[:count] + + call(*args) + end + end + end + end +end diff --git a/lib/3scale/backend/storage_async/pipeline.rb b/lib/3scale/backend/storage_async/pipeline.rb index 5df5e9e9..cd14dd98 100644 --- a/lib/3scale/backend/storage_async/pipeline.rb +++ b/lib/3scale/backend/storage_async/pipeline.rb @@ -5,6 +5,7 @@ module StorageAsync # This class accumulates commands and sends several of them in a single # request, instead of sending them one by one. class Pipeline + include Methods Error = Class.new StandardError @@ -55,6 +56,12 @@ def call(*args) 1 end + def pipelined(&block) + # When running a nested pipeline, we just need to continue + # accumulating commands. + block.call self + end + # Send to redis all the accumulated commands. # Returns an array with the result for each command in the same order # that they added with .call(). From c94bd275d67261aff1cd27ed42bf12e3a3bff7ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Wed, 22 Nov 2023 11:17:44 +0100 Subject: [PATCH 10/54] Thread-isolate redis client in async mode The block seemed to be caused by different threads sharing the redis client. async-redis calls `Async` to run the commands, and Async uses Fibers internally, so sharing the redis client between threads led to sharing fibers across threads, which is unsupported. --- lib/3scale/backend/storage_async/client.rb | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/3scale/backend/storage_async/client.rb b/lib/3scale/backend/storage_async/client.rb index 209ad6cb..75a0a077 100644 --- a/lib/3scale/backend/storage_async/client.rb +++ b/lib/3scale/backend/storage_async/client.rb @@ -1,3 +1,4 @@ +require 'concurrent' require 'async/io' require 'async/redis/client' @@ -47,13 +48,13 @@ def initialize(opts) port ||= DEFAULT_PORT endpoint = Async::IO::Endpoint.tcp(host, port) - @redis_async = Async::Redis::Client.new( + @redis_async = Concurrent::ThreadLocalVar.new{ Async::Redis::Client.new( endpoint, limit: opts[:max_connections] - ) + )} end def call(*args) - @redis_async.call(*args) + @redis_async.value.call(*args) end # This method allows us to send pipelines like this: @@ -70,11 +71,11 @@ def pipelined(&block) pipeline = Pipeline.new block.call pipeline - pipeline.run(@redis_async) + pipeline.run(@redis_async.value) end def close - @redis_async.close + @redis_async.value.close end end From 8d8c4af5753c7bea0779143f616afe122a6d6e69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Thu, 23 Nov 2023 09:54:03 +0100 Subject: [PATCH 11/54] Implement SSL and ACL in async mode --- lib/3scale/backend/storage_async/client.rb | 82 +++++++++++++++++----- 1 file changed, 64 insertions(+), 18 deletions(-) diff --git a/lib/3scale/backend/storage_async/client.rb b/lib/3scale/backend/storage_async/client.rb index 75a0a077..f24ca248 100644 --- a/lib/3scale/backend/storage_async/client.rb +++ b/lib/3scale/backend/storage_async/client.rb @@ -16,15 +16,6 @@ class Client include Configurable include Methods - DEFAULT_HOST = 'localhost'.freeze - private_constant :DEFAULT_HOST - - DEFAULT_PORT = 22121 - private_constant :DEFAULT_PORT - - HOST_PORT_REGEX = /redis:\/\/(.*):(\d+)/ - private_constant :HOST_PORT_REGEX - class << self attr_writer :instance @@ -43,14 +34,7 @@ def instance(reset = false) end def initialize(opts) - host, port = opts[:url].match(HOST_PORT_REGEX).captures if opts[:url] - host ||= DEFAULT_HOST - port ||= DEFAULT_PORT - - endpoint = Async::IO::Endpoint.tcp(host, port) - @redis_async = Concurrent::ThreadLocalVar.new{ Async::Redis::Client.new( - endpoint, limit: opts[:max_connections] - )} + @redis_async = Concurrent::ThreadLocalVar.new{ initialize_client(opts) } end def call(*args) @@ -77,8 +61,70 @@ def pipelined(&block) def close @redis_async.value.close end - end + private + + DEFAULT_SCHEME = 'redis' + DEFAULT_HOST = 'localhost'.freeze + DEFAULT_PORT = 22121 + + # Custom Redis Protocol class which sends the AUTH command on every new connection + # to authenticate before sending any other command. + class AuthenticatedRESP2 + def initialize(credentials) + @credentials = credentials + end + + def client(stream) + client = Async::Redis::Protocol::RESP2.client(stream) + + client.write_request(["AUTH", *@credentials]) + client.read_response # Ignore response. + + client + end + end + + def initialize_client(opts) + endpoint = make_redis_endpoint(opts) + protocol = make_redis_protocol(opts) + Async::Redis::Client.new(endpoint, protocol: protocol, limit: opts[:max_connections]) + end + + # Authenticated RESP2 if credentials are provided, RESP2 otherwise + def make_redis_protocol(opts) + uri = URI(opts[:url] || "") + credentials = [ uri.user || opts[:username], uri.password || opts[:password]] + + if credentials.any? + AuthenticatedRESP2.new(credentials) + else + Async::Redis::Protocol::RESP2 + end + end + + # SSL endpoint if scheme is `rediss:`, TCP endpoint otherwise. + # Note: Unix socket endpoint is not supported in async mode + def make_redis_endpoint(opts) + uri = URI(opts[:url] || "") + scheme = uri.scheme || DEFAULT_SCHEME + host = uri.host || DEFAULT_HOST + port = uri.port || DEFAULT_PORT + + tcp_endpoint = Async::IO::Endpoint.tcp(host, port) + + case scheme + when 'redis' + tcp_endpoint + when 'rediss' + ssl_context = OpenSSL::SSL::SSLContext.new + ssl_context.set_params(opts[:ssl_params]) + Async::IO::SSLEndpoint.new(tcp_endpoint, ssl_context: ssl_context) + else + raise ArgumentError + end + end + end end end end From d5fd9008020f001b92de6ce7fb5e900cc37c0952 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Mon, 27 Nov 2023 14:37:22 +0100 Subject: [PATCH 12/54] Fix config file missing quotes --- openshift/3scale_backend.conf | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/openshift/3scale_backend.conf b/openshift/3scale_backend.conf index 37eb41b8..02fb114d 100644 --- a/openshift/3scale_backend.conf +++ b/openshift/3scale_backend.conf @@ -25,9 +25,9 @@ ThreeScale::Backend.configure do |config| config.queues.username = "#{ENV['CONFIG_QUEUES_USERNAME']}" config.queues.password = "#{ENV['CONFIG_QUEUES_PASSWORD']}" config.queues.ssl_params = { - ca_file: "#{ENV['CONFIG_QUEUES_CA_FILE]}", - cert: "#{ENV['CONFIG_QUEUES_CERT]}", - key: "#{ENV['CONFIG_QUEUES_PRIVATE_KEY]}" + ca_file: "#{ENV['CONFIG_QUEUES_CA_FILE']}", + cert: "#{ENV['CONFIG_QUEUES_CERT']}", + key: "#{ENV['CONFIG_QUEUES_PRIVATE_KEY']}" } config.queues.sentinels = "#{ENV['CONFIG_QUEUES_SENTINEL_HOSTS'] && !ENV['CONFIG_QUEUES_SENTINEL_HOSTS'].empty? ? ENV['CONFIG_QUEUES_SENTINEL_HOSTS'] : ENV['SENTINEL_HOSTS']}" config.queues.role = "#{ENV['CONFIG_QUEUES_SENTINEL_ROLE']}".to_sym @@ -39,9 +39,9 @@ ThreeScale::Backend.configure do |config| config.redis.username = "#{ENV['CONFIG_REDIS_USERNAME']}" config.redis.password = "#{ENV['CONFIG_REDIS_PASSWORD']}" config.redis.ssl_params = { - ca_file: "#{ENV['CONFIG_REDIS_CA_FILE]}", - cert: "#{ENV['CONFIG_REDIS_CERT]}", - key: "#{ENV['CONFIG_REDIS_PRIVATE_KEY]}" + ca_file: "#{ENV['CONFIG_REDIS_CA_FILE']}", + cert: "#{ENV['CONFIG_REDIS_CERT']}", + key: "#{ENV['CONFIG_REDIS_PRIVATE_KEY']}" } config.redis.sentinels = "#{ENV['CONFIG_REDIS_SENTINEL_HOSTS']}" config.redis.role = "#{ENV['CONFIG_REDIS_SENTINEL_ROLE']}".to_sym From 5e8f66c565874d49235f3467ab7e07880c4118cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Fri, 1 Dec 2023 13:12:29 +0100 Subject: [PATCH 13/54] Revert async gems upgrade --- Gemfile | 4 ++-- Gemfile.lock | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Gemfile b/Gemfile index 024ba242..446ef959 100644 --- a/Gemfile +++ b/Gemfile @@ -64,8 +64,8 @@ gem 'sinatra-contrib', '~> 2.2.4' # Optional external error logging services gem 'bugsnag', '~> 6', require: nil gem 'yabeda-prometheus', '~> 0.5.0' -gem 'async-redis', '~> 0.8' -gem 'async-pool', '~> 0.4' +gem 'async-redis', '~> 0.7.0' +gem 'async-pool', '~> 0.3.12' gem 'falcon', '~> 0.35' gem 'webrick', '~> 1.8' diff --git a/Gemfile.lock b/Gemfile.lock index dc15048f..a2a0b800 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -44,15 +44,15 @@ GEM traces (>= 0.8.0) async-http-cache (0.4.3) async-http (~> 0.56) - async-io (1.36.0) + async-io (1.34.3) async - async-pool (0.4.0) + async-pool (0.3.12) async (>= 1.25) - async-redis (0.8.0) + async-redis (0.7.0) async (>= 1.8, < 3.0) async-io (~> 1.10) async-pool (~> 0.2) - protocol-redis (~> 0.8.0) + protocol-redis (~> 0.6.0) async-rspec (1.17.0) rspec (~> 3.0) rspec-files (~> 1.0) @@ -140,7 +140,7 @@ GEM protocol-rack (0.2.4) protocol-http (~> 0.23) rack (>= 1.0) - protocol-redis (0.8.0) + protocol-redis (0.6.1) pry (0.14.0) coderay (~> 1.1) method_source (~> 1.0) @@ -250,8 +250,8 @@ PLATFORMS DEPENDENCIES apisonator! - async-pool (~> 0.4) - async-redis (~> 0.8) + async-pool (~> 0.3.12) + async-redis (~> 0.7.0) async-rspec benchmark-ips (~> 2.7.2) bugsnag (~> 6) From 8c281104a65f4cdf3115e4d3ed67355cf0c9a90a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Thu, 14 Dec 2023 17:18:45 +0100 Subject: [PATCH 14/54] Remove thread-safety for the async client We don't need it in production --- lib/3scale/backend/storage_async/client.rb | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/3scale/backend/storage_async/client.rb b/lib/3scale/backend/storage_async/client.rb index f24ca248..db38c2a9 100644 --- a/lib/3scale/backend/storage_async/client.rb +++ b/lib/3scale/backend/storage_async/client.rb @@ -1,4 +1,3 @@ -require 'concurrent' require 'async/io' require 'async/redis/client' @@ -34,11 +33,11 @@ def instance(reset = false) end def initialize(opts) - @redis_async = Concurrent::ThreadLocalVar.new{ initialize_client(opts) } + @redis_async = initialize_client(opts) end def call(*args) - @redis_async.value.call(*args) + @redis_async.call(*args) end # This method allows us to send pipelines like this: @@ -55,11 +54,11 @@ def pipelined(&block) pipeline = Pipeline.new block.call pipeline - pipeline.run(@redis_async.value) + pipeline.run(@redis_async) end def close - @redis_async.value.close + @redis_async.close end private From cdc25bb790695ea7e6f0aa03fd76282f161fc19f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Thu, 14 Dec 2023 17:19:31 +0100 Subject: [PATCH 15/54] Fix worker_async_spec test suite Avoid sharing the redis client between threads --- spec/integration/worker_async_spec.rb | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/spec/integration/worker_async_spec.rb b/spec/integration/worker_async_spec.rb index 4ed7f1e5..4dbe0855 100644 --- a/spec/integration/worker_async_spec.rb +++ b/spec/integration/worker_async_spec.rb @@ -5,6 +5,9 @@ module ThreeScale module Backend + + DEFAULT_SERVER = '127.0.0.1:22121'.freeze + context 'when there are jobs enqueued' do let(:provider_key) { 'a_provider_key' } let(:service_id) { 'a_service_id' } @@ -60,7 +63,10 @@ module Backend # kind of error. t_start = Time.now - while Storage.instance.get(stats_key).to_i < n_reports + storage = Redis.new(Storage::Helpers.config_with( + ThreeScale::Backend.configuration.redis, options: { default_url: "#{DEFAULT_SERVER}" } + )) + while storage.get(stats_key).to_i < n_reports if Time.now - t_start > 10 raise 'The worker is taking too much to process the jobs' end @@ -72,7 +78,7 @@ module Backend worker_thread.join - expect(Storage.instance.get(stats_key).to_i).to eq n_reports + expect(storage.get(stats_key).to_i).to eq n_reports end end end From e928e848bf2681e5b6c7e7e8ff7da9cda9347cd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Wed, 20 Dec 2023 13:23:28 +0100 Subject: [PATCH 16/54] Fix TLS connection in async mode --- lib/3scale/backend/storage_async/client.rb | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/3scale/backend/storage_async/client.rb b/lib/3scale/backend/storage_async/client.rb index db38c2a9..a411cab9 100644 --- a/lib/3scale/backend/storage_async/client.rb +++ b/lib/3scale/backend/storage_async/client.rb @@ -117,12 +117,24 @@ def make_redis_endpoint(opts) tcp_endpoint when 'rediss' ssl_context = OpenSSL::SSL::SSLContext.new - ssl_context.set_params(opts[:ssl_params]) + ssl_context.set_params(format_ssl_params(opts[:ssl_params])) Async::IO::SSLEndpoint.new(tcp_endpoint, ssl_context: ssl_context) else raise ArgumentError end end + + def format_ssl_params(ssl_params) + cert = ssl_params[:cert].to_s.strip + key = ssl_params[:key].to_s.strip + return ssl_params if cert.empty? && key.empty? + + updated_ssl_params = ssl_params.dup + updated_ssl_params[:cert] = OpenSSL::X509::Certificate.new(File.read(cert)) + updated_ssl_params[:key] = OpenSSL::PKey.read(File.read(key)) + + updated_ssl_params + end end end end From b7ffd43cf589183e5ed9c4cb3d0cbd5c3df93f4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Tue, 19 Dec 2023 17:29:48 +0100 Subject: [PATCH 17/54] Functions: start a redis TLS server For tests purposes --- .gitleaks.toml | 8 +++++ script/config/ca-root-cert.pem | 33 ++++++++++++++++++ script/config/ca-root-key.pem | 52 ++++++++++++++++++++++++++++ script/config/redis-client.crt | 31 +++++++++++++++++ script/config/redis-client.key | 52 ++++++++++++++++++++++++++++ script/config/redis-dsa.crt | 53 +++++++++++++++++++++++++++++ script/config/redis-dsa.pem | 26 ++++++++++++++ script/config/redis-ec.crt | 21 ++++++++++++ script/config/redis-ec.key | 5 +++ script/config/redis-server-acl.conf | 1 + script/config/redis-server.crt | 31 +++++++++++++++++ script/config/redis-server.key | 52 ++++++++++++++++++++++++++++ script/lib/functions | 14 ++++++-- 13 files changed, 377 insertions(+), 2 deletions(-) create mode 100644 .gitleaks.toml create mode 100644 script/config/ca-root-cert.pem create mode 100644 script/config/ca-root-key.pem create mode 100644 script/config/redis-client.crt create mode 100644 script/config/redis-client.key create mode 100644 script/config/redis-dsa.crt create mode 100644 script/config/redis-dsa.pem create mode 100644 script/config/redis-ec.crt create mode 100644 script/config/redis-ec.key create mode 100644 script/config/redis-server-acl.conf create mode 100644 script/config/redis-server.crt create mode 100644 script/config/redis-server.key diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 00000000..e1120f0a --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,8 @@ +[allowlist] + description = "Global Allowlist" + + # Ignore based on any subset of the file path + paths = [ + # Ignore all fake private keys, they are for tests + '''script\/config\/.+\.(pem|key)$''' + ] diff --git a/script/config/ca-root-cert.pem b/script/config/ca-root-cert.pem new file mode 100644 index 00000000..eeb3e2a0 --- /dev/null +++ b/script/config/ca-root-cert.pem @@ -0,0 +1,33 @@ +-----BEGIN CERTIFICATE----- +MIIFnzCCA4egAwIBAgIUAJ8jy2MbK36XVVuAZHMryfbJRLwwDQYJKoZIhvcNAQEL +BQAwXjELMAkGA1UEBhMCWFgxFTATBgNVBAcMDERlZmF1bHQgQ2l0eTEPMA0GA1UE +CgwGUmVkSGF0MQ8wDQYDVQQLDAYzc2NhbGUxFjAUBgNVBAMMDUFwaXNvbmF0b3Ig +Q0EwIBcNMjMxMjE5MTQwMjU2WhgPMjA1MTA1MDYxNDAyNTZaMF4xCzAJBgNVBAYT +AlhYMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxDzANBgNVBAoMBlJlZEhhdDEPMA0G +A1UECwwGM3NjYWxlMRYwFAYDVQQDDA1BcGlzb25hdG9yIENBMIICIjANBgkqhkiG +9w0BAQEFAAOCAg8AMIICCgKCAgEA3JCowOXBo9q+DlgYEnPsnmnCVQPGuLsP7qU7 +1lmo0+7y/T9Tg+E1pOgiJVlzYX8P+bGmAnNhJdBDONdgY0b2zAjFfGOngC2eDRmJ +CR56OccubYVbdOt/2HkMNbvlORdOutwBe2BZeLhQbQ0+znW7WN8A6sh7hPtfZVJD +fAZ0W6WQFCwfDW64njZP5/9sfvQm8pRmLrNykPOWOOQw65zvicvG0IF8nFqSChrM +QUDAtGa92v/Kuykl0aAZ9wOVaAMArxiudqNlpDXw6vsHDGmKklcDkPhqA0LWjoqQ +aIGq++PczuGI5rpisGJ099bnXvZFbhiOA3eT0m7V40g1r+AI6lF+Cd+BwxSV0wXM +9YtdsYOrGbj9k9iSIhy/8uInbpVWY4gkF40BFLNcMIuoq96eOWrKlgIoAqbEqvrT +D4+ecTdD7c57K3Xbvz7NTq/sFhPd//2aqsvf0x0TjGDnCfGCNtjJ6EjYWyQ0p/79 +DP/1v07xF7fhl226deMpFOF55KD++bCvq5Y83HcEd0MNHLcmVjp6OTEu8WN3BL94 +Foqo2CbMCz6/0mtDUicdHm7OWihYtD0Wl6LSfO9nFp+xRO45STP97uwpAu4uKESD +fvT/+4GUJxa4p1IftcDcLotVRI+2CHrW4QDwO/4CjqmT71KDgvFK4ojd3ZOFgLgS +edlvSccCAwEAAaNTMFEwHQYDVR0OBBYEFFCdcRM4rv32FPeptnMGuO42anduMB8G +A1UdIwQYMBaAFFCdcRM4rv32FPeptnMGuO42anduMA8GA1UdEwEB/wQFMAMBAf8w +DQYJKoZIhvcNAQELBQADggIBAD6yiFPClxwGkvxvncomM7ujNK2mZDP923HBegoA +I+O86QdDrV19dl+GK1n3+iaFWlLTWXI9PinMyG756QbUbO3p4sVHPWtA9XOnB2pr +LchaFqY91MFeF3n/qbhT9z1KwOqwHsxnS1EZp9GNKA7lZHEpiTf6NvjlShSl1kGC +YrXgD67eJvIqQuzD4s/fCkLLIv7TG7Goo2YJ9+y5f5WXEULXi5OMxnwI+mvloAFY +FTV93OEjJ1YGlJ0u0bnQBF9Wdtz55442CrotSvxU58gVhA9jCjKypDVr0Nx9Rz9v +pdr4FlclYgOe0MPlFUnABTFHTtiCSpLvtoyt7GoLSlL7xdSPYhd6XctE/v7xgHHR +X4ETnQij55Qce3IZHVTcQbzm6RNtJ5QMJkyEdrUPW4kK4S+lDMFJtlgJbXrjNSqZ +V+pymQvlFRsHo3EpLxWgRWgDXvDge/3kNmIh06rtHDAVBnRODiMayl7gpq1jQcAA +HSdJmmvtmEkJN8sZGNJY8vBnWxC/XDBWHXt0Xs3XUomnKJIH6tJZ92nDvnMKopI2 +qIFZ176nQj1w/Oo6QmhRiEtE+NK8s+YU5eHnKpkLWxtzKqH98ShSQs7rCE0Bz3Q1 +p3T1xNYj60ZmJ8q6f8GXBHdN/sO19nvjwejBwC61OjIsYtweA7rG3QERylNwRfZA +mJQc +-----END CERTIFICATE----- diff --git a/script/config/ca-root-key.pem b/script/config/ca-root-key.pem new file mode 100644 index 00000000..316a22e4 --- /dev/null +++ b/script/config/ca-root-key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDckKjA5cGj2r4O +WBgSc+yeacJVA8a4uw/upTvWWajT7vL9P1OD4TWk6CIlWXNhfw/5saYCc2El0EM4 +12BjRvbMCMV8Y6eALZ4NGYkJHno5xy5thVt063/YeQw1u+U5F0663AF7YFl4uFBt +DT7OdbtY3wDqyHuE+19lUkN8BnRbpZAULB8NbrieNk/n/2x+9CbylGYus3KQ85Y4 +5DDrnO+Jy8bQgXycWpIKGsxBQMC0Zr3a/8q7KSXRoBn3A5VoAwCvGK52o2WkNfDq ++wcMaYqSVwOQ+GoDQtaOipBogar749zO4YjmumKwYnT31ude9kVuGI4Dd5PSbtXj +SDWv4AjqUX4J34HDFJXTBcz1i12xg6sZuP2T2JIiHL/y4idulVZjiCQXjQEUs1ww +i6ir3p45asqWAigCpsSq+tMPj55xN0Ptznsrddu/Ps1Or+wWE93//Zqqy9/THROM +YOcJ8YI22MnoSNhbJDSn/v0M//W/TvEXt+GXbbp14ykU4XnkoP75sK+rljzcdwR3 +Qw0ctyZWOno5MS7xY3cEv3gWiqjYJswLPr/Sa0NSJx0ebs5aKFi0PRaXotJ872cW +n7FE7jlJM/3u7CkC7i4oRIN+9P/7gZQnFrinUh+1wNwui1VEj7YIetbhAPA7/gKO +qZPvUoOC8UriiN3dk4WAuBJ52W9JxwIDAQABAoICAA8/xRsJB3k/n1I4jvl/oEVX +hIMCLtQIwtI6BEgOjNiSNaSCo1CD2oSbM/knFQ4RjdYyjQqwVXAa1PryEQzsFBYJ +KTYvYZ5ACSRYtpu1yreB6NjTln4f5gfeXpS8d92ZmezbVxLjl2v8Ojs/5i65/wjm +6hZRZBDOnErSBC5n99V6PdrXLhJsrEDr+lxM0YB5etPcpK/n4izZWnJ2fFdd00XQ +m99AhI/+xwgB0EtWobwXOlsQwkBsWNwEWKc2TTcSthJk/3LeWyGeRRjJ4DPgfB8f ++vQj6JhvH4xdUhyXkm211EdNWnyxOC9/rxyPdBGhry0sb20X8FVlKk57aJLPR25i +8/XdzsLRh9F3rUkF3cMUAblbUIBuBK8+uGcpFT9rj+j0qarVku1xezvii7sbJvfB +rec5VAs06zpjETXXOQXY+b8KIy/lladEYMz9elllJJukdHz4ev15KEE6FkVZCnai +Np/CJ66CGVminu2EUW9DrpLO1IfjgHjSa+BbPUBZMiPX1/nsGLdXvmLB5C+t9u9L +yEDZ8HQkAaixU7nhBtjluCEE3qwEo3wITHH38CTRluOnbY4cx55qvr61AhlKlfLj +yFbPHWm1NiZC1DiSu4rQIhwLRoZ8QFjUXuSuUVMXPLCY5xP0xaAQMoqPt7qalmpL +9M3GCf0PNx7abXJeFhYBAoIBAQDzg86LmKVQvoYH1qT34E5IPhbkYVEGlRPNvhJc +nbyNiyW88KPghiLto5yU5lF5+Gjug6YGxuzfCsILH4Zf1d/cwsTvAWf/vLqxE0hm +jlMa1ncAlGUqOIcJRXwlRdIPeTr6bMobYfej1bnseZRibYPiTdiK2iu5WnqzXoOK +VoLLwZUYhzB0nBlqasF53h0skWZc0OPwAzX6wNvZfsEvlXbmTuuqcxYoTXd2etmw +DgWg0urI7BO95NcVP8OzAvfyia1n32HNqDYUjgglkQCs/Q9VNgg7hInx8tktcF7D +hy9AajVyBliQs44n+qpOWBxkbZf4BlewwSZ9lE6AvVUK7+MBAoIBAQDn36Fvj1wC +29IiP7mLQMRBUB6DXzMHCQTZrWqxPtyHaHNhLCKjvHuqOVHtoMFmDlgMUGDa7uzu +2x1LesJIibx2fqw9wsJYJS9D/aJENKMUQxkPGNYaVsmq9PhQczP3hACFFGeG0e/q +X0izDcZJ2+AOzQdjTlf7/wJstwgKJr0q7BZ0zK6nqmYGWCRlhXeJwY6W5JC840uD +gPQ9zqoehgWiSceqdw9a40jR8RILE3R3fkI7bbgjNMUTb2RXIyAG+OsU4MOAr9/a +7tv7xvc/7OnRk7qML1Nm6H4RX+Ml4GwsrJ4eBTLZk3jK+q9J3zPsDPDimw8Be733 +l3iG+uzl+dTHAoIBAHpyWpeDKA1T4B1s5wLlPTfCUMyByrZem9imrKD5h+g8gbth +3WV2zXVlAcU1kt6b9FeKxx89Q7pbtHrXXUAvUSO2pEPyFxNXGSEVKyC2jD9AfBJE +jCIQzoXw9J3hX+D7+pYVwgLG3jj5rUWlVOGxY9GVhz86uSW4w14SNl6HZbmDG697 +uvfHzlfeWm8nKtBQRIKDXzo+wxPVijbxL773jxP7tzC8MfYODfX7jB6kzlux8vEZ +FEw2F5jNuV/4LCk+5X0HrCw3Wwo26Q3KveY+UUlto/IZkSuiC2Z3zI7RGQdw4HL3 +ke0g/GhJ+Pzv/UbafMHwnoN4fBxFSLwfd/XJ7gECggEBAL/Jkh7TRfi7V3iqms6a +eHnJSiPBV1SHWjKuJKSMaR27AEwslE44NVVb6e8R20WNsbglt6b+no0GQZnP5OjY +vECKj+7Zq6P6xbiIDnI503WL9istkNHiXRYBX24zi5U74q4wbVzvQnDp9Rjc2Nj4 +e7/iSC0UfOPbTmcbkRPMbQUWl5BuSDw4TgELYLd0CKaxqT4XubvyiOAxhlOlH8L1 +ea19NyBBMDst6Wp06katBdVcYm4DV54PJL7pkOYeEHyT4OW47xv+3LqafOwo6xM9 +4zxM46zV7DcfMD8wv5Hxqafd8z7DcAbU2sGRTfFdI/Yn0gqs1GGWt08WQ0PqQo4J +8bsCggEBAIutXenpyzcUOIx9DBnZ+PrK2GYRk6Yrcw8h4QQFqvDyd0kAhW8arkBD +znd83+ok44yL1YbKV7VVCVyXwEwHm/OU486/SVCSVsQBh18wETXoMzBuvaF3bwAe +CsWJgoOv7+Jl72B8V40UE6TZd9LmHamnJ3kN6zMJGt8c+ALLeQmwGMiOI87rgLkK +hnAOUlWtcoiIw9bboN5vFLsOZSvK0rPa13DYOGAyIUW6xxZr+u91fC1C35EOmx4H +CwMgscGDZuurwVxywH2Pwu1PAtHR841RGHQIo2ks7SJmKpy7eClVpwyU7EPpsCCf +9CgB8JQDMpJ28hIdJRNtBjq2CZiNAp4= +-----END PRIVATE KEY----- diff --git a/script/config/redis-client.crt b/script/config/redis-client.crt new file mode 100644 index 00000000..313cb1e6 --- /dev/null +++ b/script/config/redis-client.crt @@ -0,0 +1,31 @@ +-----BEGIN CERTIFICATE----- +MIIFSTCCAzECFC9XxFWG3zqUUSBscj1YdC/tfec/MA0GCSqGSIb3DQEBCwUAMF4x +CzAJBgNVBAYTAlhYMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxDzANBgNVBAoMBlJl +ZEhhdDEPMA0GA1UECwwGM3NjYWxlMRYwFAYDVQQDDA1BcGlzb25hdG9yIENBMCAX +DTIzMTIxOTE0MDczM1oYDzIwNTEwNTA2MTQwNzMzWjBiMQswCQYDVQQGEwJYWDEV +MBMGA1UEBwwMRGVmYXVsdCBDaXR5MQ8wDQYDVQQKDAZSZWRIYXQxDzANBgNVBAsM +BjNzY2FsZTEaMBgGA1UEAwwRQXBpc29uYXRvciBDbGllbnQwggIiMA0GCSqGSIb3 +DQEBAQUAA4ICDwAwggIKAoICAQC1N48cE2K1/H03cl29uf+1KRl1JlQfx10LfY0f +BZ04lNSno17sBW1dIwkY8Gp9QYGPkIwh6M1HOLbV4ytgSwNSFWSi+bB5pl/nf70w +j/wCy1ChzYcJomlPANxV2Hw/MOcOA4BsEGSeqGMZlfT3tc30tAj/in2atMuyy+bh +Gk5rBZZz7ndPVttsBGICaqxt7vSgfi3b0D+KCJasY1iCgDe9+o8lG113zeSVoYLV +Ib2Ru7bg9pULFc1FmISgMy+++Kxw/IZ+tngFx/o/dkHczqQcdV7ixW0tlm3deDE/ +xbjaTHZdjEdq7hGdmliY3155A4Ely/OG+pojR2Fwya9KAggTB3N1jDtQkl275jEU +xk+0msjTakO8RSn4YNT8hY5lZzbQmI3Aqum6EYrIa++t+xaZ3b5F04kqyFwjp7/j +e9IgmeLTjANrMERWB1uu5LjdrUNMrDvf64Lu70m6gTUWT+nBHaqXRvnG/am4VBUx +3043qflncdZdS/x4xt3wHtmNL2Lp1Z3+YSBwo19IvzyBnSHhmiBHcRK2K9lblATi +mfsVInq3++un1GcEfwClRLkdQ2Gly6tIHAiOLTL7eD6J4EvT5Zw6v0Gx/n1QWPz1 +3AigwGCRjZcI8NrtN2GLNs4aUmeUzCBUmirAp4c4vysK9OSkKjBOQw/5beOpypn5 +7QFvJQIDAQABMA0GCSqGSIb3DQEBCwUAA4ICAQCHZci0wC+HWmnwpkHxKo71BOt6 +1OsGKCKZjYn3IwMBxiRKIIic5p9V8VweLkNlGYpnf7y5K4GB+YSOvCDBrvX2U2jk +AR+OogiXTRVeiPa41LmXKPRcRkcgcPP2yelW2Fod6j4UCaWmCOeP+sxaU4gLMlu5 +57h5ifK8OQJXHEfKAVrFFiPmrnL7DMeo64zuF/ylzQeZPgyfmvnePzICuxjlswXS +HcTtSM7o76E9v7/3HDhmlSdR5OdjascZV8hs8b1UBK15hdv5FNLA3BSkplOnVAUE ++qizj57TEO0rTqsSXquAQ2fgHCrWjxane+iOn9PP5WPUJDLkdk/77RrvXkq0W1sR +KL7GO1N8CUvDspjFpIXsIYKok8UN191soVheqAO9u9Yj+fgDP5FeQm/S6oZLd4+0 +C8rz5/E/L0aGHCnnQW6KTQHj1hmAy8AkGZa1SzQUtuR9nHHs/KA6cNf31Gz9ULUn +hyxvcMwhGM0bqe6yuxhDTa+3s/RZN7g5dOFbXneYF+M+qIauRUEaARuoFnhx3IpY +/52Wm1RUhu8kQ0gv0QC3XUnRY9aLcEIgwUYiOyE/XxLzMyDlXRz/pQ9D+YMx5JrH +iiu85IIC5epcUG4x3JLyZCH1PFXiXDOS5uquO08h8SjlWLOm6dIriXuDXi8ClbG9 +nesBi86Zd5W3BJETng== +-----END CERTIFICATE----- diff --git a/script/config/redis-client.key b/script/config/redis-client.key new file mode 100644 index 00000000..8f13e8b6 --- /dev/null +++ b/script/config/redis-client.key @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQC1N48cE2K1/H03 +cl29uf+1KRl1JlQfx10LfY0fBZ04lNSno17sBW1dIwkY8Gp9QYGPkIwh6M1HOLbV +4ytgSwNSFWSi+bB5pl/nf70wj/wCy1ChzYcJomlPANxV2Hw/MOcOA4BsEGSeqGMZ +lfT3tc30tAj/in2atMuyy+bhGk5rBZZz7ndPVttsBGICaqxt7vSgfi3b0D+KCJas +Y1iCgDe9+o8lG113zeSVoYLVIb2Ru7bg9pULFc1FmISgMy+++Kxw/IZ+tngFx/o/ +dkHczqQcdV7ixW0tlm3deDE/xbjaTHZdjEdq7hGdmliY3155A4Ely/OG+pojR2Fw +ya9KAggTB3N1jDtQkl275jEUxk+0msjTakO8RSn4YNT8hY5lZzbQmI3Aqum6EYrI +a++t+xaZ3b5F04kqyFwjp7/je9IgmeLTjANrMERWB1uu5LjdrUNMrDvf64Lu70m6 +gTUWT+nBHaqXRvnG/am4VBUx3043qflncdZdS/x4xt3wHtmNL2Lp1Z3+YSBwo19I +vzyBnSHhmiBHcRK2K9lblATimfsVInq3++un1GcEfwClRLkdQ2Gly6tIHAiOLTL7 +eD6J4EvT5Zw6v0Gx/n1QWPz13AigwGCRjZcI8NrtN2GLNs4aUmeUzCBUmirAp4c4 +vysK9OSkKjBOQw/5beOpypn57QFvJQIDAQABAoICAAD/t6b6t2BT7jfERl6BnDZ6 +LcT+y1dqgKmLzV+WpxV1ApAS/zazV/U7uHTd/GRI3Gc3b4JPV3RVTHK04BwVhcTp +z8ZzNGUTRPOQeXE/Ve8JAl6F9MDZl5WDJnPFr/ekNB3YaJiP5ZIXUQaG1FUaODKp +XGRWNdeIuBe59oGWpMiDvR7R+W/jRW3cv1KOlnk7qDcLQwLzbKo83mHAYXBEtynx +q+DTPAppROGLRJ6ao0IepAUs7sjjG6DO1KpSZei6U68CwVYI+PPlNIytgPLv46kx +f26mEnI+IkhmfXUByMFRSxS8bXdaLRgujd732jw2B/v2mhbMIlXf8ppNzkzM4pD/ +6Sx4jMIyfG5FEOoQ4zTwoKR4mqeKJfbfZ60sjdZBCUDjWz3c+GJ2Y3n0WAks5FrJ +FwqzdH2+dDhRPl46gNPCp8+YNqby8ph86eSsU30RwR8nKwiC9Zfa4Dvnr/YGXnel +47GbVIyl+MASFkoHPGfxfdxGc2ujcYOltajbtSMH55Vdle/z1ZqvOHjEjNa0DUoG +TzbMzYkbgIiLq3qShhq/7RgmuD/iVDJVpfD9IC9vk9E0B/U+U4sIpMn/U97rX6ki +CmarIDNZE8NKQzP7U1wH82KzUG6lOa3fikneBuz4GZzuRyy57Wiq8Ofudyk4qQqj +F4Hc7Ux0Ne0Td/uBwqZxAoIBAQDdJYtfAqMHvA/JeuLipFXt6HIE2b66Ms4Df7kF +LReZZQIRQa7vFnXeFRTf1/Q7fp7NJbbqguSJFep5xw/8vUw6ISZZc0ePUYf/Zdko +KbxSgTQFxSKeerbIbPMJR9MLVQcCUBD/N7CGOwj10MFbQ7PdIxr4a4B2N2kphgXH +3X14zITjEqp5NNG9Fea17v5bjgU9O4ewqSG0Dmv3vYl6fBxWaVgljarFXbHVfGFq +g8UuD2tNbXg8DtktB5H2aIw9YbKBQgMAyarC3WKLA8qrNw6l5LhQUanam3OFWJhe +L4ZnA4qpLfPFRLN8GB3Mcj4e8pQSjoQLf2/A2NmiCphpFuNxAoIBAQDRxwAp3q5u +XztwYRAxEsckOccXBOfkQKxRQE1vuBKLYNSmvXmT9xnXu51aI8JNqCdlDDdk5vPn +yBGFW+E6qDgg/Y61T7bHtyWK97jheLRnjkDWhSO0xnJIddNk415RKj18X6o6LuF6 +3syJxYJkMsfnMjv0aCRT+n/0SjnNgok3qHNa2wYdvNYfC4+6kS+2gp8sATHVFG2k +g9wn3ZZrX5juMZAoSV0z1DxFwXdb8fqITC7obL4iYq+7BkkA9Hb/KNdfXiTyHbGk +x+JyvUq0kB9CZ4uAJPqzg/euBdNDxbvDSzcJLvb+QkkoMV2nAgNZ7C7S977s3VVo +CKlB2laJTAT1AoIBAHUQXsqYlryNbL328wuOen5JwbGXokLmie+IOOTojcmBkqqX +3oZpmVNKTZaAtqSXznBCOqXBTEDU/R4hpfDgM+3Bho9rWsIfIqKrw1xjW3FBO8yi +IkVKKF/IbI7JGoqxI9ixkY3IaQVlUsTtEK3DFP9Gigxeu83YXy6/52d5c+XyPZYG +V3952l0BrvM35CWK9HMJhDacEV9DExw9lPzlngqQET0v1OpNQsoD4X2E62gKeX6v +KCqCoawYiceqaDJF+Tw7OWO9XUtx7awxAaPIrVRb0sWRtRwnletMjrus5LS7Rzdv +P895IpnaGkwTXZb8Si3Yi0Wa3Al1wR7qBOv8VcECggEBAKnHZnF8oS4A80lLtWAS +M4w1GtWdidRpJ02HPmTp1FBrqQ/eD7Taa1b3qC1Y/Zv+OQwGLBsE9Gglu1B0gUAI +/4xryoEhp+vRZZgoT8BkAZZCQmWcUb6wUbQlcFaZ0i7cgfDUOCBo6uKXyXDDonlX +xARw2qc/Vg+FkON0431ZZZbfRaVF2JheyelMYSH8+WG8K/jgaqVqqCZa9eZp56C1 +OT+ma2a/oagd4D4Qi3qlyD9hcSF5SRTpTBKQzqrwMBLoyg6MUliGAc08XSDB5OFo +N3R1+yhWRXTenKmCW27hbMptaHiNQyebHDxkX8iXz4CKr4bdLe/kXogZYmtCu4DZ +PrUCggEAfFbIFa3pQy3tqJbVFVSoNfepYLKWkHLXCJmHcIqjCqkWHkqgR3wTgr6V +32844PzbQUm1V7JlZMpUe2zFDCWlwNL2BtLlAl67wancMsdBBvZ2aEGTctWKJaDp +pzWZEbqg8qoVHsj3YWcFm7IocnI03zawuLqi/jKZgaMOIY+2ol/4npAdMCGGbwOk +K4a/QJzD5sM/RTU1/4fvfwfst+HQZxJrrVgDWvYoVFqKiUXx9/KyK7qhQ6URhjRW +oRZt8aqvxj5XE2IFXEP4jeCMIguBYdaYAAQdezkYlIYAYA/H5zkF2CvLnaZaL7t3 +ehRUDfVprDaPWXW2yZMD3hgaNiY+IA== +-----END PRIVATE KEY----- diff --git a/script/config/redis-dsa.crt b/script/config/redis-dsa.crt new file mode 100644 index 00000000..42bc5f19 --- /dev/null +++ b/script/config/redis-dsa.crt @@ -0,0 +1,53 @@ +-----BEGIN CERTIFICATE----- +MIIJYTCCB0kCFFD7Tn6DRSrdioXN32i/0W8obhXbMA0GCSqGSIb3DQEBCwUAMF4x +CzAJBgNVBAYTAlhYMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxDzANBgNVBAoMBlJl +ZEhhdDEPMA0GA1UECwwGM3NjYWxlMRYwFAYDVQQDDA1BcGlzb25hdG9yIENBMCAX +DTIzMTIyMDEyNTU0NVoYDzIwNTEwNTA3MTI1NTQ1WjBaMQswCQYDVQQGEwJYWDEV +MBMGA1UEBwwMRGVmYXVsdCBDaXR5MQ8wDQYDVQQKDAZSZWRIYXQxDzANBgNVBAsM +BjNzY2FsZTESMBAGA1UEAwwJbG9jYWxob3N0MIIGQjCCBDUGByqGSM44BAEwggQo +AoICAQCEmNNJH6M5zgr+0Ve18m+36JUy45IWGbwj/ABku/vbvGb5YMvgIFORHXoE +bg9/71PGWjosNRZKQS+2YlbmF2YAz6F2AEy+CHfV6ypmpVJAqoqeLWgnLfAPXBV8 +e2aM156dIWhfp/mcf1QkOtiYtkbfK4jFGz84Kq/BhwVdPmRex5FGHGOrItLJx8DX +YZivgMj0+cRVVVEV2Pu20TrHMiDrjZVK0w1Cqu9EdptjbFrNDNIKUCRZhhbWdG0Q +JuIlbrgRKs1SRnvEZwH9/IDqDEexkSx5SmWHf9fEPqd4vpwjKfGkTLYUYbkw+sJg +dKFZSx+0SNqCcbzOrO/2YxpwidUbvxUu2zYbWFHnsEwkN6+DZ4j9HWT+9LeXulQn +BYtZn1Rozn6prCRKkpP8RuZX9Khd/eWVBopHZ6a+vnkY3TIw9eA3m8XcgfreUygn +RkGVFdAbhsWsVYMusq8Dd7DMXiOsYMpSD+2ZIqT5LFIDWQmWmy6LiA+rpk8su02T +0k46G2rpxHz1LYnrZ0ruqvSXF8daESV7jMdSjLvcTdosz2xstT4WgKeklVA7zQXW +aT0vrbxIz65xBwhsywC3NmGsd0YdJJ89swpi0pj6qwZR3LpHzHCFtFaHRrvs8S+Q +u56HoXge+/5gMa6kpAogG88j1QvVV5uUdsiA5nq0olRKsMImKwIdANO+TEKeQXz6 +9pdp9HBYCPcKTV4CI7QwWKQzz4MCggIAEJUHeHJjhGD32KBDoLz/DNEkgg/gxTOs +AADNaO6fEinhKNCy/gqt9ZspbMRr7LP35fIAPkudW/fNzuZ4iBJ0rwPEC6u1N4Cd +kd5e7hpSwU4pB9ZfdzzNRc+y0qIv4Nb1hmzk+zUWVx6kFOWA3Ut8ja/q2F4FAg1b +FnSb2/wF27B0Kyetbc04UQUWPxGeD67QqwB6FU2hEQXSPdjERsDjRjCKCbP75LiG +0kKyMB2Bu1qj15LG9pjI07wYQHUAGcYV5nfuL2T1XYNEWmYbhUWcChHwjfqFRZ1v +UmrC8mR9/m8Sqk+kHs4kvB/nOG4yrFAzgjQ0ntOaJF1p8XuWAgUTzGAlpk/XpPde +/Y2vy71BD3+pyaP3bn1stAtFedh6afomxzgTxpOrD6vFFs10l2R/Y1Cog4rcOvhP +1aTrY8tyyCTztWe+G+JsTqUKzZR9SwL8yJjRL8TpB9F+T1Gc/e9pGMG0mDNL97oK +SFY5yxBKe7qoscDnu2yH8ekYyf8QAIf31TFw5bRWKlm33DVxRR0R8l4jWlqVEHzU +sRzhvZolcu+8HzGNww4+AxuPx8dn5KGQai0lPwEX7uv6znKLeAjSlN/Q8SFViiID ++gcp5mox1JfifWK7xtjThLuGe+siYmh07uVC+J92Sj7xFxkIBkqtjbEwjJS4ZqPK +HT/9e79LCVoDggIFAAKCAgBdMYses1sc8lUgzIjqAvzaAzutjnzejzw2y4Fjoqr4 +pnyDPtFHOxsc4g+ifYSYxtpzJOpJZdwBf8eaaBhZURLA/VjrO2zs/Q03irSX3XYn +ge7uJOgm1CEjTaWmxdGCwi5QjPbTiQPc+c9Up5G+TJEKyEwCyBA+JKuMB8yJeH0b +Ww14+mIu83it+eK1JGSP0Ps4YAEtsgcaDzJQMJy9sMezQxoNn9XA9A5xQX0UpudK +pqtPYzPiENeI9XqeScxFeqpW4W+0khsFS0nXtM5oIyuymRnhtHZIGgnmyKA3i4n5 +X8FQobJmiFY037r1IOQ7Y8m0R3DQaIaD6mmW0z6Eev6Qgimd4jrGHRvTAsWgutUy +P9SWvGJG0gAvawCbaHJw2kd5fnBN7rBNOVZ+dqFU/jVqc+/zhbGUoiqDBqZoqZ8U +NW3rDHLDYWPMVP56dESTOFB31ZhYuJJ7kbYWXJ4NCKyYzLOzuDDl1bCayQct5s9E +0V9fJn5nd0bJIHtDZLw+4Zi2blPR4VD8dMt+UCiOHDQvKuW/XbholTzCGZDvDAqb +iHbN1IEFrOcteFBy+ecCeN8uAZHPc32aEjBkz3oedxOA6sMlVxXsz7xmQkvdpIkc ++6C1eohn3FqztMdUG4tQ6loJDD0qv7P9Lz1l4ybiPA4ktBOnrNbelbKwhNOqThGC +AjANBgkqhkiG9w0BAQsFAAOCAgEAkuKFPngVOQV/oMbq59WnatOayIykdA+wwLFU +/0c21kQxOYtpl5oMAF8PmODNXWqeG8EyKWPKbwWkUuLDo3qDJ0gZ3SthCHvVekge +ggJ5teNZkVWAJsHLJlS6p/zRhLlBN5JIjlaJE3TusAxtnCWFHW2j0vIIcUhj1FCV +NXE3S9o/lx1HKC7dR1f/7L3xic2aWl/biM5WO6DSVE4piGkotS5b/QPxN76NC3fA +o2TO3eX87nrnUruBjdB/c1l6rX5nuCzn3Bj9NRyDpJSB606ou1F+VG0+QkbUQwSt +m6/DZWd4fZ3Qs/Ago7JtnrIFOf7g8d0R7AYHKgTrvhA7oYx/Vdq6JSAQkXHmtowB +b8HojKSILrlb8RuvDtDyt1/ccdVLcagmQrX36DhrDSxFRfYCgYluTkgqgXNn0Pv9 +bH/Dq2G631zUqx+NJydCGqsyJ0DEYb7ZZF29XXAS6INttXX0MBnqpqfL/Avbku3F +6cqDRK6XMaREqOjVYEvXVZUKRIqRiSdbYdOKc78ROvnbfVxRGWdKziluvrD8fqmP +gQBBO/NgukK0hOnTAEtyoHLK/KWHxKKrhay8zTx+olk3ajO90kFAm3GG7Qrl3eO/ +DeJaxtjB9w0e+jz/gotkaPe/mWcE1LzGTnHC1OuiEkbzCBFe89o+GWeAt8I9WYHN +FAWNe/g= +-----END CERTIFICATE----- diff --git a/script/config/redis-dsa.pem b/script/config/redis-dsa.pem new file mode 100644 index 00000000..ce949ffb --- /dev/null +++ b/script/config/redis-dsa.pem @@ -0,0 +1,26 @@ +-----BEGIN PRIVATE KEY----- +MIIEXAIBADCCBDUGByqGSM44BAEwggQoAoICAQCEmNNJH6M5zgr+0Ve18m+36JUy +45IWGbwj/ABku/vbvGb5YMvgIFORHXoEbg9/71PGWjosNRZKQS+2YlbmF2YAz6F2 +AEy+CHfV6ypmpVJAqoqeLWgnLfAPXBV8e2aM156dIWhfp/mcf1QkOtiYtkbfK4jF +Gz84Kq/BhwVdPmRex5FGHGOrItLJx8DXYZivgMj0+cRVVVEV2Pu20TrHMiDrjZVK +0w1Cqu9EdptjbFrNDNIKUCRZhhbWdG0QJuIlbrgRKs1SRnvEZwH9/IDqDEexkSx5 +SmWHf9fEPqd4vpwjKfGkTLYUYbkw+sJgdKFZSx+0SNqCcbzOrO/2YxpwidUbvxUu +2zYbWFHnsEwkN6+DZ4j9HWT+9LeXulQnBYtZn1Rozn6prCRKkpP8RuZX9Khd/eWV +BopHZ6a+vnkY3TIw9eA3m8XcgfreUygnRkGVFdAbhsWsVYMusq8Dd7DMXiOsYMpS +D+2ZIqT5LFIDWQmWmy6LiA+rpk8su02T0k46G2rpxHz1LYnrZ0ruqvSXF8daESV7 +jMdSjLvcTdosz2xstT4WgKeklVA7zQXWaT0vrbxIz65xBwhsywC3NmGsd0YdJJ89 +swpi0pj6qwZR3LpHzHCFtFaHRrvs8S+Qu56HoXge+/5gMa6kpAogG88j1QvVV5uU +dsiA5nq0olRKsMImKwIdANO+TEKeQXz69pdp9HBYCPcKTV4CI7QwWKQzz4MCggIA +EJUHeHJjhGD32KBDoLz/DNEkgg/gxTOsAADNaO6fEinhKNCy/gqt9ZspbMRr7LP3 +5fIAPkudW/fNzuZ4iBJ0rwPEC6u1N4Cdkd5e7hpSwU4pB9ZfdzzNRc+y0qIv4Nb1 +hmzk+zUWVx6kFOWA3Ut8ja/q2F4FAg1bFnSb2/wF27B0Kyetbc04UQUWPxGeD67Q +qwB6FU2hEQXSPdjERsDjRjCKCbP75LiG0kKyMB2Bu1qj15LG9pjI07wYQHUAGcYV +5nfuL2T1XYNEWmYbhUWcChHwjfqFRZ1vUmrC8mR9/m8Sqk+kHs4kvB/nOG4yrFAz +gjQ0ntOaJF1p8XuWAgUTzGAlpk/XpPde/Y2vy71BD3+pyaP3bn1stAtFedh6afom +xzgTxpOrD6vFFs10l2R/Y1Cog4rcOvhP1aTrY8tyyCTztWe+G+JsTqUKzZR9SwL8 +yJjRL8TpB9F+T1Gc/e9pGMG0mDNL97oKSFY5yxBKe7qoscDnu2yH8ekYyf8QAIf3 +1TFw5bRWKlm33DVxRR0R8l4jWlqVEHzUsRzhvZolcu+8HzGNww4+AxuPx8dn5KGQ +ai0lPwEX7uv6znKLeAjSlN/Q8SFViiID+gcp5mox1JfifWK7xtjThLuGe+siYmh0 +7uVC+J92Sj7xFxkIBkqtjbEwjJS4ZqPKHT/9e79LCVoEHgIceDE1WAkxmvBcKRSw +mSyEtLORubfpcBEwmPhm/A== +-----END PRIVATE KEY----- diff --git a/script/config/redis-ec.crt b/script/config/redis-ec.crt new file mode 100644 index 00000000..2a4e0cad --- /dev/null +++ b/script/config/redis-ec.crt @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDdjCCAV4CFFD7Tn6DRSrdioXN32i/0W8obhXaMA0GCSqGSIb3DQEBCwUAMF4x +CzAJBgNVBAYTAlhYMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxDzANBgNVBAoMBlJl +ZEhhdDEPMA0GA1UECwwGM3NjYWxlMRYwFAYDVQQDDA1BcGlzb25hdG9yIENBMCAX +DTIzMTIyMDEyNTQ1OFoYDzIwNTEwNTA3MTI1NDU4WjBaMQswCQYDVQQGEwJYWDEV +MBMGA1UEBwwMRGVmYXVsdCBDaXR5MQ8wDQYDVQQKDAZSZWRIYXQxDzANBgNVBAsM +BjNzY2FsZTESMBAGA1UEAwwJbG9jYWxob3N0MFkwEwYHKoZIzj0CAQYIKoZIzj0D +AQcDQgAES6kvZIlhbsg0P/ZP/1tqJSbjQN09bsTj1Lz5kZ+gEErS+auXdMx+WRLC +NUsh/VgVNDy3TZlx0k67NW1wmxkI6jANBgkqhkiG9w0BAQsFAAOCAgEADMi2NbQj +laaRAcmhNHliCZsHuRCvQvmIikILYP45ec6RH1IMB4SngEin1ePdd5Psyd+jE0Hm +2mi4QTuscg6zL4lf/dx/7oam6V8Cr0vhHvqD/nW2nFHQCT22VjuoBdcAGvt3MVmK +H6mnYwxHtxkSvPq726CU0T3yujlCoTaue0ahN31J7bVqbM8muG6sZnMTlloprbSm +Mn5d98W7HIaDPSCipHukv7srZ0+OPNzSEcy1jonGbw5YLjYxM0le4w6TwL2xOVfQ +9N8FaCIUSWGoY36Qqm0cYqRLmb4rE3z9iPxTJfy1/nQi3mbKyKYlRlkAcyFaLY+B +OLs6C2BeLCNm6YtDFls5pgExP2ZOBRmieyEQaiV1kmdRoCeXqa/ulYVUzFNcexoi +Lok52TlS28pXi8eZEykdQjWJ68Deb7KmA9Ue3bWf0xbl9GSyD/77nCBlZlLjQbvF +Y7AFWfvMmCuKbJVGm9sKhcgY4R+4iZPozGLZr313BcPcORKSgTVZU3BiwOAT/O6C +jouJ/+of9UEj22G2YtH3hg5IsdY/Omc8HcVx62j8bNTaIpARbV9YSPoLUDaJ+Ohb +c4ovfh+ksdjCu0/xQzq2BvRtdLo5OZslbpGeLqV4GAWmHYXhfCq+k/Jv33re5DcN +R/Q/rT8ClEIi/3nfWR9eIpklzxGTOeMk8RY= +-----END CERTIFICATE----- diff --git a/script/config/redis-ec.key b/script/config/redis-ec.key new file mode 100644 index 00000000..104b9949 --- /dev/null +++ b/script/config/redis-ec.key @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIDEzNzOlKNVpUbwbkG/Ud45bJYJ/SWxTmCMSPrJ4VQ7doAoGCCqGSM49 +AwEHoUQDQgAES6kvZIlhbsg0P/ZP/1tqJSbjQN09bsTj1Lz5kZ+gEErS+auXdMx+ +WRLCNUsh/VgVNDy3TZlx0k67NW1wmxkI6g== +-----END EC PRIVATE KEY----- diff --git a/script/config/redis-server-acl.conf b/script/config/redis-server-acl.conf new file mode 100644 index 00000000..ae34bbd7 --- /dev/null +++ b/script/config/redis-server-acl.conf @@ -0,0 +1 @@ +user apisonator-test on >p4ssW0rd ~* &* +@all diff --git a/script/config/redis-server.crt b/script/config/redis-server.crt new file mode 100644 index 00000000..528fd806 --- /dev/null +++ b/script/config/redis-server.crt @@ -0,0 +1,31 @@ +-----BEGIN CERTIFICATE----- +MIIFQTCCAykCFB8hf+obX1FxnppOwU9T+j2TV0XMMA0GCSqGSIb3DQEBCwUAMF4x +CzAJBgNVBAYTAlhYMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxDzANBgNVBAoMBlJl +ZEhhdDEPMA0GA1UECwwGM3NjYWxlMRYwFAYDVQQDDA1BcGlzb25hdG9yIENBMCAX +DTIzMTIyMDEzNDY1MFoYDzIwNTEwNTA3MTM0NjUwWjBaMQswCQYDVQQGEwJYWDEV +MBMGA1UEBwwMRGVmYXVsdCBDaXR5MQ8wDQYDVQQKDAZSZWRIYXQxDzANBgNVBAsM +BjNzY2FsZTESMBAGA1UEAwwJbG9jYWxob3N0MIICIjANBgkqhkiG9w0BAQEFAAOC +Ag8AMIICCgKCAgEAtLQcPGp1V91LCMqfjcIsBAvuL+/HH8y81ktN+DrMOgMt3gbm +ngf/zghGX+XQUuYGNfeH7BottqHrh4WgP0RtqZ6KWcc5A4zyqwaDfPcRNPPnZEAe +adUYTuH4ew8mk+KW46gluuDuE4JZ+9gZQBl/y1tdWdlKEczdLlyrfLRYaStmwD0F +Kvp/I3IHMjo6hfRgy41kv9FT41oyZ1zmdUoqsQTSCpEj4iTi4BPHApj8jbgmLBhs +a1q559ORJo6famSV2Zi3Ej/xI3ljPKN+iVrxRY7n2NvqWgIe6xOXxFAuu7DMe51r ++zRp1Y7mhQqTUvrMOF64/P2Jl9LoMNq9GuKlVWTWvMXy9idb5utxEYaWyivvdRAQ +1va1mB9g2kLlRD2rrxV9Rq0llJF3muDr3ch8oX4qrq78AExjFeJF18eYkwri/l7g +qdLlZzmqLreDR9NTyXM4YyzeoX9Yfq7BJUUNQlMgpOF2KGJ1fAIwop6tJAQOiOgn +VUOHKQH6j0/tN7ysGhR3epbjUGs/kCq81oDE3O/wpnxIybNvglG6Btg6AD5ID4qC +aHJyHCuOqSy2LuZzF/R/6noFiCM2qSJQOGu0URRwEjplcUl5JRCNBHfPUPNQaQtn +xzG5bxAaiDNu+sYNTk/CnpUMU0s2Enk3AXMVFzcDOXRSZImmVezysrcY1b0CAwEA +ATANBgkqhkiG9w0BAQsFAAOCAgEAYqlltVsQlOyti++hmUeZ7uyN2OWQJg1zRbvm +7Wi+pDqYGgDy04l/ESfQEjuP85FbWWrTPL5lbH2sGsaMSIMKvS2mUV0V2G0qCVuY +0SgJS2bUbxhgd7aPMFKm8tOwP6iLQgEkqoSV1edBSY95+vjVF1WKXiLLSeSs9YI6 +XmmrF71jbGUDyUi1fd7upxZ18QkB02F9o9MrA1UVHUP6Se8I7m1vOVCtfEnbt2tE +Yi4m66axvJCjl2RZOu7mDoOI+o5rLBVWGCLsJ9Z10QJ6i6cb8PpaEX4JXJ7l1p+C +sreTGQMGsiuz7izsOs767gn6GQ+xuN+zAfTBUJqSnG2JyC8PO6MfDHJ6iS9aUDYV +3GCgrxWVaSsrQkpZidKUulWRWGERJ2MfdnmC3r/Sob7u2ypSRBMWymaZeK1sUrK1 +V9oBopDJo2EKoRYHHLOcLcTKmKNzb0m8X0XsvFOqscOIs6BeSNFonOmSc+wLPtOw +4oIFcaBh+JIMHkm7AoEhI+GFGzmdHUn7dCa+MeBzZeux24XLqlhd5FvfbcU5RFHO +BQ7GyAx6FsY+Y/FBJSw5CboakXATQpfbC7nFTGAFUglxInMqXLgtI2PYiAQEvwuO +Z1HAuzsmpdlXCoPN4jY+q/MQY0xs/E2wAFKmfRoUlvGPuJOF3xo8O82eU3bpkMlt +zZ+AyTg= +-----END CERTIFICATE----- diff --git a/script/config/redis-server.key b/script/config/redis-server.key new file mode 100644 index 00000000..1d909170 --- /dev/null +++ b/script/config/redis-server.key @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQC0tBw8anVX3UsI +yp+NwiwEC+4v78cfzLzWS034Osw6Ay3eBuaeB//OCEZf5dBS5gY194fsGi22oeuH +haA/RG2pnopZxzkDjPKrBoN89xE08+dkQB5p1RhO4fh7DyaT4pbjqCW64O4Tgln7 +2BlAGX/LW11Z2UoRzN0uXKt8tFhpK2bAPQUq+n8jcgcyOjqF9GDLjWS/0VPjWjJn +XOZ1SiqxBNIKkSPiJOLgE8cCmPyNuCYsGGxrWrnn05Emjp9qZJXZmLcSP/EjeWM8 +o36JWvFFjufY2+paAh7rE5fEUC67sMx7nWv7NGnVjuaFCpNS+sw4Xrj8/YmX0ugw +2r0a4qVVZNa8xfL2J1vm63ERhpbKK+91EBDW9rWYH2DaQuVEPauvFX1GrSWUkXea +4OvdyHyhfiqurvwATGMV4kXXx5iTCuL+XuCp0uVnOaout4NH01PJczhjLN6hf1h+ +rsElRQ1CUyCk4XYoYnV8AjCinq0kBA6I6CdVQ4cpAfqPT+03vKwaFHd6luNQaz+Q +KrzWgMTc7/CmfEjJs2+CUboG2DoAPkgPioJocnIcK46pLLYu5nMX9H/qegWIIzap +IlA4a7RRFHASOmVxSXklEI0Ed89Q81BpC2fHMblvEBqIM276xg1OT8KelQxTSzYS +eTcBcxUXNwM5dFJkiaZV7PKytxjVvQIDAQABAoICAAZgPIbQZgEkDvVQDb7D7sDa +mU+zsPdcRmV0e46aQU58TwgAfyHTiNRNbAsO+5sqzP+zRjNJRGFA571jNG5HDQXp +HcFhMcMAT1rdfF8FnX1pOO1Laq2zr4t8MBCxGvYo22FMM4vOflgBFDyHBkBB8t8x +nOLPGezBQvFtCT5f/Q0ChqBnbS4N1jwuVjaqSBpZFXk/k4h5Hr6gFhszjSYiTpm3 +k9NJOVaA+q6+Kk10wte1SBOFbeRNa5w61Z173TNOoHYj2OgziiyeqECH4q6yBZIj +73/lWPWBq0ACjV5USr6j1FmgPYkiWYGhhly3YAKTh6Hz8nVqwYoYqC/hGyVxtDR2 +rCANHfXHUMqMVFXnZWbln3+hPlzosJqMbkEDrUN/ip4rAGjUKv1nM4bJFAbTyR1+ +VcXZlRu/B1mUqaY0xNhgw0JJYn3atimCJYCsfh2vhBZrtuzWSDB1i9vJK8JJ6ca5 +g2eaFU2sBprH4vFjoI5KmW3bO5wS2kGE7zj/Yu90lJZnM2sYw0NugqHFwnD5+y9O +tMW0rgL/v/Tkpd/3ZbLEfipwq7XS4YLxIsGksvYETQdE/djTJyaK837BfrqXY7V4 +NQUBvHO+z36dlWlZaRec98Td308nlxZG1yBiABlTZH5LmL3L66HR3N8oYpyu+eGl +R0C42xbNWvQTfvboz1/BAoIBAQDwEJ/hHZ/sno6AK9AljRRK1APWZH3ywIuZtvv+ +600kN0fi2L5D5TngdviATAwKErinYKc2XpkVBsVe71cu0DYVUsT1WV/2rcrGnEaq +TpMCJ4JwyWLkbOXcI0hs7dJwOGjVWWniEt2P6YloSuZfL1Z3TJaIQN3l0xETdIKl +uZ9hW3Ed79dFUlUpA4vHB2fJxWcTvBWgQcjre2GPAgxhC52Eg1maB1DIttDkwAG3 +CwmofdJqQ2bhq3YdS3LcI7cMuuVS0aqQ3rBvo5t2Ix5w+GEK3DNxx81NL21RPPcP +5XaLXQ+V/PGEGlBPi0Hm7t67sv6t4J33xPO7P1QGcOog2rElAoIBAQDAssUIICod +fnqGlXhogpIS4bxV7TSWO88TXm1Z/6LfYwg1abMUjt8IOcLFGd+rYN32RFzU4hJm +LeH7NWpSCO8A8KUgMr95uuLH6+UVzERsXZ2ay5R/VINAET1RJ0y2W4g43A2ICJVh +dijhzJ5WkaaV9IMEpdh4xKiPobCIdx9AhOTEuzHYdO3HCDgRlZmfYJAtJVTIqaVR +nLsQcFlbJJSzMyNFIh4QGVOTDTpzbogH0S8sw6RMUhbACTlaVdsp7y9dFiUFbeEg +VAdsterwj4c3Y+56HscLezsk9aeCg+Ds8pI7wsa4qf3ySpA4z7NNN/VKdr65SQ+w +USZ5A0Q0p+q5AoIBAQDbiniMCOmeHQq40FiUXsa1/9pakKBWBhwkwSEHaeysOBn3 +rLujn9gkHOj5e5JvC9vuxgdlQ3G/tud9AGi8mRE8wur4bXnDQAB3TaeaTC1JBRTX +THSx7ZzwixvK8ltG4W+N3S1l4FbbK3nL9ONlHQd4JoRrqtbDPRkar6xjHrRQ9q0l +8ZbvAHJ9BY8ovu79qRexBorIeiDiqXz4qajkfxjYfL7Oi+4nAHldGPL4kJGOkzKA +l49YxdNIyx0JkWfvrmPWL4RK5Dx6fCbKDSfjrGIi6mnpng10jvDsIFLZKMIWhU6h +nibI5r+XXwUvyjfLc9BQ91+/nVBAZ5PSKCUVUqo5AoIBABHPM+iZWGsbamWP4O3/ +y1JVbJugbbDI3E2U95ROzRXKSFLvbu6iebh/5iFjT8m+DSWakd3W+w5YhT87Y5Ur +YDN9CjXcmte8TwNdKPIHy/cb+9n3oMTX4b2FGl6qvCtKcF+Y7uM4r92vFv4vdB7a +HURUsds3HLekrVOBBRjHsaq73YzVJDk2LVesycnOQsF5IjoZ7ikRjMbrLlrJl2iW +C+w3+LWF/rNMM+4uVc2fVJEsYyjG+CNZnuKmt5dCwhuxEHaQ+mDzE9ByE7GXapxL +F5sl8D93IrxL6l5zGcU1PQ+JJ7zANq+QfnjrLUMi5xZm6pO0P9JOD1YjWePrSl13 +e3kCggEAECRWltLyR3+U6ufOvqj0Mgv0nKy9SoLzRY4XRK0uHesoBYXEmlOepsUi +23NkzjUFY3PsUflR4/N1IMbdyqR9dsvHUmvMJZ9SIp/fvnt68vOjlFQqHClRRHnc +HO6++q1e5Kj4XnzVnCRsy++kt974mVgl1EFivR42e1CPMvr92Ge/JcUFOArkuqUK +jN0yOUpH10wI5bQbziIcXJ8byPHT8cCKubH4rGEjppeVSur/elmxJzLcRaXVIwRK +YjS3wPGQZaQzkink9CQKaYsQaOVCkuUHPiJdpZwE9xZfLLAgKyw3DqvD/W8K0q9s +WJDHXBisqy4WGbW3D2CYLr1XXwO32A== +-----END PRIVATE KEY----- diff --git a/script/lib/functions b/script/lib/functions index 2023afd4..508e7ac8 100644 --- a/script/lib/functions +++ b/script/lib/functions @@ -1,4 +1,6 @@ REDIS_PORTS=(7379 7380 6379) +REDIS_TLS_PORT=46379 +ALL_PORTS=(${REDIS_PORTS[@]} $REDIS_TLS_PORT) REDIS_UNIX_SOCKET_PREFIX="/tmp/redis_unix" TWEMPROXY_PIDFILE="/tmp/twemproxy_backend_tests.pid" @@ -19,6 +21,8 @@ function kill_pid { } function start_redis { + local basedir="${SCRIPT_DIR:-$(pwd -P)}" + if [ -e /usr/local/bin/redis-server ]; then cmd="/usr/local/bin/redis-server" elif [ -e /usr/bin/redis-server ]; then @@ -28,13 +32,19 @@ function start_redis { fi for port in "${REDIS_PORTS[@]}" do - $cmd --port "$port" --unixsocket "${REDIS_UNIX_SOCKET_PREFIX}.$port.sock" --unixsocketperm 775 --logfile "/tmp/redis.$port.log" --daemonize yes + $cmd --port "$port" --unixsocket "${REDIS_UNIX_SOCKET_PREFIX}.$port.sock" --unixsocketperm 775 --logfile "/tmp/redis.$port.log" --daemonize yes --aclfile ${basedir}/config/redis-server-acl.conf done + + $cmd --tls-port "$REDIS_TLS_PORT" --port 0 --unixsocket "${REDIS_UNIX_SOCKET_PREFIX}.$REDIS_TLS_PORT.sock" \ + --unixsocketperm 775 --logfile "/tmp/redis.$REDIS_TLS_PORT.log" --daemonize yes \ + --tls-cert-file ${basedir}/config/redis-server.crt --tls-key-file ${basedir}/config/redis-server.key \ + --tls-ca-cert-file ${basedir}/config/ca-root-cert.pem --tls-auth-clients optional \ + --aclfile ${basedir}/config/redis-server-acl.conf } function stop_redis { local pids - for port in "${REDIS_PORTS[@]}"; do + for port in "${ALL_PORTS[@]}"; do kill_pid `ps aux | grep redis-server | grep "$port" | awk '{ print $2 }'` & pids[${port}]=$! done From a2ef56987bb98d22ef029e57201919f3d56c8d9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Tue, 19 Dec 2023 18:07:32 +0100 Subject: [PATCH 18/54] StorageSync: Add tests for TLS connections --- test/unit/storage_sync_test.rb | 81 +++++++++++++++++++++++++++++++--- 1 file changed, 74 insertions(+), 7 deletions(-) diff --git a/test/unit/storage_sync_test.rb b/test/unit/storage_sync_test.rb index 5c9cf055..e9c6a732 100644 --- a/test/unit/storage_sync_test.rb +++ b/test/unit/storage_sync_test.rb @@ -1,6 +1,6 @@ require File.expand_path(File.dirname(__FILE__) + '/../test_helper') -class StorageTest < Test::Unit::TestCase +class StorageSyncTest < Test::Unit::TestCase def test_basic_operations storage = StorageSync.instance(true) storage.del('foo') @@ -36,12 +36,6 @@ def test_redis_malformed_url end end - def test_redis_url_without_scheme - assert_nothing_raised do - StorageSync.send :new, url('foo') - end - end - def test_sentinels_connection_string config_obj = { url: 'redis://master-group-name', @@ -247,6 +241,79 @@ def test_sentinels_empty end end + def test_tls_no_client_certificate + config_obj = { + url: 'rediss://localhost:46379/0', + ssl_params: { + ca_file: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'ca-root-cert.pem')) + } + } + storage = StorageSync.send :new, Storage::Helpers.config_with(config_obj) + assert_connection(storage) + end + + def test_tls_client_cert_rsa + config_obj = { + url: 'rediss://localhost:46379/0', + ssl_params: { + ca_file: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'ca-root-cert.pem')), + cert: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'redis-client.crt')), + key: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'redis-client.key')) + } + } + storage = StorageSync.send :new, Storage::Helpers.config_with(config_obj) + assert_connection(storage) + end + + def test_tls_client_cert_dsa + config_obj = { + url: 'rediss://localhost:46379/0', + ssl_params: { + ca_file: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'ca-root-cert.pem')), + cert: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'redis-dsa.crt')), + key: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'redis-dsa.pem')) + } + } + storage = StorageSync.send :new, Storage::Helpers.config_with(config_obj) + assert_connection(storage) + end + + def test_tls_client_cert_ec + config_obj = { + url: 'rediss://localhost:46379/0', + ssl_params: { + ca_file: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'ca-root-cert.pem')), + cert: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'redis-ec.crt')), + key: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'redis-ec.key')) + } + } + storage = StorageSync.send :new, Storage::Helpers.config_with(config_obj) + assert_connection(storage) + end + + def test_acl + config_obj = { + url: 'redis://localhost:6379/0', + username: 'apisonator-test', + password: 'p4ssW0rd' + } + storage = StorageSync.send :new, Storage::Helpers.config_with(config_obj) + assert_connection(storage) + end + + def test_acl_tls + config_obj = { + url: 'rediss://localhost:46379/0', + ssl_params: { + ca_file: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'ca-root-cert.pem')) + }, + username: 'apisonator-test', + password: 'p4ssW0rd' + } + storage = StorageSync.send :new, Storage::Helpers.config_with(config_obj) + assert_connection(storage) + end + private def assert_connection(client) From a9c4dd99f01b5e9af55850f1e2834f8587785169 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Tue, 19 Dec 2023 18:08:06 +0100 Subject: [PATCH 19/54] Add connection tests for StorageAsync::Client --- test/unit/storage_async_test.rb | 136 ++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 test/unit/storage_async_test.rb diff --git a/test/unit/storage_async_test.rb b/test/unit/storage_async_test.rb new file mode 100644 index 00000000..9273e214 --- /dev/null +++ b/test/unit/storage_async_test.rb @@ -0,0 +1,136 @@ +require File.expand_path(File.dirname(__FILE__) + '/../test_helper') + +class StorageAsyncTest < Test::Unit::TestCase + def test_basic_operations + storage = StorageAsync::Client.instance(true) + storage.del('foo') + assert_nil storage.get('foo') + storage.set('foo', 'bar') + assert_equal 'bar', storage.get('foo') + end + + def test_redis_host_and_port + storage = StorageAsync::Client.send :new, url('127.0.0.1:6379') + assert_connection(storage) + end + + def test_redis_url + storage = StorageAsync::Client.send :new, url('redis://127.0.0.1:6379/0') + assert_connection(storage) + end + + def test_redis_unix + storage = StorageAsync::Client.send :new, url('unix:///tmp/redis_unix.6379.sock') + assert_connection(storage) + end + + def test_redis_protected_url + assert_nothing_raised do + StorageAsync::Client.send :new, url('redis://user:passwd@127.0.0.1:6379/0') + end + end + + def test_redis_malformed_url + assert_raise Storage::InvalidURI do + StorageAsync::Client.send :new, url('a_malformed_url:1:10') + end + end + + def test_redis_no_scheme + assert_nothing_raised do + StorageAsync::Client.send :new, url('backend-redis:6379') + end + end + + def test_redis_unknown_scheme + assert_raise ArgumentError do + StorageAsync::Client.send :new, url('myscheme://127.0.0.1:6379') + end + end + + def test_tls_no_client_certificate + config_obj = { + url: 'rediss://localhost:46379/0', + ssl_params: { + ca_file: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'ca-root-cert.pem')) + } + } + storage = StorageAsync::Client.send :new, Storage::Helpers.config_with(config_obj) + assert_connection(storage) + end + + def test_tls_client_cert_rsa + config_obj = { + url: 'rediss://localhost:46379/0', + ssl_params: { + ca_file: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'ca-root-cert.pem')), + cert: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'redis-client.crt')), + key: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'redis-client.key')) + } + } + storage = StorageAsync::Client.send :new, Storage::Helpers.config_with(config_obj) + assert_connection(storage) + end + + def test_tls_client_cert_dsa + config_obj = { + url: 'rediss://localhost:46379/0', + ssl_params: { + ca_file: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'ca-root-cert.pem')), + cert: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'redis-dsa.crt')), + key: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'redis-dsa.pem')) + } + } + storage = StorageAsync::Client.send :new, Storage::Helpers.config_with(config_obj) + assert_connection(storage) + end + + def test_tls_client_cert_ec + config_obj = { + url: 'rediss://localhost:46379/0', + ssl_params: { + ca_file: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'ca-root-cert.pem')), + cert: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'redis-ec.crt')), + key: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'redis-ec.key')) + } + } + storage = StorageAsync::Client.send :new, Storage::Helpers.config_with(config_obj) + assert_connection(storage) + end + + def test_acl + config_obj = { + url: 'redis://localhost:6379/0', + username: 'apisonator-test', + password: 'p4ssW0rd' + } + storage = StorageAsync::Client.send :new, Storage::Helpers.config_with(config_obj) + assert_connection(storage) + end + + def test_acl_tls + config_obj = { + url: 'rediss://localhost:46379/0', + ssl_params: { + ca_file: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'ca-root-cert.pem')) + }, + username: 'apisonator-test', + password: 'p4ssW0rd' + } + storage = StorageAsync::Client.send :new, Storage::Helpers.config_with(config_obj) + assert_connection(storage) + end + + private + + def assert_connection(client) + client.flushdb + client.set('foo', 'bar') + assert_equal 'bar', client.get('foo') + end + + + def url(url) + Storage::Helpers.config_with({ url: url }) + end +end From d12c72d86ccffe64cbebfaa9aeece478e28b6556 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Tue, 16 Jan 2024 14:42:05 +0100 Subject: [PATCH 20/54] Update redis gems To solve sentinels failover error --- Gemfile.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index a2a0b800..06247fb6 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -91,8 +91,8 @@ GEM fiber-annotation (0.2.0) fiber-local (1.0.0) gli (2.16.1) - hiredis-client (0.17.0) - redis-client (= 0.17.0) + hiredis-client (0.19.1) + redis-client (= 0.19.1) i18n (1.8.2) concurrent-ruby (~> 1.0) json (2.6.3) @@ -157,9 +157,9 @@ GEM rack-test (0.8.2) rack (>= 1.0, < 3) rake (13.0.1) - redis (5.0.7) - redis-client (>= 0.9.0) - redis-client (0.17.0) + redis (5.0.8) + redis-client (>= 0.17.0) + redis-client (0.19.1) connection_pool redis-namespace (1.11.0) redis (>= 4) From 0972555603eb0f55bd87a4fd5c1f37a76060d4ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Thu, 18 Jan 2024 13:01:58 +0100 Subject: [PATCH 21/54] Storage config: ignore empty ssl params --- lib/3scale/backend/storage_helpers.rb | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/lib/3scale/backend/storage_helpers.rb b/lib/3scale/backend/storage_helpers.rb index d7c93ac8..a6fcfae1 100644 --- a/lib/3scale/backend/storage_helpers.rb +++ b/lib/3scale/backend/storage_helpers.rb @@ -99,7 +99,8 @@ def config_with(config, end.merge(options) cfg_with_sentinels = cfg_sentinels_handler cfg - cfg_defaults_handler cfg_with_sentinels, defaults + cfg_with_ssl = cfg_ssl_handler cfg_with_sentinels + cfg_defaults_handler cfg_with_ssl, defaults end private @@ -240,6 +241,20 @@ def cfg_sentinels_handler(options) options end + def cfg_ssl_handler(options) + return options unless options.key? :ssl_params + + ssl_params = options[:ssl_params] + + ssl_params.delete(:ca_file) if ssl_params[:ca_file].to_s.strip.empty? + ssl_params.delete(:ca_path) if ssl_params[:ca_path].to_s.strip.empty? + ssl_params.delete(:cert) if ssl_params[:cert].to_s.strip.empty? + ssl_params.delete(:key) if ssl_params[:key].to_s.strip.empty? + + options[:ssl_params] = ssl_params + options + end + # The new Redis client accepts either `:url` or `:path`, but not both. # In the case of a path, Redis expects it to not include the `unix://` prefix. # On the other hand, Apisonator accepts only `:url`, for both Sockets and TCP connections. From f53877fbdc3a6de54fe79a407ca8796ef1083300 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Thu, 18 Jan 2024 13:04:06 +0100 Subject: [PATCH 22/54] Add CA_PATH env variables for SSL --- docs/configuration.md | 14 ++++++++++++++ openshift/3scale_backend.conf | 2 ++ 2 files changed, 16 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index 24fdbe70..559fdf26 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -47,6 +47,13 @@ variables. - Applies to: listener, worker, cron. - Format: path to file as string. +### CONFIG_REDIS_CA_PATH + +- System certificates location path, `/etc/ssl/certs` for RHEL +- Optional. Defaults to empty. +- Applies to: listener, worker, cron. +- Format: path to file as string. + ### CONFIG_REDIS_CERT - The path to the client SSL certificate @@ -136,6 +143,13 @@ sentinels. - Applies to: listener, worker, cron. - Format: path to file as string. +### CONFIG_QUEUES_CA_PATH + +- System certificates location path, `/etc/ssl/certs` for RHEL +- Optional. Defaults to empty. +- Applies to: listener, worker, cron. +- Format: path to file as string. + ### CONFIG_QUEUES_CERT - User certificate to connect to Redis through TLS diff --git a/openshift/3scale_backend.conf b/openshift/3scale_backend.conf index 02fb114d..0b3d8a2e 100644 --- a/openshift/3scale_backend.conf +++ b/openshift/3scale_backend.conf @@ -26,6 +26,7 @@ ThreeScale::Backend.configure do |config| config.queues.password = "#{ENV['CONFIG_QUEUES_PASSWORD']}" config.queues.ssl_params = { ca_file: "#{ENV['CONFIG_QUEUES_CA_FILE']}", + ca_path: "#{ENV['CONFIG_QUEUES_CA_PATH']}", cert: "#{ENV['CONFIG_QUEUES_CERT']}", key: "#{ENV['CONFIG_QUEUES_PRIVATE_KEY']}" } @@ -40,6 +41,7 @@ ThreeScale::Backend.configure do |config| config.redis.password = "#{ENV['CONFIG_REDIS_PASSWORD']}" config.redis.ssl_params = { ca_file: "#{ENV['CONFIG_REDIS_CA_FILE']}", + ca_path: "#{ENV['CONFIG_REDIS_CA_PATH']}", cert: "#{ENV['CONFIG_REDIS_CERT']}", key: "#{ENV['CONFIG_REDIS_PRIVATE_KEY']}" } From a81a4e9d2b45b49c6d8be418963de9dc2c03664e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Thu, 18 Jan 2024 14:49:32 +0100 Subject: [PATCH 23/54] Load a default CA cert if present Load config/ca_cert.pem if it exists and no other CA cert has been specified. This way we make it easy to load a CA cert from Openshift --- lib/3scale/backend/storage_helpers.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/3scale/backend/storage_helpers.rb b/lib/3scale/backend/storage_helpers.rb index a6fcfae1..584be28f 100644 --- a/lib/3scale/backend/storage_helpers.rb +++ b/lib/3scale/backend/storage_helpers.rb @@ -241,6 +241,8 @@ def cfg_sentinels_handler(options) options end + # Ensure only ssl_params with value are added to the config + # Also load a default CA cert if none is provided def cfg_ssl_handler(options) return options unless options.key? :ssl_params @@ -252,6 +254,20 @@ def cfg_ssl_handler(options) ssl_params.delete(:key) if ssl_params[:key].to_s.strip.empty? options[:ssl_params] = ssl_params + load_default_ca_cert(options) + end + + # If no CA cert given, look for the default one at `config/ca_cert.pem` + def load_default_ca_cert(options) + return options if options[:ssl_params]&.key?(:ca_file) || options[:ssl_params]&.key?(:ca_path) + + cert_path = "#{Backend::Util.root_dir}/config/ca_cert.pem" + + return options unless File.exist?(cert_path) + + options[:ssl_params] ||= {} + options[:ssl_params][:ca_file] = cert_path + options end From c429e716213fe2ff7c9a66d0d838188f51c13ec4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Mon, 22 Jan 2024 10:01:59 +0100 Subject: [PATCH 24/54] Redis config: remove all empty keys It causes errors in the redis client --- lib/3scale/backend/storage_helpers.rb | 57 +++++++++++---------------- 1 file changed, 24 insertions(+), 33 deletions(-) diff --git a/lib/3scale/backend/storage_helpers.rb b/lib/3scale/backend/storage_helpers.rb index 584be28f..36cd96a4 100644 --- a/lib/3scale/backend/storage_helpers.rb +++ b/lib/3scale/backend/storage_helpers.rb @@ -98,9 +98,9 @@ def config_with(config, h[k] = val if val end.merge(options) - cfg_with_sentinels = cfg_sentinels_handler cfg - cfg_with_ssl = cfg_ssl_handler cfg_with_sentinels - cfg_defaults_handler cfg_with_ssl, defaults + cfg_compacted = cfg_compact cfg + cfg_with_sentinels = cfg_sentinels_handler cfg_compacted + cfg_defaults_handler cfg_with_sentinels, defaults end private @@ -183,6 +183,12 @@ def ensure_url_param(options) options end + def cfg_compact(options) + compact = ->(_k,v) { v.to_s.strip.empty? } + options[:ssl_params].reject!(&compact) if options.key? :ssl_params + options.reject!(&compact) + end + # Expected sentinel input cfg format: # # Either a String with one or more URLs: @@ -241,36 +247,6 @@ def cfg_sentinels_handler(options) options end - # Ensure only ssl_params with value are added to the config - # Also load a default CA cert if none is provided - def cfg_ssl_handler(options) - return options unless options.key? :ssl_params - - ssl_params = options[:ssl_params] - - ssl_params.delete(:ca_file) if ssl_params[:ca_file].to_s.strip.empty? - ssl_params.delete(:ca_path) if ssl_params[:ca_path].to_s.strip.empty? - ssl_params.delete(:cert) if ssl_params[:cert].to_s.strip.empty? - ssl_params.delete(:key) if ssl_params[:key].to_s.strip.empty? - - options[:ssl_params] = ssl_params - load_default_ca_cert(options) - end - - # If no CA cert given, look for the default one at `config/ca_cert.pem` - def load_default_ca_cert(options) - return options if options[:ssl_params]&.key?(:ca_file) || options[:ssl_params]&.key?(:ca_path) - - cert_path = "#{Backend::Util.root_dir}/config/ca_cert.pem" - - return options unless File.exist?(cert_path) - - options[:ssl_params] ||= {} - options[:ssl_params][:ca_file] = cert_path - - options - end - # The new Redis client accepts either `:url` or `:path`, but not both. # In the case of a path, Redis expects it to not include the `unix://` prefix. # On the other hand, Apisonator accepts only `:url`, for both Sockets and TCP connections. @@ -290,6 +266,20 @@ def cfg_unix_path_handler(options) options end + # If no CA cert given, look for the default one at `config/ca_cert.pem` + def cfg_ca_cert_handler(options) + return options if options[:ssl_params]&.key?(:ca_file) || options[:ssl_params]&.key?(:ca_path) + + cert_path = "#{Backend::Util.root_dir}/config/ca_cert.pem" + + return options unless File.exist?(cert_path) + + options[:ssl_params] ||= {} + options[:ssl_params][:ca_file] = cert_path + + options + end + # This ensures some default values are valid for the redis client. # In particular: # @@ -299,6 +289,7 @@ def cfg_unix_path_handler(options) def cfg_defaults_handler(options, defaults) cfg_with_defaults = defaults.merge(ensure_url_param(options)) cfg_with_defaults = cfg_unix_path_handler(cfg_with_defaults) + cfg_with_defaults = cfg_ca_cert_handler(cfg_with_defaults) cfg_with_defaults.delete(:max_connections) unless options[:async] cfg_with_defaults end From 73da74bf580ebc7939b06325142d12e4cf53e0a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Mon, 22 Jan 2024 12:50:56 +0100 Subject: [PATCH 25/54] Monkey patch the redis gem to fix timeout bug When `reconnect_attempts` is > 1, and there's a timeout, commands are sent to redis as many times as attempts configured. We used a fork to fix this in previous redis gem versions. See: https://github.com/3scale/redis-rb/pull/2 This commit translates that hack but as a monkey patch. --- lib/3scale/backend.rb | 1 + lib/redis_client/config/timeout.rb | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 lib/redis_client/config/timeout.rb diff --git a/lib/3scale/backend.rb b/lib/3scale/backend.rb index 2524b0ea..a6627ac1 100644 --- a/lib/3scale/backend.rb +++ b/lib/3scale/backend.rb @@ -5,6 +5,7 @@ require 'hiredis-client' require 'redis' +require 'redis_client/config/timeout' # Monkey patch redis-client require 'resque' require 'securerandom' diff --git a/lib/redis_client/config/timeout.rb b/lib/redis_client/config/timeout.rb new file mode 100644 index 00000000..2196a23e --- /dev/null +++ b/lib/redis_client/config/timeout.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class RedisClient + class Config + module Timeout + def retry_connecting?(attempt, error) + # Timeouts are the only "ConnectionError" that are not safe to + # retry. + # This conditional raise solves this issue + # https://github.com/redis/redis-rb/issues/668. In the example shown, + # there's a timeout while deleting a bit set, and the command is + # executed twice in Redis. + return false if error.is_a?(TimeoutError) + super attempt, error + end + end + end +end + +RedisClient::Config::Common.prepend(RedisClient::Config::Timeout) From decfeb959a2faf8dfeec69a1adae008049ee6146 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Mon, 22 Jan 2024 13:57:47 +0100 Subject: [PATCH 26/54] Rename `compact` lambda to `empty` Co-authored-by: Aleksandar N. Kostadinov --- lib/3scale/backend/storage_helpers.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/3scale/backend/storage_helpers.rb b/lib/3scale/backend/storage_helpers.rb index 36cd96a4..dd64dadf 100644 --- a/lib/3scale/backend/storage_helpers.rb +++ b/lib/3scale/backend/storage_helpers.rb @@ -184,9 +184,9 @@ def ensure_url_param(options) end def cfg_compact(options) - compact = ->(_k,v) { v.to_s.strip.empty? } - options[:ssl_params].reject!(&compact) if options.key? :ssl_params - options.reject!(&compact) + empty = ->(_k,v) { v.to_s.strip.empty? } + options[:ssl_params]&.reject!(&empty) + options.reject!(&empty) end # Expected sentinel input cfg format: From 312179adf5a87707ac368f0af67aaf0f64323033 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Tue, 23 Jan 2024 09:50:05 +0100 Subject: [PATCH 27/54] Remove Airbrake integration It's dead code since https://github.com/3scale/apisonator/commit/281309948e882e0d87fd602f27053e5cb1999055 --- .../backend/logging/external/impl/airbrake.rb | 66 ------------------- 1 file changed, 66 deletions(-) delete mode 100644 lib/3scale/backend/logging/external/impl/airbrake.rb diff --git a/lib/3scale/backend/logging/external/impl/airbrake.rb b/lib/3scale/backend/logging/external/impl/airbrake.rb deleted file mode 100644 index c7553143..00000000 --- a/lib/3scale/backend/logging/external/impl/airbrake.rb +++ /dev/null @@ -1,66 +0,0 @@ -require '3scale/backend/environment' -require '3scale/backend/configuration' - -module ThreeScale - module Backend - module Logging - module External - module Impl - module Airbrake - class << self - def setup(api_key) - do_require - - configure api_key - end - - def setup_rack(rack) - rack.use middleware - end - - def setup_rake - require 'airbrake/tasks' - require 'airbrake/rake_handler' - - ::Airbrake.configure do |config| - config.rescue_rake_exceptions = true - end - end - - def setup_worker - require '3scale/backend/logging/external/resque' - - External::Resque.setup klass - end - - def notify_proc - klass.method(:notify).to_proc - end - - private - - def do_require - require 'airbrake' - end - - def klass - ::Airbrake - end - - def middleware - ::Airbrake::Sinatra - end - - def configure(api_key) - ::Airbrake.configure do |config| - config.api_key = api_key - config.environment_name = Backend.environment - end - end - end - end - end - end - end - end -end From ad3dc2a25e13c7fce1b707bb9198ad291e047adb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Tue, 23 Jan 2024 09:50:49 +0100 Subject: [PATCH 28/54] Update Bugsnag To ensure it includes an updated integration with last Resque version --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 2fe9f47d..3db998e7 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -58,7 +58,7 @@ GEM rspec-files (~> 1.0) rspec-memory (~> 1.0) benchmark-ips (2.7.2) - bugsnag (6.6.4) + bugsnag (6.26.2) concurrent-ruby (~> 1.0) build-environment (1.13.0) builder (3.2.3) @@ -66,7 +66,7 @@ GEM codeclimate-test-reporter (0.6.0) simplecov (>= 0.7.1, < 1.0.0) coderay (1.1.3) - concurrent-ruby (1.1.6) + concurrent-ruby (1.2.3) connection_pool (2.4.1) console (1.23.2) fiber-annotation From 18a0840e9042dc5abe1989d19cb09097589494d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Tue, 23 Jan 2024 09:51:51 +0100 Subject: [PATCH 29/54] Fix Resque-Bugsnag integration Stop using our own integration and use Bugsnag one --- .../backend/logging/external/impl/bugsnag.rb | 6 +- lib/3scale/backend/logging/external/resque.rb | 57 ------------------- 2 files changed, 2 insertions(+), 61 deletions(-) delete mode 100644 lib/3scale/backend/logging/external/resque.rb diff --git a/lib/3scale/backend/logging/external/impl/bugsnag.rb b/lib/3scale/backend/logging/external/impl/bugsnag.rb index 8a263af8..c3d0f42c 100644 --- a/lib/3scale/backend/logging/external/impl/bugsnag.rb +++ b/lib/3scale/backend/logging/external/impl/bugsnag.rb @@ -25,9 +25,7 @@ def setup_rake end def setup_worker - require '3scale/backend/logging/external/resque' - - External::Resque.setup klass + # Bugsnag should integrate automatically with Resque end def notify_proc @@ -56,7 +54,7 @@ def configure(api_key) config.timeout = 3 config.logger = Backend.logger config.meta_data_filters = [] - config.notify_release_stages = ['production', 'preview'] + config.notify_release_stages = ['production', 'staging', 'development'] config.project_root = Backend::Util.root_dir end end diff --git a/lib/3scale/backend/logging/external/resque.rb b/lib/3scale/backend/logging/external/resque.rb deleted file mode 100644 index 64748dfc..00000000 --- a/lib/3scale/backend/logging/external/resque.rb +++ /dev/null @@ -1,57 +0,0 @@ -# This is a module to configure an external error logging service for Resque. -# -# The requirement is that an object be passed in that implements the same -# interface as Airbrake. -# -# This requires a Resque version with https://github.com/resque/resque/pull/1602 -# -module ThreeScale - module Backend - module Logging - module External - module Resque - class << self - def setup(klass) - load_resque_failure_for klass - - ::Resque::Failure::Multiple.classes = [ - ::Resque::Failure::Redis, - Class.new(::Resque::Failure::Airbrake) do - def self.configure(&block) - # calling this hook is an error - raise "error: tried to configure #{self.inspect} from Resque" - end - end, - ] - ::Resque::Failure.backend = ::Resque::Failure::Multiple - end - - private - - # set the argument as ::Airbrake and load Resque::Failure - def load_resque_failure_for(klass) - require 'resque/failure/base' - require 'resque/failure/multiple' - require 'resque/failure/redis' - - # ensure we have a matching ::Airbrake top-level constant or - # define it if missing - begin - airbrake = ::Kernel.const_get(:Airbrake) - rescue NameError - # not defined, so set our own - ::Kernel.const_set(:Airbrake, klass) - else - # defined, expect it's our own object - raise "Airbrake constant pre-defined as #{airbrake.inspect}, " \ - " required to be #{klass.inspect}!" if airbrake != klass - end - - require 'resque/failure/airbrake' - end - end - end - end - end - end -end From fc397b7b7161f4e8540300a93d41e84f48c034d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Fri, 9 Feb 2024 15:02:30 +0100 Subject: [PATCH 30/54] Remove comment: The limitation doesn't exist anymore --- lib/3scale/backend/storage_async/client.rb | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/3scale/backend/storage_async/client.rb b/lib/3scale/backend/storage_async/client.rb index a411cab9..88980c54 100644 --- a/lib/3scale/backend/storage_async/client.rb +++ b/lib/3scale/backend/storage_async/client.rb @@ -48,9 +48,6 @@ def call(*args) def pipelined(&block) # This replaces the client with a Pipeline that accumulates the Redis # commands run in a block and sends all of them in a single request. - # - # There's an important limitation: this assumes that the fiber will - # not yield in the block. pipeline = Pipeline.new block.call pipeline From 70034ab91ee2f928d7f260f525e3024758e43cef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Mon, 19 Feb 2024 16:07:12 +0100 Subject: [PATCH 31/54] Fix config compact method It was returning `nil` on some scenarios. --- lib/3scale/backend/storage_helpers.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/3scale/backend/storage_helpers.rb b/lib/3scale/backend/storage_helpers.rb index dd64dadf..b7acc9ee 100644 --- a/lib/3scale/backend/storage_helpers.rb +++ b/lib/3scale/backend/storage_helpers.rb @@ -185,8 +185,8 @@ def ensure_url_param(options) def cfg_compact(options) empty = ->(_k,v) { v.to_s.strip.empty? } - options[:ssl_params]&.reject!(&empty) - options.reject!(&empty) + options[:ssl_params]&.delete_if(&empty) + options.delete_if(&empty) end # Expected sentinel input cfg format: From e6d1e5d54728425ca6fe7af317511a80bc6dcb09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Thu, 22 Feb 2024 13:51:02 +0100 Subject: [PATCH 32/54] Fix typo Co-authored-by: Daria Mayorova --- lib/3scale/backend/stats/aggregators/base.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/3scale/backend/stats/aggregators/base.rb b/lib/3scale/backend/stats/aggregators/base.rb index d7d2ac00..f4917e4c 100644 --- a/lib/3scale/backend/stats/aggregators/base.rb +++ b/lib/3scale/backend/stats/aggregators/base.rb @@ -11,7 +11,7 @@ module Base # @param [Time] timestamp # @param [Array] keys array of {(service|application|user) => "key"} # @param [Symbol] cmd - # # @param [Redis] client + # @param [Redis] client def aggregate_values(value, timestamp, keys, cmd, client = storage) keys_for_bucket = [] From c8bdef8def476831050f2e4fefd85e9c97b2a595 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Fri, 8 Mar 2024 09:31:39 +0100 Subject: [PATCH 33/54] Redis: Add SSL Param Most of times the gem infers its value correctly, but there's a particular situation where `ssl: true` must be explicitly given, for Sentinels + TLS --- docs/configuration.md | 14 ++++++++++++++ lib/3scale/backend/configuration.rb | 4 ++-- lib/3scale/backend/storage_helpers.rb | 2 +- openshift/3scale_backend.conf | 2 ++ 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 559fdf26..677bb327 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -40,6 +40,13 @@ variables. - Applies to: listener, worker, cron. - Format: string. +### CONFIG_REDIS_SSL + +- Whether use SSL to connect to Redis +- Optional. Defaults to empty. +- Applies to: listener, worker, cron. +- Format: true or false. + ### CONFIG_REDIS_CA_FILE - Certification authority to validate Redis server TLS connections with @@ -136,6 +143,13 @@ sentinels. - Applies to: listener, worker, cron. - Format: string. +### CONFIG_QUEUES_SSL + +- Whether use SSL to connect to Redis +- Optional. Defaults to empty. +- Applies to: listener, worker, cron. +- Format: true or false. + ### CONFIG_QUEUES_CA_FILE - Certification authority certificate Redis should trust to accept TLS connections diff --git a/lib/3scale/backend/configuration.rb b/lib/3scale/backend/configuration.rb index 6952c1c4..2e36d183 100644 --- a/lib/3scale/backend/configuration.rb +++ b/lib/3scale/backend/configuration.rb @@ -39,9 +39,9 @@ def parse_int(value, default) config.workers_logger_formatter = :text # Add configuration sections - config.add_section(:queues, :master_name, :username, :password, :ssl_params, :sentinels, :role, + config.add_section(:queues, :master_name, :username, :password, :ssl, :ssl_params, :sentinels, :role, :connect_timeout, :read_timeout, :write_timeout, :max_connections) - config.add_section(:redis, :url, :proxy, :username, :password, :ssl_params, :sentinels, :role, + config.add_section(:redis, :url, :proxy, :username, :password, :ssl, :ssl_params, :sentinels, :role, :connect_timeout, :read_timeout, :write_timeout, :max_connections, :async) config.add_section(:hoptoad, :service, :api_key) config.add_section(:internal_api, :user, :password) diff --git a/lib/3scale/backend/storage_helpers.rb b/lib/3scale/backend/storage_helpers.rb index b7acc9ee..07038ef3 100644 --- a/lib/3scale/backend/storage_helpers.rb +++ b/lib/3scale/backend/storage_helpers.rb @@ -62,7 +62,7 @@ class << self # CONN_WHITELIST - Connection options that can be specified in config # Note: we don't expose reconnect_attempts until the bug above is fixed CONN_WHITELIST = [ - :connect_timeout, :read_timeout, :write_timeout, :max_connections, :username, :password, :ssl_params + :connect_timeout, :read_timeout, :write_timeout, :max_connections, :username, :password, :ssl, :ssl_params ].freeze private_constant :CONN_WHITELIST diff --git a/openshift/3scale_backend.conf b/openshift/3scale_backend.conf index 0b3d8a2e..2d51020d 100644 --- a/openshift/3scale_backend.conf +++ b/openshift/3scale_backend.conf @@ -24,6 +24,7 @@ ThreeScale::Backend.configure do |config| config.queues.master_name = "#{ENV['CONFIG_QUEUES_MASTER_NAME']}" config.queues.username = "#{ENV['CONFIG_QUEUES_USERNAME']}" config.queues.password = "#{ENV['CONFIG_QUEUES_PASSWORD']}" + config.queues.ssl = parse_boolean_env('CONFIG_QUEUES_SSL') config.queues.ssl_params = { ca_file: "#{ENV['CONFIG_QUEUES_CA_FILE']}", ca_path: "#{ENV['CONFIG_QUEUES_CA_PATH']}", @@ -39,6 +40,7 @@ ThreeScale::Backend.configure do |config| config.redis.proxy = "#{ENV['CONFIG_REDIS_PROXY']}" config.redis.username = "#{ENV['CONFIG_REDIS_USERNAME']}" config.redis.password = "#{ENV['CONFIG_REDIS_PASSWORD']}" + config.redis.ssl = parse_boolean_env('CONFIG_REDIS_SSL') config.redis.ssl_params = { ca_file: "#{ENV['CONFIG_REDIS_CA_FILE']}", ca_path: "#{ENV['CONFIG_REDIS_CA_PATH']}", From 19dda59077ce7463e731ccbcabdb9fd568cfeed3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Fri, 8 Mar 2024 12:44:12 +0100 Subject: [PATCH 34/54] Fix tests after merging master --- lib/3scale/backend/storage_async/methods.rb | 3 + spec/unit/queue_storage_spec.rb | 4 +- test/unit/storage_async_test.rb | 69 +++++++----- test/unit/storage_sync_test.rb | 115 ++++++++------------ 4 files changed, 92 insertions(+), 99 deletions(-) diff --git a/lib/3scale/backend/storage_async/methods.rb b/lib/3scale/backend/storage_async/methods.rb index d07ba3df..f5b2915f 100644 --- a/lib/3scale/backend/storage_async/methods.rb +++ b/lib/3scale/backend/storage_async/methods.rb @@ -35,10 +35,13 @@ module Methods :incr, :incrby, :keys, + :lindex, :llen, :lpop, :lpush, :lrange, + :lrem, + :lset, :ltrim, :mget, :ping, diff --git a/spec/unit/queue_storage_spec.rb b/spec/unit/queue_storage_spec.rb index 46099b4d..6d5eb915 100644 --- a/spec/unit/queue_storage_spec.rb +++ b/spec/unit/queue_storage_spec.rb @@ -50,9 +50,9 @@ def is_sentinel?(connection) connector.instance_of?(Async::Redis::SentinelsClient) else connector = connection.instance_variable_get(:@client) - ..instance_variable_get(:@config) + .instance_variable_get(:@config) - !!config&.sentinel? + !!connector&.sentinel? end end end diff --git a/test/unit/storage_async_test.rb b/test/unit/storage_async_test.rb index 6bf09226..a13bbc2a 100644 --- a/test/unit/storage_async_test.rb +++ b/test/unit/storage_async_test.rb @@ -11,13 +11,15 @@ def test_basic_operations end def test_redis_host_and_port - storage = StorageAsync::Client.send :new, url('127.0.0.1:6379') - assert_connection(storage) + config_obj = url('127.0.0.1:6379') + storage = StorageAsync::Client.send :new, config_obj + assert_client_config(storage, **config_obj) end def test_redis_url - storage = StorageAsync::Client.send :new, url('redis://127.0.0.1:6379/0') - assert_connection(storage) + config_obj = url('redis://127.0.0.1:6379/0') + storage = StorageAsync::Client.send :new, config_obj + assert_client_config(storage, **config_obj) end def test_redis_protected_url @@ -39,7 +41,6 @@ def test_sentinels_connection_string } conn = StorageAsync::Client.send :new, Storage::Helpers.config_with(config_obj) - assert_sentinel_client(conn) assert_sentinel_config(conn, url: config_obj[:url], sentinels: [{ host: '127.0.0.1', port: 26_379 }, { host: '127.0.0.1', port: 36_379 }]) @@ -52,7 +53,6 @@ def test_sentinels_connection_array_strings } conn = StorageAsync::Client.send :new, Storage::Helpers.config_with(config_obj) - assert_sentinel_client(conn) assert_sentinel_config(conn, url: config_obj[:url], sentinels: [{ host: '127.0.0.1', port: 26_379 }, { host: '127.0.0.1', port: 36_379 }]) @@ -68,7 +68,6 @@ def test_sentinels_connection_array_hashes } conn = StorageAsync::Client.send :new, Storage::Helpers.config_with(config_obj) - assert_sentinel_client(conn) assert_sentinel_config(conn, url: config_obj[:url], sentinels: config_obj[:sentinels].compact.reject(&:empty?)) end @@ -90,7 +89,6 @@ def test_sentinels_simple_url } conn = StorageAsync::Client.send :new, Storage::Helpers.config_with(config_obj) - assert_sentinel_client(conn) assert_sentinel_config(conn, url: "redis://#{config_obj[:url]}", sentinels: [{ host: '127.0.0.1', port: 26_379 }]) end @@ -105,7 +103,6 @@ def test_sentinels_array_hashes_default_port } conn = StorageAsync::Client.send :new, Storage::Helpers.config_with(config_obj) - assert_sentinel_client(conn) assert_sentinel_config(conn, url: config_obj[:url], sentinels: [{ host: '127.0.0.1', port: default_sentinel_port }, { host: '192.168.1.1', port: default_sentinel_port }, @@ -123,7 +120,6 @@ def test_sentinels_array_strings_default_port } conn = StorageAsync::Client.send :new, Storage::Helpers.config_with(config_obj) - assert_sentinel_client(conn) assert_sentinel_config(conn, url: config_obj[:url], sentinels: [{ host: '127.0.0.2', port: default_sentinel_port }, { host: '127.0.0.1', port: default_sentinel_port }, @@ -141,7 +137,6 @@ def test_sentinels_correct_role } conn = StorageAsync::Client.send :new, Storage::Helpers.config_with(config_obj) - assert_sentinel_client(conn) assert_sentinel_config(conn, url: config_obj[:url], sentinels: [{ host: '127.0.0.1', port: 26_379 }], role: role) @@ -200,7 +195,7 @@ def test_tls_no_client_certificate } } storage = StorageAsync::Client.send :new, Storage::Helpers.config_with(config_obj) - assert_connection(storage) + assert_client_config(storage, **config_obj) end def test_tls_client_cert_rsa @@ -213,7 +208,7 @@ def test_tls_client_cert_rsa } } storage = StorageAsync::Client.send :new, Storage::Helpers.config_with(config_obj) - assert_connection(storage) + assert_client_config(storage, **config_obj, test_cert_type: :rsa) end def test_tls_client_cert_dsa @@ -226,7 +221,7 @@ def test_tls_client_cert_dsa } } storage = StorageAsync::Client.send :new, Storage::Helpers.config_with(config_obj) - assert_connection(storage) + assert_client_config(storage, **config_obj, test_cert_type: :dsa) end def test_tls_client_cert_ec @@ -239,7 +234,7 @@ def test_tls_client_cert_ec } } storage = StorageAsync::Client.send :new, Storage::Helpers.config_with(config_obj) - assert_connection(storage) + assert_client_config(storage, **config_obj, test_cert_type: :ec) end def test_acl @@ -249,7 +244,7 @@ def test_acl password: 'p4ssW0rd' } storage = StorageAsync::Client.send :new, Storage::Helpers.config_with(config_obj) - assert_connection(storage) + assert_client_config(storage, **config_obj) end def test_acl_tls @@ -262,20 +257,42 @@ def test_acl_tls password: 'p4ssW0rd' } storage = StorageAsync::Client.send :new, Storage::Helpers.config_with(config_obj) - assert_connection(storage) + assert_client_config(storage, **config_obj) end private - def assert_connection(client) - client.flushdb - client.set('foo', 'bar') - assert_equal 'bar', client.get('foo') - end + def assert_client_config(conn, **conf) + client = conn.instance_variable_get(:@redis_async) - def assert_sentinel_client(client) - inner_client = client.instance_variable_get(:@redis_async) - assert_instance_of Async::Redis::SentinelsClient, inner_client + url = URI(conf[:url]) + host, port = client.endpoint.address + assert_equal url.host, host + assert_equal url.port, port + + unless conf[:username].to_s.strip.empty? && conf[:password].to_s.strip.empty? + assert_instance_of ThreeScale::Backend::StorageAsync::Client::AuthenticatedRESP2, client.protocol + username, password = client.protocol.instance_variable_get(:@credentials) + assert_equal conf[:username], username + assert_equal conf[:password], password + end + + unless conf[:ssl_params].to_s.strip.empty? + assert_instance_of Async::IO::SSLEndpoint, client.endpoint + %i[ca_file ca_path].each do |key| + assert_equal conf[:ssl_params][key], client.endpoint.options[:ssl_context].send(key) + end + assert_instance_of(OpenSSL::X509::Certificate, client.endpoint.options[:ssl_context].cert) unless conf[:ssl_params][:cert].to_s.strip.empty? + + unless conf[:test_cert_type].to_s.strip.empty? + expected_classes = { + rsa: OpenSSL::PKey::RSA, + dsa: OpenSSL::PKey::DSA, + ec: OpenSSL::PKey::EC, + } + assert_instance_of(expected_classes[conf[:test_cert_type]], client.endpoint.options[:ssl_context].key) unless conf[:ssl_params][:key].to_s.strip.empty? + end + end end def assert_sentinel_config(conn, url:, **conf) @@ -285,6 +302,8 @@ def assert_sentinel_config(conn, url:, **conf) role = conf[:role] || :master password = client.instance_variable_get(:@protocol).instance_variable_get(:@password) + assert_instance_of Async::Redis::SentinelsClient, client + assert_equal name, client.instance_variable_get(:@master_name) assert_equal role, client.instance_variable_get(:@role) diff --git a/test/unit/storage_sync_test.rb b/test/unit/storage_sync_test.rb index 79e8fa54..0040b726 100644 --- a/test/unit/storage_sync_test.rb +++ b/test/unit/storage_sync_test.rb @@ -11,19 +11,21 @@ def test_basic_operations end def test_redis_host_and_port - storage = StorageSync.send :new, url('127.0.0.1:6379') - assert_connection(storage) + config_obj = url('127.0.0.1:6379') + storage = StorageSync.send :new, config_obj + assert_client_config(storage, **config_obj) end def test_redis_url - storage = StorageSync.send :new, url('redis://127.0.0.1:6379/0') - assert_connection(storage) + config_obj = url('redis://127.0.0.1:6379/0') + storage = StorageSync.send :new, config_obj + assert_client_config(storage, **config_obj) end def test_redis_unix config_obj = url('unix:///tmp/redis_unix.6379.sock') storage = StorageSync.send :new, config_obj - assert_client_config(storage, url: config_obj[:url]) + assert_client_config(storage, **config_obj) end def test_redis_protected_url @@ -45,8 +47,7 @@ def test_sentinels_connection_string } conn = StorageSync.send :new, Storage::Helpers.config_with(config_obj) - assert_sentinel_config(conn) - assert_client_config(conn, url: config_obj[:url], + assert_sentinel_config(conn, url: config_obj[:url], sentinels: [{ host: '127.0.0.1', port: 26_379 }, { host: '127.0.0.1', port: 36_379 }]) end @@ -58,8 +59,7 @@ def test_sentinels_connection_string_escaped } conn = StorageSync.send :new, Storage::Helpers.config_with(config_obj) - assert_sentinel_config(conn) - assert_client_config(conn, url: config_obj[:url], + assert_sentinel_config(conn, url: config_obj[:url], sentinels: [{ host: '127.0.0.1', port: 26_379, password: 'passw,ord' }, { host: '127.0.0.1', port: 36_379 }]) end @@ -71,8 +71,7 @@ def test_sentinels_connection_array_strings } conn = StorageSync.send :new, Storage::Helpers.config_with(config_obj) - assert_sentinel_config(conn) - assert_client_config(conn, url: config_obj[:url], + assert_sentinel_config(conn, url: config_obj[:url], sentinels: [{ host: '127.0.0.1', port: 26_379 }, { host: '127.0.0.1', port: 36_379 }]) end @@ -87,8 +86,7 @@ def test_sentinels_connection_array_hashes } conn = StorageSync.send :new, Storage::Helpers.config_with(config_obj) - assert_sentinel_config(conn) - assert_client_config(conn, url: config_obj[:url], + assert_sentinel_config(conn, url: config_obj[:url], sentinels: config_obj[:sentinels].compact.reject(&:empty?)) end @@ -109,8 +107,7 @@ def test_sentinels_simple_url } conn = StorageSync.send :new, Storage::Helpers.config_with(config_obj) - assert_sentinel_config(conn) - assert_client_config(conn, url: "redis://#{config_obj[:url]}", + assert_sentinel_config(conn, url: "redis://#{config_obj[:url]}", sentinels: [{ host: '127.0.0.1', port: 26_379 }]) end @@ -124,8 +121,7 @@ def test_sentinels_array_hashes_default_port } conn = StorageSync.send :new, Storage::Helpers.config_with(config_obj) - assert_sentinel_config(conn) - assert_client_config(conn, url: config_obj[:url], + assert_sentinel_config(conn, url: config_obj[:url], sentinels: [{ host: '127.0.0.1', port: default_sentinel_port }, { host: '192.168.1.1', port: default_sentinel_port }, { host: '192.168.1.2', port: default_sentinel_port }, @@ -142,8 +138,7 @@ def test_sentinels_array_strings_default_port } conn = StorageSync.send :new, Storage::Helpers.config_with(config_obj) - assert_sentinel_config(conn) - assert_client_config(conn, url: config_obj[:url], + assert_sentinel_config(conn, url: config_obj[:url], sentinels: [{ host: '127.0.0.2', port: default_sentinel_port }, { host: '127.0.0.1', port: default_sentinel_port }, { host: '192.168.1.1', port: default_sentinel_port }, @@ -160,8 +155,7 @@ def test_sentinels_array_hashes_password } conn = StorageSync.send :new, Storage::Helpers.config_with(config_obj) - assert_sentinel_config(conn) - assert_client_config(conn, url: config_obj[:url], + assert_sentinel_config(conn, url: config_obj[:url], sentinels: [{ host: '192.168.1.1', port: 3333, password: 'abc' }, { host: '192.168.1.2', port: 4444 }, { host: '192.168.1.3', port: 5555 }]) @@ -176,8 +170,7 @@ def test_sentinels_array_strings_password } conn = StorageSync.send :new, Storage::Helpers.config_with(config_obj) - assert_sentinel_config(conn) - assert_client_config(conn, url: config_obj[:url], + assert_sentinel_config(conn, url: config_obj[:url], sentinels: [{ host: '192.168.1.1', port: 3333, password: 'abc' }, { host: '192.168.1.2', port: 4444 }, { host: '192.168.1.3', port: 5555 }]) @@ -192,8 +185,7 @@ def test_sentinels_correct_role } conn = StorageSync.send :new, Storage::Helpers.config_with(config_obj) - assert_sentinel_config(conn) - assert_client_config(conn, url: config_obj[:url], + assert_sentinel_config(conn, url: config_obj[:url], sentinels: [{ host: '127.0.0.1', port: 26_379 }], role: role) end @@ -220,12 +212,6 @@ def test_role_empty_when_sentinels_does_not_exist refute redis_cfg.key?(:role) end - def test_redis_no_scheme - assert_nothing_raised do - StorageSync.send :new, url('backend-redis:6379') - end - end - def test_redis_unknown_scheme assert_raise ArgumentError do StorageSync.send :new, url('myscheme://127.0.0.1:6379') @@ -251,10 +237,10 @@ def test_tls_no_client_certificate } } storage = StorageSync.send :new, Storage::Helpers.config_with(config_obj) - assert_connection(storage) + assert_client_config(storage, **config_obj) end - def test_tls_client_cert_rsa + def test_tls_client_cert config_obj = { url: 'rediss://localhost:46379/0', ssl_params: { @@ -264,33 +250,7 @@ def test_tls_client_cert_rsa } } storage = StorageSync.send :new, Storage::Helpers.config_with(config_obj) - assert_connection(storage) - end - - def test_tls_client_cert_dsa - config_obj = { - url: 'rediss://localhost:46379/0', - ssl_params: { - ca_file: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'ca-root-cert.pem')), - cert: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'redis-dsa.crt')), - key: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'redis-dsa.pem')) - } - } - storage = StorageSync.send :new, Storage::Helpers.config_with(config_obj) - assert_connection(storage) - end - - def test_tls_client_cert_ec - config_obj = { - url: 'rediss://localhost:46379/0', - ssl_params: { - ca_file: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'ca-root-cert.pem')), - cert: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'redis-ec.crt')), - key: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'redis-ec.key')) - } - } - storage = StorageSync.send :new, Storage::Helpers.config_with(config_obj) - assert_connection(storage) + assert_client_config(storage, **config_obj) end def test_acl @@ -300,7 +260,7 @@ def test_acl password: 'p4ssW0rd' } storage = StorageSync.send :new, Storage::Helpers.config_with(config_obj) - assert_connection(storage) + assert_client_config(storage, **config_obj) end def test_acl_tls @@ -313,25 +273,36 @@ def test_acl_tls password: 'p4ssW0rd' } storage = StorageSync.send :new, Storage::Helpers.config_with(config_obj) - assert_connection(storage) + assert_client_config(storage, **config_obj) end private - def assert_connection(client) - client.flushdb - client.set('foo', 'bar') - assert_equal 'bar', client.get('foo') + def assert_client_config(conn, **conf) + config = conn.instance_variable_get(:@client).instance_variable_get(:@config) + + if conf[:url].to_s.strip.empty? + assert_equal conf[:path], config.path + else + url = URI(conf[:url]) + assert_equal url.host, config.host + assert_equal url.port, config.port + end + + assert_equal conf[:username] || 'default', config.username + assert_equal conf[:password], config.password + + unless conf[:ssl_params].to_s.strip.empty? + %i[ca_file ca_path cert key].each do |key| + assert_equal conf[:ssl_params][key], config.ssl_params[key] + end + end end - def assert_sentinel_config(client) + def assert_sentinel_config(client, **conf) config = client.instance_variable_get(:@client).instance_variable_get(:@config) assert config.sentinel? - end - - def assert_client_config(conn, url:, **conf) - config = conn.instance_variable_get(:@client).instance_variable_get(:@config) - assert_equal URI(url).host, config.name + assert_equal URI(conf[:url]).host, config.name assert_equal conf[:role] || :master, config.instance_variable_get(:@role) conf[:sentinels].each_with_index do |s, i| assert_equal s[:host], config.sentinels[i].host From a126a9b781b02f040d7743f50c3788319cc23de0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Fri, 26 Apr 2024 13:46:12 +0200 Subject: [PATCH 35/54] Use single redis protocol Move it to it's own file --- lib/3scale/backend/storage_async/client.rb | 24 ++------------- .../redis/protocol/authenticated_resp2.rb | 29 +++++++++++++++++++ 2 files changed, 31 insertions(+), 22 deletions(-) create mode 100644 lib/async/redis/protocol/authenticated_resp2.rb diff --git a/lib/3scale/backend/storage_async/client.rb b/lib/3scale/backend/storage_async/client.rb index 5bfd0ed3..466e9d86 100644 --- a/lib/3scale/backend/storage_async/client.rb +++ b/lib/3scale/backend/storage_async/client.rb @@ -1,6 +1,7 @@ require 'async/io' require 'async/redis/client' require 'async/redis/sentinels' +require 'async/redis/protocol/authenticated_resp2' module ThreeScale module Backend @@ -65,23 +66,6 @@ def close DEFAULT_HOST = 'localhost'.freeze DEFAULT_PORT = 6379 - # Custom Redis Protocol class which sends the AUTH command on every new connection - # to authenticate before sending any other command. - class AuthenticatedRESP2 - def initialize(credentials) - @credentials = credentials - end - - def client(stream) - client = Async::Redis::Protocol::RESP2.client(stream) - - client.write_request(["AUTH", *@credentials]) - client.read_response # Ignore response. - - client - end - end - def initialize_client(opts) return init_host_client(opts) unless opts.key? :sentinels @@ -107,11 +91,7 @@ def make_redis_protocol(opts) uri = URI(opts[:url] || "") credentials = [ uri.user || opts[:username], uri.password || opts[:password]] - if credentials.any? - AuthenticatedRESP2.new(credentials) - else - Async::Redis::Protocol::RESP2 - end + Async::Redis::Protocol::AuthenticatedRESP2.new(credentials) end # SSL endpoint if scheme is `rediss:`, TCP endpoint otherwise. diff --git a/lib/async/redis/protocol/authenticated_resp2.rb b/lib/async/redis/protocol/authenticated_resp2.rb new file mode 100644 index 00000000..affab039 --- /dev/null +++ b/lib/async/redis/protocol/authenticated_resp2.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require 'async/redis/protocol/resp2' + +module Async + module Redis + module Protocol + + # Custom Redis Protocol class which sends the AUTH command on every new connection + # to authenticate before sending any other command. + class AuthenticatedRESP2 + def initialize(credentials) + @credentials = credentials + end + + def client(stream) + client = Async::Redis::Protocol::RESP2.client(stream) + + if @credentials.any? + client.write_request(["AUTH", *@credentials]) + client.read_response # Ignore response. + end + + client + end + end + end + end +end From 4dc7cab14273a98d662c2c0f292b89d67895abae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Mon, 29 Apr 2024 09:55:25 +0200 Subject: [PATCH 36/54] Remove fake certificates for tests Not needed after we simplified and cleaned up test scripts --- .gitleaks.toml | 8 ---- Gemfile | 1 + Gemfile.lock | 2 + script/config/ca-root-cert.pem | 33 -------------- script/config/ca-root-key.pem | 52 ---------------------- script/config/redis-client.crt | 31 ------------- script/config/redis-client.key | 52 ---------------------- script/config/redis-dsa.crt | 53 ----------------------- script/config/redis-dsa.pem | 26 ----------- script/config/redis-ec.crt | 21 --------- script/config/redis-ec.key | 5 --- script/config/redis-server-acl.conf | 1 - script/config/redis-server.crt | 31 ------------- script/config/redis-server.key | 52 ---------------------- test/test_helpers/certificates.rb | 67 +++++++++++++++++++++++++++++ test/unit/storage_async_test.rb | 67 +++++++++++++---------------- 16 files changed, 99 insertions(+), 403 deletions(-) delete mode 100644 .gitleaks.toml delete mode 100644 script/config/ca-root-cert.pem delete mode 100644 script/config/ca-root-key.pem delete mode 100644 script/config/redis-client.crt delete mode 100644 script/config/redis-client.key delete mode 100644 script/config/redis-dsa.crt delete mode 100644 script/config/redis-dsa.pem delete mode 100644 script/config/redis-ec.crt delete mode 100644 script/config/redis-ec.key delete mode 100644 script/config/redis-server-acl.conf delete mode 100644 script/config/redis-server.crt delete mode 100644 script/config/redis-server.key create mode 100644 test/test_helpers/certificates.rb diff --git a/.gitleaks.toml b/.gitleaks.toml deleted file mode 100644 index e1120f0a..00000000 --- a/.gitleaks.toml +++ /dev/null @@ -1,8 +0,0 @@ -[allowlist] - description = "Global Allowlist" - - # Ignore based on any subset of the file path - paths = [ - # Ignore all fake private keys, they are for tests - '''script\/config\/.+\.(pem|key)$''' - ] diff --git a/Gemfile b/Gemfile index a8294240..2e53feb6 100644 --- a/Gemfile +++ b/Gemfile @@ -31,6 +31,7 @@ group :test do gem 'rspec', '~> 3.7.0', require: nil gem 'codeclimate-test-reporter', '~> 0.6.0', require: nil gem 'async-rspec' + gem 'fakefs' end group :development do diff --git a/Gemfile.lock b/Gemfile.lock index 0558b117..bd229842 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -76,6 +76,7 @@ GEM docile (1.1.5) dotenv (2.8.1) dry-initializer (3.0.3) + fakefs (2.5.0) falcon (0.42.3) async async-container (~> 0.16.0) @@ -260,6 +261,7 @@ DEPENDENCIES codeclimate-test-reporter (~> 0.6.0) daemons (= 1.2.4) dotenv (~> 2.8.1) + fakefs falcon (~> 0.35) gli (~> 2.16.1) hiredis-client diff --git a/script/config/ca-root-cert.pem b/script/config/ca-root-cert.pem deleted file mode 100644 index eeb3e2a0..00000000 --- a/script/config/ca-root-cert.pem +++ /dev/null @@ -1,33 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIFnzCCA4egAwIBAgIUAJ8jy2MbK36XVVuAZHMryfbJRLwwDQYJKoZIhvcNAQEL -BQAwXjELMAkGA1UEBhMCWFgxFTATBgNVBAcMDERlZmF1bHQgQ2l0eTEPMA0GA1UE -CgwGUmVkSGF0MQ8wDQYDVQQLDAYzc2NhbGUxFjAUBgNVBAMMDUFwaXNvbmF0b3Ig -Q0EwIBcNMjMxMjE5MTQwMjU2WhgPMjA1MTA1MDYxNDAyNTZaMF4xCzAJBgNVBAYT -AlhYMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxDzANBgNVBAoMBlJlZEhhdDEPMA0G -A1UECwwGM3NjYWxlMRYwFAYDVQQDDA1BcGlzb25hdG9yIENBMIICIjANBgkqhkiG -9w0BAQEFAAOCAg8AMIICCgKCAgEA3JCowOXBo9q+DlgYEnPsnmnCVQPGuLsP7qU7 -1lmo0+7y/T9Tg+E1pOgiJVlzYX8P+bGmAnNhJdBDONdgY0b2zAjFfGOngC2eDRmJ -CR56OccubYVbdOt/2HkMNbvlORdOutwBe2BZeLhQbQ0+znW7WN8A6sh7hPtfZVJD -fAZ0W6WQFCwfDW64njZP5/9sfvQm8pRmLrNykPOWOOQw65zvicvG0IF8nFqSChrM -QUDAtGa92v/Kuykl0aAZ9wOVaAMArxiudqNlpDXw6vsHDGmKklcDkPhqA0LWjoqQ -aIGq++PczuGI5rpisGJ099bnXvZFbhiOA3eT0m7V40g1r+AI6lF+Cd+BwxSV0wXM -9YtdsYOrGbj9k9iSIhy/8uInbpVWY4gkF40BFLNcMIuoq96eOWrKlgIoAqbEqvrT -D4+ecTdD7c57K3Xbvz7NTq/sFhPd//2aqsvf0x0TjGDnCfGCNtjJ6EjYWyQ0p/79 -DP/1v07xF7fhl226deMpFOF55KD++bCvq5Y83HcEd0MNHLcmVjp6OTEu8WN3BL94 -Foqo2CbMCz6/0mtDUicdHm7OWihYtD0Wl6LSfO9nFp+xRO45STP97uwpAu4uKESD -fvT/+4GUJxa4p1IftcDcLotVRI+2CHrW4QDwO/4CjqmT71KDgvFK4ojd3ZOFgLgS -edlvSccCAwEAAaNTMFEwHQYDVR0OBBYEFFCdcRM4rv32FPeptnMGuO42anduMB8G -A1UdIwQYMBaAFFCdcRM4rv32FPeptnMGuO42anduMA8GA1UdEwEB/wQFMAMBAf8w -DQYJKoZIhvcNAQELBQADggIBAD6yiFPClxwGkvxvncomM7ujNK2mZDP923HBegoA -I+O86QdDrV19dl+GK1n3+iaFWlLTWXI9PinMyG756QbUbO3p4sVHPWtA9XOnB2pr -LchaFqY91MFeF3n/qbhT9z1KwOqwHsxnS1EZp9GNKA7lZHEpiTf6NvjlShSl1kGC -YrXgD67eJvIqQuzD4s/fCkLLIv7TG7Goo2YJ9+y5f5WXEULXi5OMxnwI+mvloAFY -FTV93OEjJ1YGlJ0u0bnQBF9Wdtz55442CrotSvxU58gVhA9jCjKypDVr0Nx9Rz9v -pdr4FlclYgOe0MPlFUnABTFHTtiCSpLvtoyt7GoLSlL7xdSPYhd6XctE/v7xgHHR -X4ETnQij55Qce3IZHVTcQbzm6RNtJ5QMJkyEdrUPW4kK4S+lDMFJtlgJbXrjNSqZ -V+pymQvlFRsHo3EpLxWgRWgDXvDge/3kNmIh06rtHDAVBnRODiMayl7gpq1jQcAA -HSdJmmvtmEkJN8sZGNJY8vBnWxC/XDBWHXt0Xs3XUomnKJIH6tJZ92nDvnMKopI2 -qIFZ176nQj1w/Oo6QmhRiEtE+NK8s+YU5eHnKpkLWxtzKqH98ShSQs7rCE0Bz3Q1 -p3T1xNYj60ZmJ8q6f8GXBHdN/sO19nvjwejBwC61OjIsYtweA7rG3QERylNwRfZA -mJQc ------END CERTIFICATE----- diff --git a/script/config/ca-root-key.pem b/script/config/ca-root-key.pem deleted file mode 100644 index 316a22e4..00000000 --- a/script/config/ca-root-key.pem +++ /dev/null @@ -1,52 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDckKjA5cGj2r4O -WBgSc+yeacJVA8a4uw/upTvWWajT7vL9P1OD4TWk6CIlWXNhfw/5saYCc2El0EM4 -12BjRvbMCMV8Y6eALZ4NGYkJHno5xy5thVt063/YeQw1u+U5F0663AF7YFl4uFBt -DT7OdbtY3wDqyHuE+19lUkN8BnRbpZAULB8NbrieNk/n/2x+9CbylGYus3KQ85Y4 -5DDrnO+Jy8bQgXycWpIKGsxBQMC0Zr3a/8q7KSXRoBn3A5VoAwCvGK52o2WkNfDq -+wcMaYqSVwOQ+GoDQtaOipBogar749zO4YjmumKwYnT31ude9kVuGI4Dd5PSbtXj -SDWv4AjqUX4J34HDFJXTBcz1i12xg6sZuP2T2JIiHL/y4idulVZjiCQXjQEUs1ww -i6ir3p45asqWAigCpsSq+tMPj55xN0Ptznsrddu/Ps1Or+wWE93//Zqqy9/THROM -YOcJ8YI22MnoSNhbJDSn/v0M//W/TvEXt+GXbbp14ykU4XnkoP75sK+rljzcdwR3 -Qw0ctyZWOno5MS7xY3cEv3gWiqjYJswLPr/Sa0NSJx0ebs5aKFi0PRaXotJ872cW -n7FE7jlJM/3u7CkC7i4oRIN+9P/7gZQnFrinUh+1wNwui1VEj7YIetbhAPA7/gKO -qZPvUoOC8UriiN3dk4WAuBJ52W9JxwIDAQABAoICAA8/xRsJB3k/n1I4jvl/oEVX -hIMCLtQIwtI6BEgOjNiSNaSCo1CD2oSbM/knFQ4RjdYyjQqwVXAa1PryEQzsFBYJ -KTYvYZ5ACSRYtpu1yreB6NjTln4f5gfeXpS8d92ZmezbVxLjl2v8Ojs/5i65/wjm -6hZRZBDOnErSBC5n99V6PdrXLhJsrEDr+lxM0YB5etPcpK/n4izZWnJ2fFdd00XQ -m99AhI/+xwgB0EtWobwXOlsQwkBsWNwEWKc2TTcSthJk/3LeWyGeRRjJ4DPgfB8f -+vQj6JhvH4xdUhyXkm211EdNWnyxOC9/rxyPdBGhry0sb20X8FVlKk57aJLPR25i -8/XdzsLRh9F3rUkF3cMUAblbUIBuBK8+uGcpFT9rj+j0qarVku1xezvii7sbJvfB -rec5VAs06zpjETXXOQXY+b8KIy/lladEYMz9elllJJukdHz4ev15KEE6FkVZCnai -Np/CJ66CGVminu2EUW9DrpLO1IfjgHjSa+BbPUBZMiPX1/nsGLdXvmLB5C+t9u9L -yEDZ8HQkAaixU7nhBtjluCEE3qwEo3wITHH38CTRluOnbY4cx55qvr61AhlKlfLj -yFbPHWm1NiZC1DiSu4rQIhwLRoZ8QFjUXuSuUVMXPLCY5xP0xaAQMoqPt7qalmpL -9M3GCf0PNx7abXJeFhYBAoIBAQDzg86LmKVQvoYH1qT34E5IPhbkYVEGlRPNvhJc -nbyNiyW88KPghiLto5yU5lF5+Gjug6YGxuzfCsILH4Zf1d/cwsTvAWf/vLqxE0hm -jlMa1ncAlGUqOIcJRXwlRdIPeTr6bMobYfej1bnseZRibYPiTdiK2iu5WnqzXoOK -VoLLwZUYhzB0nBlqasF53h0skWZc0OPwAzX6wNvZfsEvlXbmTuuqcxYoTXd2etmw -DgWg0urI7BO95NcVP8OzAvfyia1n32HNqDYUjgglkQCs/Q9VNgg7hInx8tktcF7D -hy9AajVyBliQs44n+qpOWBxkbZf4BlewwSZ9lE6AvVUK7+MBAoIBAQDn36Fvj1wC -29IiP7mLQMRBUB6DXzMHCQTZrWqxPtyHaHNhLCKjvHuqOVHtoMFmDlgMUGDa7uzu -2x1LesJIibx2fqw9wsJYJS9D/aJENKMUQxkPGNYaVsmq9PhQczP3hACFFGeG0e/q -X0izDcZJ2+AOzQdjTlf7/wJstwgKJr0q7BZ0zK6nqmYGWCRlhXeJwY6W5JC840uD -gPQ9zqoehgWiSceqdw9a40jR8RILE3R3fkI7bbgjNMUTb2RXIyAG+OsU4MOAr9/a -7tv7xvc/7OnRk7qML1Nm6H4RX+Ml4GwsrJ4eBTLZk3jK+q9J3zPsDPDimw8Be733 -l3iG+uzl+dTHAoIBAHpyWpeDKA1T4B1s5wLlPTfCUMyByrZem9imrKD5h+g8gbth -3WV2zXVlAcU1kt6b9FeKxx89Q7pbtHrXXUAvUSO2pEPyFxNXGSEVKyC2jD9AfBJE -jCIQzoXw9J3hX+D7+pYVwgLG3jj5rUWlVOGxY9GVhz86uSW4w14SNl6HZbmDG697 -uvfHzlfeWm8nKtBQRIKDXzo+wxPVijbxL773jxP7tzC8MfYODfX7jB6kzlux8vEZ -FEw2F5jNuV/4LCk+5X0HrCw3Wwo26Q3KveY+UUlto/IZkSuiC2Z3zI7RGQdw4HL3 -ke0g/GhJ+Pzv/UbafMHwnoN4fBxFSLwfd/XJ7gECggEBAL/Jkh7TRfi7V3iqms6a -eHnJSiPBV1SHWjKuJKSMaR27AEwslE44NVVb6e8R20WNsbglt6b+no0GQZnP5OjY -vECKj+7Zq6P6xbiIDnI503WL9istkNHiXRYBX24zi5U74q4wbVzvQnDp9Rjc2Nj4 -e7/iSC0UfOPbTmcbkRPMbQUWl5BuSDw4TgELYLd0CKaxqT4XubvyiOAxhlOlH8L1 -ea19NyBBMDst6Wp06katBdVcYm4DV54PJL7pkOYeEHyT4OW47xv+3LqafOwo6xM9 -4zxM46zV7DcfMD8wv5Hxqafd8z7DcAbU2sGRTfFdI/Yn0gqs1GGWt08WQ0PqQo4J -8bsCggEBAIutXenpyzcUOIx9DBnZ+PrK2GYRk6Yrcw8h4QQFqvDyd0kAhW8arkBD -znd83+ok44yL1YbKV7VVCVyXwEwHm/OU486/SVCSVsQBh18wETXoMzBuvaF3bwAe -CsWJgoOv7+Jl72B8V40UE6TZd9LmHamnJ3kN6zMJGt8c+ALLeQmwGMiOI87rgLkK -hnAOUlWtcoiIw9bboN5vFLsOZSvK0rPa13DYOGAyIUW6xxZr+u91fC1C35EOmx4H -CwMgscGDZuurwVxywH2Pwu1PAtHR841RGHQIo2ks7SJmKpy7eClVpwyU7EPpsCCf -9CgB8JQDMpJ28hIdJRNtBjq2CZiNAp4= ------END PRIVATE KEY----- diff --git a/script/config/redis-client.crt b/script/config/redis-client.crt deleted file mode 100644 index 313cb1e6..00000000 --- a/script/config/redis-client.crt +++ /dev/null @@ -1,31 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIFSTCCAzECFC9XxFWG3zqUUSBscj1YdC/tfec/MA0GCSqGSIb3DQEBCwUAMF4x -CzAJBgNVBAYTAlhYMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxDzANBgNVBAoMBlJl -ZEhhdDEPMA0GA1UECwwGM3NjYWxlMRYwFAYDVQQDDA1BcGlzb25hdG9yIENBMCAX -DTIzMTIxOTE0MDczM1oYDzIwNTEwNTA2MTQwNzMzWjBiMQswCQYDVQQGEwJYWDEV -MBMGA1UEBwwMRGVmYXVsdCBDaXR5MQ8wDQYDVQQKDAZSZWRIYXQxDzANBgNVBAsM -BjNzY2FsZTEaMBgGA1UEAwwRQXBpc29uYXRvciBDbGllbnQwggIiMA0GCSqGSIb3 -DQEBAQUAA4ICDwAwggIKAoICAQC1N48cE2K1/H03cl29uf+1KRl1JlQfx10LfY0f -BZ04lNSno17sBW1dIwkY8Gp9QYGPkIwh6M1HOLbV4ytgSwNSFWSi+bB5pl/nf70w -j/wCy1ChzYcJomlPANxV2Hw/MOcOA4BsEGSeqGMZlfT3tc30tAj/in2atMuyy+bh -Gk5rBZZz7ndPVttsBGICaqxt7vSgfi3b0D+KCJasY1iCgDe9+o8lG113zeSVoYLV -Ib2Ru7bg9pULFc1FmISgMy+++Kxw/IZ+tngFx/o/dkHczqQcdV7ixW0tlm3deDE/ -xbjaTHZdjEdq7hGdmliY3155A4Ely/OG+pojR2Fwya9KAggTB3N1jDtQkl275jEU -xk+0msjTakO8RSn4YNT8hY5lZzbQmI3Aqum6EYrIa++t+xaZ3b5F04kqyFwjp7/j -e9IgmeLTjANrMERWB1uu5LjdrUNMrDvf64Lu70m6gTUWT+nBHaqXRvnG/am4VBUx -3043qflncdZdS/x4xt3wHtmNL2Lp1Z3+YSBwo19IvzyBnSHhmiBHcRK2K9lblATi -mfsVInq3++un1GcEfwClRLkdQ2Gly6tIHAiOLTL7eD6J4EvT5Zw6v0Gx/n1QWPz1 -3AigwGCRjZcI8NrtN2GLNs4aUmeUzCBUmirAp4c4vysK9OSkKjBOQw/5beOpypn5 -7QFvJQIDAQABMA0GCSqGSIb3DQEBCwUAA4ICAQCHZci0wC+HWmnwpkHxKo71BOt6 -1OsGKCKZjYn3IwMBxiRKIIic5p9V8VweLkNlGYpnf7y5K4GB+YSOvCDBrvX2U2jk -AR+OogiXTRVeiPa41LmXKPRcRkcgcPP2yelW2Fod6j4UCaWmCOeP+sxaU4gLMlu5 -57h5ifK8OQJXHEfKAVrFFiPmrnL7DMeo64zuF/ylzQeZPgyfmvnePzICuxjlswXS -HcTtSM7o76E9v7/3HDhmlSdR5OdjascZV8hs8b1UBK15hdv5FNLA3BSkplOnVAUE -+qizj57TEO0rTqsSXquAQ2fgHCrWjxane+iOn9PP5WPUJDLkdk/77RrvXkq0W1sR -KL7GO1N8CUvDspjFpIXsIYKok8UN191soVheqAO9u9Yj+fgDP5FeQm/S6oZLd4+0 -C8rz5/E/L0aGHCnnQW6KTQHj1hmAy8AkGZa1SzQUtuR9nHHs/KA6cNf31Gz9ULUn -hyxvcMwhGM0bqe6yuxhDTa+3s/RZN7g5dOFbXneYF+M+qIauRUEaARuoFnhx3IpY -/52Wm1RUhu8kQ0gv0QC3XUnRY9aLcEIgwUYiOyE/XxLzMyDlXRz/pQ9D+YMx5JrH -iiu85IIC5epcUG4x3JLyZCH1PFXiXDOS5uquO08h8SjlWLOm6dIriXuDXi8ClbG9 -nesBi86Zd5W3BJETng== ------END CERTIFICATE----- diff --git a/script/config/redis-client.key b/script/config/redis-client.key deleted file mode 100644 index 8f13e8b6..00000000 --- a/script/config/redis-client.key +++ /dev/null @@ -1,52 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQC1N48cE2K1/H03 -cl29uf+1KRl1JlQfx10LfY0fBZ04lNSno17sBW1dIwkY8Gp9QYGPkIwh6M1HOLbV -4ytgSwNSFWSi+bB5pl/nf70wj/wCy1ChzYcJomlPANxV2Hw/MOcOA4BsEGSeqGMZ -lfT3tc30tAj/in2atMuyy+bhGk5rBZZz7ndPVttsBGICaqxt7vSgfi3b0D+KCJas -Y1iCgDe9+o8lG113zeSVoYLVIb2Ru7bg9pULFc1FmISgMy+++Kxw/IZ+tngFx/o/ -dkHczqQcdV7ixW0tlm3deDE/xbjaTHZdjEdq7hGdmliY3155A4Ely/OG+pojR2Fw -ya9KAggTB3N1jDtQkl275jEUxk+0msjTakO8RSn4YNT8hY5lZzbQmI3Aqum6EYrI -a++t+xaZ3b5F04kqyFwjp7/je9IgmeLTjANrMERWB1uu5LjdrUNMrDvf64Lu70m6 -gTUWT+nBHaqXRvnG/am4VBUx3043qflncdZdS/x4xt3wHtmNL2Lp1Z3+YSBwo19I -vzyBnSHhmiBHcRK2K9lblATimfsVInq3++un1GcEfwClRLkdQ2Gly6tIHAiOLTL7 -eD6J4EvT5Zw6v0Gx/n1QWPz13AigwGCRjZcI8NrtN2GLNs4aUmeUzCBUmirAp4c4 -vysK9OSkKjBOQw/5beOpypn57QFvJQIDAQABAoICAAD/t6b6t2BT7jfERl6BnDZ6 -LcT+y1dqgKmLzV+WpxV1ApAS/zazV/U7uHTd/GRI3Gc3b4JPV3RVTHK04BwVhcTp -z8ZzNGUTRPOQeXE/Ve8JAl6F9MDZl5WDJnPFr/ekNB3YaJiP5ZIXUQaG1FUaODKp -XGRWNdeIuBe59oGWpMiDvR7R+W/jRW3cv1KOlnk7qDcLQwLzbKo83mHAYXBEtynx -q+DTPAppROGLRJ6ao0IepAUs7sjjG6DO1KpSZei6U68CwVYI+PPlNIytgPLv46kx -f26mEnI+IkhmfXUByMFRSxS8bXdaLRgujd732jw2B/v2mhbMIlXf8ppNzkzM4pD/ -6Sx4jMIyfG5FEOoQ4zTwoKR4mqeKJfbfZ60sjdZBCUDjWz3c+GJ2Y3n0WAks5FrJ -FwqzdH2+dDhRPl46gNPCp8+YNqby8ph86eSsU30RwR8nKwiC9Zfa4Dvnr/YGXnel -47GbVIyl+MASFkoHPGfxfdxGc2ujcYOltajbtSMH55Vdle/z1ZqvOHjEjNa0DUoG -TzbMzYkbgIiLq3qShhq/7RgmuD/iVDJVpfD9IC9vk9E0B/U+U4sIpMn/U97rX6ki -CmarIDNZE8NKQzP7U1wH82KzUG6lOa3fikneBuz4GZzuRyy57Wiq8Ofudyk4qQqj -F4Hc7Ux0Ne0Td/uBwqZxAoIBAQDdJYtfAqMHvA/JeuLipFXt6HIE2b66Ms4Df7kF -LReZZQIRQa7vFnXeFRTf1/Q7fp7NJbbqguSJFep5xw/8vUw6ISZZc0ePUYf/Zdko -KbxSgTQFxSKeerbIbPMJR9MLVQcCUBD/N7CGOwj10MFbQ7PdIxr4a4B2N2kphgXH -3X14zITjEqp5NNG9Fea17v5bjgU9O4ewqSG0Dmv3vYl6fBxWaVgljarFXbHVfGFq -g8UuD2tNbXg8DtktB5H2aIw9YbKBQgMAyarC3WKLA8qrNw6l5LhQUanam3OFWJhe -L4ZnA4qpLfPFRLN8GB3Mcj4e8pQSjoQLf2/A2NmiCphpFuNxAoIBAQDRxwAp3q5u -XztwYRAxEsckOccXBOfkQKxRQE1vuBKLYNSmvXmT9xnXu51aI8JNqCdlDDdk5vPn -yBGFW+E6qDgg/Y61T7bHtyWK97jheLRnjkDWhSO0xnJIddNk415RKj18X6o6LuF6 -3syJxYJkMsfnMjv0aCRT+n/0SjnNgok3qHNa2wYdvNYfC4+6kS+2gp8sATHVFG2k -g9wn3ZZrX5juMZAoSV0z1DxFwXdb8fqITC7obL4iYq+7BkkA9Hb/KNdfXiTyHbGk -x+JyvUq0kB9CZ4uAJPqzg/euBdNDxbvDSzcJLvb+QkkoMV2nAgNZ7C7S977s3VVo -CKlB2laJTAT1AoIBAHUQXsqYlryNbL328wuOen5JwbGXokLmie+IOOTojcmBkqqX -3oZpmVNKTZaAtqSXznBCOqXBTEDU/R4hpfDgM+3Bho9rWsIfIqKrw1xjW3FBO8yi -IkVKKF/IbI7JGoqxI9ixkY3IaQVlUsTtEK3DFP9Gigxeu83YXy6/52d5c+XyPZYG -V3952l0BrvM35CWK9HMJhDacEV9DExw9lPzlngqQET0v1OpNQsoD4X2E62gKeX6v -KCqCoawYiceqaDJF+Tw7OWO9XUtx7awxAaPIrVRb0sWRtRwnletMjrus5LS7Rzdv -P895IpnaGkwTXZb8Si3Yi0Wa3Al1wR7qBOv8VcECggEBAKnHZnF8oS4A80lLtWAS -M4w1GtWdidRpJ02HPmTp1FBrqQ/eD7Taa1b3qC1Y/Zv+OQwGLBsE9Gglu1B0gUAI -/4xryoEhp+vRZZgoT8BkAZZCQmWcUb6wUbQlcFaZ0i7cgfDUOCBo6uKXyXDDonlX -xARw2qc/Vg+FkON0431ZZZbfRaVF2JheyelMYSH8+WG8K/jgaqVqqCZa9eZp56C1 -OT+ma2a/oagd4D4Qi3qlyD9hcSF5SRTpTBKQzqrwMBLoyg6MUliGAc08XSDB5OFo -N3R1+yhWRXTenKmCW27hbMptaHiNQyebHDxkX8iXz4CKr4bdLe/kXogZYmtCu4DZ -PrUCggEAfFbIFa3pQy3tqJbVFVSoNfepYLKWkHLXCJmHcIqjCqkWHkqgR3wTgr6V -32844PzbQUm1V7JlZMpUe2zFDCWlwNL2BtLlAl67wancMsdBBvZ2aEGTctWKJaDp -pzWZEbqg8qoVHsj3YWcFm7IocnI03zawuLqi/jKZgaMOIY+2ol/4npAdMCGGbwOk -K4a/QJzD5sM/RTU1/4fvfwfst+HQZxJrrVgDWvYoVFqKiUXx9/KyK7qhQ6URhjRW -oRZt8aqvxj5XE2IFXEP4jeCMIguBYdaYAAQdezkYlIYAYA/H5zkF2CvLnaZaL7t3 -ehRUDfVprDaPWXW2yZMD3hgaNiY+IA== ------END PRIVATE KEY----- diff --git a/script/config/redis-dsa.crt b/script/config/redis-dsa.crt deleted file mode 100644 index 42bc5f19..00000000 --- a/script/config/redis-dsa.crt +++ /dev/null @@ -1,53 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIJYTCCB0kCFFD7Tn6DRSrdioXN32i/0W8obhXbMA0GCSqGSIb3DQEBCwUAMF4x -CzAJBgNVBAYTAlhYMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxDzANBgNVBAoMBlJl -ZEhhdDEPMA0GA1UECwwGM3NjYWxlMRYwFAYDVQQDDA1BcGlzb25hdG9yIENBMCAX -DTIzMTIyMDEyNTU0NVoYDzIwNTEwNTA3MTI1NTQ1WjBaMQswCQYDVQQGEwJYWDEV -MBMGA1UEBwwMRGVmYXVsdCBDaXR5MQ8wDQYDVQQKDAZSZWRIYXQxDzANBgNVBAsM -BjNzY2FsZTESMBAGA1UEAwwJbG9jYWxob3N0MIIGQjCCBDUGByqGSM44BAEwggQo -AoICAQCEmNNJH6M5zgr+0Ve18m+36JUy45IWGbwj/ABku/vbvGb5YMvgIFORHXoE -bg9/71PGWjosNRZKQS+2YlbmF2YAz6F2AEy+CHfV6ypmpVJAqoqeLWgnLfAPXBV8 -e2aM156dIWhfp/mcf1QkOtiYtkbfK4jFGz84Kq/BhwVdPmRex5FGHGOrItLJx8DX -YZivgMj0+cRVVVEV2Pu20TrHMiDrjZVK0w1Cqu9EdptjbFrNDNIKUCRZhhbWdG0Q -JuIlbrgRKs1SRnvEZwH9/IDqDEexkSx5SmWHf9fEPqd4vpwjKfGkTLYUYbkw+sJg -dKFZSx+0SNqCcbzOrO/2YxpwidUbvxUu2zYbWFHnsEwkN6+DZ4j9HWT+9LeXulQn -BYtZn1Rozn6prCRKkpP8RuZX9Khd/eWVBopHZ6a+vnkY3TIw9eA3m8XcgfreUygn -RkGVFdAbhsWsVYMusq8Dd7DMXiOsYMpSD+2ZIqT5LFIDWQmWmy6LiA+rpk8su02T -0k46G2rpxHz1LYnrZ0ruqvSXF8daESV7jMdSjLvcTdosz2xstT4WgKeklVA7zQXW -aT0vrbxIz65xBwhsywC3NmGsd0YdJJ89swpi0pj6qwZR3LpHzHCFtFaHRrvs8S+Q -u56HoXge+/5gMa6kpAogG88j1QvVV5uUdsiA5nq0olRKsMImKwIdANO+TEKeQXz6 -9pdp9HBYCPcKTV4CI7QwWKQzz4MCggIAEJUHeHJjhGD32KBDoLz/DNEkgg/gxTOs -AADNaO6fEinhKNCy/gqt9ZspbMRr7LP35fIAPkudW/fNzuZ4iBJ0rwPEC6u1N4Cd -kd5e7hpSwU4pB9ZfdzzNRc+y0qIv4Nb1hmzk+zUWVx6kFOWA3Ut8ja/q2F4FAg1b -FnSb2/wF27B0Kyetbc04UQUWPxGeD67QqwB6FU2hEQXSPdjERsDjRjCKCbP75LiG -0kKyMB2Bu1qj15LG9pjI07wYQHUAGcYV5nfuL2T1XYNEWmYbhUWcChHwjfqFRZ1v -UmrC8mR9/m8Sqk+kHs4kvB/nOG4yrFAzgjQ0ntOaJF1p8XuWAgUTzGAlpk/XpPde -/Y2vy71BD3+pyaP3bn1stAtFedh6afomxzgTxpOrD6vFFs10l2R/Y1Cog4rcOvhP -1aTrY8tyyCTztWe+G+JsTqUKzZR9SwL8yJjRL8TpB9F+T1Gc/e9pGMG0mDNL97oK -SFY5yxBKe7qoscDnu2yH8ekYyf8QAIf31TFw5bRWKlm33DVxRR0R8l4jWlqVEHzU -sRzhvZolcu+8HzGNww4+AxuPx8dn5KGQai0lPwEX7uv6znKLeAjSlN/Q8SFViiID -+gcp5mox1JfifWK7xtjThLuGe+siYmh07uVC+J92Sj7xFxkIBkqtjbEwjJS4ZqPK -HT/9e79LCVoDggIFAAKCAgBdMYses1sc8lUgzIjqAvzaAzutjnzejzw2y4Fjoqr4 -pnyDPtFHOxsc4g+ifYSYxtpzJOpJZdwBf8eaaBhZURLA/VjrO2zs/Q03irSX3XYn -ge7uJOgm1CEjTaWmxdGCwi5QjPbTiQPc+c9Up5G+TJEKyEwCyBA+JKuMB8yJeH0b -Ww14+mIu83it+eK1JGSP0Ps4YAEtsgcaDzJQMJy9sMezQxoNn9XA9A5xQX0UpudK -pqtPYzPiENeI9XqeScxFeqpW4W+0khsFS0nXtM5oIyuymRnhtHZIGgnmyKA3i4n5 -X8FQobJmiFY037r1IOQ7Y8m0R3DQaIaD6mmW0z6Eev6Qgimd4jrGHRvTAsWgutUy -P9SWvGJG0gAvawCbaHJw2kd5fnBN7rBNOVZ+dqFU/jVqc+/zhbGUoiqDBqZoqZ8U -NW3rDHLDYWPMVP56dESTOFB31ZhYuJJ7kbYWXJ4NCKyYzLOzuDDl1bCayQct5s9E -0V9fJn5nd0bJIHtDZLw+4Zi2blPR4VD8dMt+UCiOHDQvKuW/XbholTzCGZDvDAqb -iHbN1IEFrOcteFBy+ecCeN8uAZHPc32aEjBkz3oedxOA6sMlVxXsz7xmQkvdpIkc -+6C1eohn3FqztMdUG4tQ6loJDD0qv7P9Lz1l4ybiPA4ktBOnrNbelbKwhNOqThGC -AjANBgkqhkiG9w0BAQsFAAOCAgEAkuKFPngVOQV/oMbq59WnatOayIykdA+wwLFU -/0c21kQxOYtpl5oMAF8PmODNXWqeG8EyKWPKbwWkUuLDo3qDJ0gZ3SthCHvVekge -ggJ5teNZkVWAJsHLJlS6p/zRhLlBN5JIjlaJE3TusAxtnCWFHW2j0vIIcUhj1FCV -NXE3S9o/lx1HKC7dR1f/7L3xic2aWl/biM5WO6DSVE4piGkotS5b/QPxN76NC3fA -o2TO3eX87nrnUruBjdB/c1l6rX5nuCzn3Bj9NRyDpJSB606ou1F+VG0+QkbUQwSt -m6/DZWd4fZ3Qs/Ago7JtnrIFOf7g8d0R7AYHKgTrvhA7oYx/Vdq6JSAQkXHmtowB -b8HojKSILrlb8RuvDtDyt1/ccdVLcagmQrX36DhrDSxFRfYCgYluTkgqgXNn0Pv9 -bH/Dq2G631zUqx+NJydCGqsyJ0DEYb7ZZF29XXAS6INttXX0MBnqpqfL/Avbku3F -6cqDRK6XMaREqOjVYEvXVZUKRIqRiSdbYdOKc78ROvnbfVxRGWdKziluvrD8fqmP -gQBBO/NgukK0hOnTAEtyoHLK/KWHxKKrhay8zTx+olk3ajO90kFAm3GG7Qrl3eO/ -DeJaxtjB9w0e+jz/gotkaPe/mWcE1LzGTnHC1OuiEkbzCBFe89o+GWeAt8I9WYHN -FAWNe/g= ------END CERTIFICATE----- diff --git a/script/config/redis-dsa.pem b/script/config/redis-dsa.pem deleted file mode 100644 index ce949ffb..00000000 --- a/script/config/redis-dsa.pem +++ /dev/null @@ -1,26 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEXAIBADCCBDUGByqGSM44BAEwggQoAoICAQCEmNNJH6M5zgr+0Ve18m+36JUy -45IWGbwj/ABku/vbvGb5YMvgIFORHXoEbg9/71PGWjosNRZKQS+2YlbmF2YAz6F2 -AEy+CHfV6ypmpVJAqoqeLWgnLfAPXBV8e2aM156dIWhfp/mcf1QkOtiYtkbfK4jF -Gz84Kq/BhwVdPmRex5FGHGOrItLJx8DXYZivgMj0+cRVVVEV2Pu20TrHMiDrjZVK -0w1Cqu9EdptjbFrNDNIKUCRZhhbWdG0QJuIlbrgRKs1SRnvEZwH9/IDqDEexkSx5 -SmWHf9fEPqd4vpwjKfGkTLYUYbkw+sJgdKFZSx+0SNqCcbzOrO/2YxpwidUbvxUu -2zYbWFHnsEwkN6+DZ4j9HWT+9LeXulQnBYtZn1Rozn6prCRKkpP8RuZX9Khd/eWV -BopHZ6a+vnkY3TIw9eA3m8XcgfreUygnRkGVFdAbhsWsVYMusq8Dd7DMXiOsYMpS -D+2ZIqT5LFIDWQmWmy6LiA+rpk8su02T0k46G2rpxHz1LYnrZ0ruqvSXF8daESV7 -jMdSjLvcTdosz2xstT4WgKeklVA7zQXWaT0vrbxIz65xBwhsywC3NmGsd0YdJJ89 -swpi0pj6qwZR3LpHzHCFtFaHRrvs8S+Qu56HoXge+/5gMa6kpAogG88j1QvVV5uU -dsiA5nq0olRKsMImKwIdANO+TEKeQXz69pdp9HBYCPcKTV4CI7QwWKQzz4MCggIA -EJUHeHJjhGD32KBDoLz/DNEkgg/gxTOsAADNaO6fEinhKNCy/gqt9ZspbMRr7LP3 -5fIAPkudW/fNzuZ4iBJ0rwPEC6u1N4Cdkd5e7hpSwU4pB9ZfdzzNRc+y0qIv4Nb1 -hmzk+zUWVx6kFOWA3Ut8ja/q2F4FAg1bFnSb2/wF27B0Kyetbc04UQUWPxGeD67Q -qwB6FU2hEQXSPdjERsDjRjCKCbP75LiG0kKyMB2Bu1qj15LG9pjI07wYQHUAGcYV -5nfuL2T1XYNEWmYbhUWcChHwjfqFRZ1vUmrC8mR9/m8Sqk+kHs4kvB/nOG4yrFAz -gjQ0ntOaJF1p8XuWAgUTzGAlpk/XpPde/Y2vy71BD3+pyaP3bn1stAtFedh6afom -xzgTxpOrD6vFFs10l2R/Y1Cog4rcOvhP1aTrY8tyyCTztWe+G+JsTqUKzZR9SwL8 -yJjRL8TpB9F+T1Gc/e9pGMG0mDNL97oKSFY5yxBKe7qoscDnu2yH8ekYyf8QAIf3 -1TFw5bRWKlm33DVxRR0R8l4jWlqVEHzUsRzhvZolcu+8HzGNww4+AxuPx8dn5KGQ -ai0lPwEX7uv6znKLeAjSlN/Q8SFViiID+gcp5mox1JfifWK7xtjThLuGe+siYmh0 -7uVC+J92Sj7xFxkIBkqtjbEwjJS4ZqPKHT/9e79LCVoEHgIceDE1WAkxmvBcKRSw -mSyEtLORubfpcBEwmPhm/A== ------END PRIVATE KEY----- diff --git a/script/config/redis-ec.crt b/script/config/redis-ec.crt deleted file mode 100644 index 2a4e0cad..00000000 --- a/script/config/redis-ec.crt +++ /dev/null @@ -1,21 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDdjCCAV4CFFD7Tn6DRSrdioXN32i/0W8obhXaMA0GCSqGSIb3DQEBCwUAMF4x -CzAJBgNVBAYTAlhYMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxDzANBgNVBAoMBlJl -ZEhhdDEPMA0GA1UECwwGM3NjYWxlMRYwFAYDVQQDDA1BcGlzb25hdG9yIENBMCAX -DTIzMTIyMDEyNTQ1OFoYDzIwNTEwNTA3MTI1NDU4WjBaMQswCQYDVQQGEwJYWDEV -MBMGA1UEBwwMRGVmYXVsdCBDaXR5MQ8wDQYDVQQKDAZSZWRIYXQxDzANBgNVBAsM -BjNzY2FsZTESMBAGA1UEAwwJbG9jYWxob3N0MFkwEwYHKoZIzj0CAQYIKoZIzj0D -AQcDQgAES6kvZIlhbsg0P/ZP/1tqJSbjQN09bsTj1Lz5kZ+gEErS+auXdMx+WRLC -NUsh/VgVNDy3TZlx0k67NW1wmxkI6jANBgkqhkiG9w0BAQsFAAOCAgEADMi2NbQj -laaRAcmhNHliCZsHuRCvQvmIikILYP45ec6RH1IMB4SngEin1ePdd5Psyd+jE0Hm -2mi4QTuscg6zL4lf/dx/7oam6V8Cr0vhHvqD/nW2nFHQCT22VjuoBdcAGvt3MVmK -H6mnYwxHtxkSvPq726CU0T3yujlCoTaue0ahN31J7bVqbM8muG6sZnMTlloprbSm -Mn5d98W7HIaDPSCipHukv7srZ0+OPNzSEcy1jonGbw5YLjYxM0le4w6TwL2xOVfQ -9N8FaCIUSWGoY36Qqm0cYqRLmb4rE3z9iPxTJfy1/nQi3mbKyKYlRlkAcyFaLY+B -OLs6C2BeLCNm6YtDFls5pgExP2ZOBRmieyEQaiV1kmdRoCeXqa/ulYVUzFNcexoi -Lok52TlS28pXi8eZEykdQjWJ68Deb7KmA9Ue3bWf0xbl9GSyD/77nCBlZlLjQbvF -Y7AFWfvMmCuKbJVGm9sKhcgY4R+4iZPozGLZr313BcPcORKSgTVZU3BiwOAT/O6C -jouJ/+of9UEj22G2YtH3hg5IsdY/Omc8HcVx62j8bNTaIpARbV9YSPoLUDaJ+Ohb -c4ovfh+ksdjCu0/xQzq2BvRtdLo5OZslbpGeLqV4GAWmHYXhfCq+k/Jv33re5DcN -R/Q/rT8ClEIi/3nfWR9eIpklzxGTOeMk8RY= ------END CERTIFICATE----- diff --git a/script/config/redis-ec.key b/script/config/redis-ec.key deleted file mode 100644 index 104b9949..00000000 --- a/script/config/redis-ec.key +++ /dev/null @@ -1,5 +0,0 @@ ------BEGIN EC PRIVATE KEY----- -MHcCAQEEIDEzNzOlKNVpUbwbkG/Ud45bJYJ/SWxTmCMSPrJ4VQ7doAoGCCqGSM49 -AwEHoUQDQgAES6kvZIlhbsg0P/ZP/1tqJSbjQN09bsTj1Lz5kZ+gEErS+auXdMx+ -WRLCNUsh/VgVNDy3TZlx0k67NW1wmxkI6g== ------END EC PRIVATE KEY----- diff --git a/script/config/redis-server-acl.conf b/script/config/redis-server-acl.conf deleted file mode 100644 index ae34bbd7..00000000 --- a/script/config/redis-server-acl.conf +++ /dev/null @@ -1 +0,0 @@ -user apisonator-test on >p4ssW0rd ~* &* +@all diff --git a/script/config/redis-server.crt b/script/config/redis-server.crt deleted file mode 100644 index 528fd806..00000000 --- a/script/config/redis-server.crt +++ /dev/null @@ -1,31 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIFQTCCAykCFB8hf+obX1FxnppOwU9T+j2TV0XMMA0GCSqGSIb3DQEBCwUAMF4x -CzAJBgNVBAYTAlhYMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxDzANBgNVBAoMBlJl -ZEhhdDEPMA0GA1UECwwGM3NjYWxlMRYwFAYDVQQDDA1BcGlzb25hdG9yIENBMCAX -DTIzMTIyMDEzNDY1MFoYDzIwNTEwNTA3MTM0NjUwWjBaMQswCQYDVQQGEwJYWDEV -MBMGA1UEBwwMRGVmYXVsdCBDaXR5MQ8wDQYDVQQKDAZSZWRIYXQxDzANBgNVBAsM -BjNzY2FsZTESMBAGA1UEAwwJbG9jYWxob3N0MIICIjANBgkqhkiG9w0BAQEFAAOC -Ag8AMIICCgKCAgEAtLQcPGp1V91LCMqfjcIsBAvuL+/HH8y81ktN+DrMOgMt3gbm -ngf/zghGX+XQUuYGNfeH7BottqHrh4WgP0RtqZ6KWcc5A4zyqwaDfPcRNPPnZEAe -adUYTuH4ew8mk+KW46gluuDuE4JZ+9gZQBl/y1tdWdlKEczdLlyrfLRYaStmwD0F -Kvp/I3IHMjo6hfRgy41kv9FT41oyZ1zmdUoqsQTSCpEj4iTi4BPHApj8jbgmLBhs -a1q559ORJo6famSV2Zi3Ej/xI3ljPKN+iVrxRY7n2NvqWgIe6xOXxFAuu7DMe51r -+zRp1Y7mhQqTUvrMOF64/P2Jl9LoMNq9GuKlVWTWvMXy9idb5utxEYaWyivvdRAQ -1va1mB9g2kLlRD2rrxV9Rq0llJF3muDr3ch8oX4qrq78AExjFeJF18eYkwri/l7g -qdLlZzmqLreDR9NTyXM4YyzeoX9Yfq7BJUUNQlMgpOF2KGJ1fAIwop6tJAQOiOgn -VUOHKQH6j0/tN7ysGhR3epbjUGs/kCq81oDE3O/wpnxIybNvglG6Btg6AD5ID4qC -aHJyHCuOqSy2LuZzF/R/6noFiCM2qSJQOGu0URRwEjplcUl5JRCNBHfPUPNQaQtn -xzG5bxAaiDNu+sYNTk/CnpUMU0s2Enk3AXMVFzcDOXRSZImmVezysrcY1b0CAwEA -ATANBgkqhkiG9w0BAQsFAAOCAgEAYqlltVsQlOyti++hmUeZ7uyN2OWQJg1zRbvm -7Wi+pDqYGgDy04l/ESfQEjuP85FbWWrTPL5lbH2sGsaMSIMKvS2mUV0V2G0qCVuY -0SgJS2bUbxhgd7aPMFKm8tOwP6iLQgEkqoSV1edBSY95+vjVF1WKXiLLSeSs9YI6 -XmmrF71jbGUDyUi1fd7upxZ18QkB02F9o9MrA1UVHUP6Se8I7m1vOVCtfEnbt2tE -Yi4m66axvJCjl2RZOu7mDoOI+o5rLBVWGCLsJ9Z10QJ6i6cb8PpaEX4JXJ7l1p+C -sreTGQMGsiuz7izsOs767gn6GQ+xuN+zAfTBUJqSnG2JyC8PO6MfDHJ6iS9aUDYV -3GCgrxWVaSsrQkpZidKUulWRWGERJ2MfdnmC3r/Sob7u2ypSRBMWymaZeK1sUrK1 -V9oBopDJo2EKoRYHHLOcLcTKmKNzb0m8X0XsvFOqscOIs6BeSNFonOmSc+wLPtOw -4oIFcaBh+JIMHkm7AoEhI+GFGzmdHUn7dCa+MeBzZeux24XLqlhd5FvfbcU5RFHO -BQ7GyAx6FsY+Y/FBJSw5CboakXATQpfbC7nFTGAFUglxInMqXLgtI2PYiAQEvwuO -Z1HAuzsmpdlXCoPN4jY+q/MQY0xs/E2wAFKmfRoUlvGPuJOF3xo8O82eU3bpkMlt -zZ+AyTg= ------END CERTIFICATE----- diff --git a/script/config/redis-server.key b/script/config/redis-server.key deleted file mode 100644 index 1d909170..00000000 --- a/script/config/redis-server.key +++ /dev/null @@ -1,52 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQC0tBw8anVX3UsI -yp+NwiwEC+4v78cfzLzWS034Osw6Ay3eBuaeB//OCEZf5dBS5gY194fsGi22oeuH -haA/RG2pnopZxzkDjPKrBoN89xE08+dkQB5p1RhO4fh7DyaT4pbjqCW64O4Tgln7 -2BlAGX/LW11Z2UoRzN0uXKt8tFhpK2bAPQUq+n8jcgcyOjqF9GDLjWS/0VPjWjJn -XOZ1SiqxBNIKkSPiJOLgE8cCmPyNuCYsGGxrWrnn05Emjp9qZJXZmLcSP/EjeWM8 -o36JWvFFjufY2+paAh7rE5fEUC67sMx7nWv7NGnVjuaFCpNS+sw4Xrj8/YmX0ugw -2r0a4qVVZNa8xfL2J1vm63ERhpbKK+91EBDW9rWYH2DaQuVEPauvFX1GrSWUkXea -4OvdyHyhfiqurvwATGMV4kXXx5iTCuL+XuCp0uVnOaout4NH01PJczhjLN6hf1h+ -rsElRQ1CUyCk4XYoYnV8AjCinq0kBA6I6CdVQ4cpAfqPT+03vKwaFHd6luNQaz+Q -KrzWgMTc7/CmfEjJs2+CUboG2DoAPkgPioJocnIcK46pLLYu5nMX9H/qegWIIzap -IlA4a7RRFHASOmVxSXklEI0Ed89Q81BpC2fHMblvEBqIM276xg1OT8KelQxTSzYS -eTcBcxUXNwM5dFJkiaZV7PKytxjVvQIDAQABAoICAAZgPIbQZgEkDvVQDb7D7sDa -mU+zsPdcRmV0e46aQU58TwgAfyHTiNRNbAsO+5sqzP+zRjNJRGFA571jNG5HDQXp -HcFhMcMAT1rdfF8FnX1pOO1Laq2zr4t8MBCxGvYo22FMM4vOflgBFDyHBkBB8t8x -nOLPGezBQvFtCT5f/Q0ChqBnbS4N1jwuVjaqSBpZFXk/k4h5Hr6gFhszjSYiTpm3 -k9NJOVaA+q6+Kk10wte1SBOFbeRNa5w61Z173TNOoHYj2OgziiyeqECH4q6yBZIj -73/lWPWBq0ACjV5USr6j1FmgPYkiWYGhhly3YAKTh6Hz8nVqwYoYqC/hGyVxtDR2 -rCANHfXHUMqMVFXnZWbln3+hPlzosJqMbkEDrUN/ip4rAGjUKv1nM4bJFAbTyR1+ -VcXZlRu/B1mUqaY0xNhgw0JJYn3atimCJYCsfh2vhBZrtuzWSDB1i9vJK8JJ6ca5 -g2eaFU2sBprH4vFjoI5KmW3bO5wS2kGE7zj/Yu90lJZnM2sYw0NugqHFwnD5+y9O -tMW0rgL/v/Tkpd/3ZbLEfipwq7XS4YLxIsGksvYETQdE/djTJyaK837BfrqXY7V4 -NQUBvHO+z36dlWlZaRec98Td308nlxZG1yBiABlTZH5LmL3L66HR3N8oYpyu+eGl -R0C42xbNWvQTfvboz1/BAoIBAQDwEJ/hHZ/sno6AK9AljRRK1APWZH3ywIuZtvv+ -600kN0fi2L5D5TngdviATAwKErinYKc2XpkVBsVe71cu0DYVUsT1WV/2rcrGnEaq -TpMCJ4JwyWLkbOXcI0hs7dJwOGjVWWniEt2P6YloSuZfL1Z3TJaIQN3l0xETdIKl -uZ9hW3Ed79dFUlUpA4vHB2fJxWcTvBWgQcjre2GPAgxhC52Eg1maB1DIttDkwAG3 -CwmofdJqQ2bhq3YdS3LcI7cMuuVS0aqQ3rBvo5t2Ix5w+GEK3DNxx81NL21RPPcP -5XaLXQ+V/PGEGlBPi0Hm7t67sv6t4J33xPO7P1QGcOog2rElAoIBAQDAssUIICod -fnqGlXhogpIS4bxV7TSWO88TXm1Z/6LfYwg1abMUjt8IOcLFGd+rYN32RFzU4hJm -LeH7NWpSCO8A8KUgMr95uuLH6+UVzERsXZ2ay5R/VINAET1RJ0y2W4g43A2ICJVh -dijhzJ5WkaaV9IMEpdh4xKiPobCIdx9AhOTEuzHYdO3HCDgRlZmfYJAtJVTIqaVR -nLsQcFlbJJSzMyNFIh4QGVOTDTpzbogH0S8sw6RMUhbACTlaVdsp7y9dFiUFbeEg -VAdsterwj4c3Y+56HscLezsk9aeCg+Ds8pI7wsa4qf3ySpA4z7NNN/VKdr65SQ+w -USZ5A0Q0p+q5AoIBAQDbiniMCOmeHQq40FiUXsa1/9pakKBWBhwkwSEHaeysOBn3 -rLujn9gkHOj5e5JvC9vuxgdlQ3G/tud9AGi8mRE8wur4bXnDQAB3TaeaTC1JBRTX -THSx7ZzwixvK8ltG4W+N3S1l4FbbK3nL9ONlHQd4JoRrqtbDPRkar6xjHrRQ9q0l -8ZbvAHJ9BY8ovu79qRexBorIeiDiqXz4qajkfxjYfL7Oi+4nAHldGPL4kJGOkzKA -l49YxdNIyx0JkWfvrmPWL4RK5Dx6fCbKDSfjrGIi6mnpng10jvDsIFLZKMIWhU6h -nibI5r+XXwUvyjfLc9BQ91+/nVBAZ5PSKCUVUqo5AoIBABHPM+iZWGsbamWP4O3/ -y1JVbJugbbDI3E2U95ROzRXKSFLvbu6iebh/5iFjT8m+DSWakd3W+w5YhT87Y5Ur -YDN9CjXcmte8TwNdKPIHy/cb+9n3oMTX4b2FGl6qvCtKcF+Y7uM4r92vFv4vdB7a -HURUsds3HLekrVOBBRjHsaq73YzVJDk2LVesycnOQsF5IjoZ7ikRjMbrLlrJl2iW -C+w3+LWF/rNMM+4uVc2fVJEsYyjG+CNZnuKmt5dCwhuxEHaQ+mDzE9ByE7GXapxL -F5sl8D93IrxL6l5zGcU1PQ+JJ7zANq+QfnjrLUMi5xZm6pO0P9JOD1YjWePrSl13 -e3kCggEAECRWltLyR3+U6ufOvqj0Mgv0nKy9SoLzRY4XRK0uHesoBYXEmlOepsUi -23NkzjUFY3PsUflR4/N1IMbdyqR9dsvHUmvMJZ9SIp/fvnt68vOjlFQqHClRRHnc -HO6++q1e5Kj4XnzVnCRsy++kt974mVgl1EFivR42e1CPMvr92Ge/JcUFOArkuqUK -jN0yOUpH10wI5bQbziIcXJ8byPHT8cCKubH4rGEjppeVSur/elmxJzLcRaXVIwRK -YjS3wPGQZaQzkink9CQKaYsQaOVCkuUHPiJdpZwE9xZfLLAgKyw3DqvD/W8K0q9s -WJDHXBisqy4WGbW3D2CYLr1XXwO32A== ------END PRIVATE KEY----- diff --git a/test/test_helpers/certificates.rb b/test/test_helpers/certificates.rb new file mode 100644 index 00000000..6749e9b6 --- /dev/null +++ b/test/test_helpers/certificates.rb @@ -0,0 +1,67 @@ +# Copyright © 2020 Nicky Peeters +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the “Software”), to deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions +# of the Software. +# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +require 'rubygems' +require 'openssl' + +module TestHelpers + module Certificates + def create_key(alg) + case alg + when :rsa + OpenSSL::PKey::RSA.new(1024) + when :dsa + OpenSSL::PKey::DSA.new(1024) + when :ec + OpenSSL::PKey::EC.generate("prime256v1") + end + end + + def create_cert(key = create_key(:rsa)) + public_key = get_public_key(key) + + subject = "/C=BE/O=Test/OU=Test/CN=Test" + + cert = OpenSSL::X509::Certificate.new + cert.subject = cert.issuer = OpenSSL::X509::Name.parse(subject) + cert.not_before = Time.now + cert.not_after = Time.now + 365 * 24 * 60 * 60 + cert.public_key = public_key + cert.serial = 0x0 + cert.version = 2 + + ef = OpenSSL::X509::ExtensionFactory.new + ef.subject_certificate = cert + ef.issuer_certificate = cert + cert.extensions = [ + ef.create_extension("basicConstraints","CA:TRUE", true), + ef.create_extension("subjectKeyIdentifier", "hash"), + # ef.create_extension("keyUsage", "cRLSign,keyCertSign", true), + ] + cert.add_extension ef.create_extension("authorityKeyIdentifier", + "keyid:always,issuer:always") + + cert.sign key, OpenSSL::Digest::SHA1.new + cert + end + + private + + def get_public_key(key) + return OpenSSL::PKey::EC.new key if key.is_a? OpenSSL::PKey::EC + + key.public_key + end + end +end \ No newline at end of file diff --git a/test/unit/storage_async_test.rb b/test/unit/storage_async_test.rb index a13bbc2a..f4b707dd 100644 --- a/test/unit/storage_async_test.rb +++ b/test/unit/storage_async_test.rb @@ -1,7 +1,10 @@ +require 'fakefs/safe' require File.expand_path(File.dirname(__FILE__) + '/../test_helper') require '3scale/backend/storage_async' class StorageAsyncTest < Test::Unit::TestCase + include TestHelpers::Certificates + def test_basic_operations storage = StorageAsync::Client.instance(true) storage.del('foo') @@ -198,43 +201,31 @@ def test_tls_no_client_certificate assert_client_config(storage, **config_obj) end - def test_tls_client_cert_rsa - config_obj = { - url: 'rediss://localhost:46379/0', - ssl_params: { - ca_file: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'ca-root-cert.pem')), - cert: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'redis-client.crt')), - key: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'redis-client.key')) - } - } - storage = StorageAsync::Client.send :new, Storage::Helpers.config_with(config_obj) - assert_client_config(storage, **config_obj, test_cert_type: :rsa) - end - - def test_tls_client_cert_dsa - config_obj = { - url: 'rediss://localhost:46379/0', - ssl_params: { - ca_file: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'ca-root-cert.pem')), - cert: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'redis-dsa.crt')), - key: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'redis-dsa.pem')) - } - } - storage = StorageAsync::Client.send :new, Storage::Helpers.config_with(config_obj) - assert_client_config(storage, **config_obj, test_cert_type: :dsa) - end - - def test_tls_client_cert_ec - config_obj = { - url: 'rediss://localhost:46379/0', - ssl_params: { - ca_file: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'ca-root-cert.pem')), - cert: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'redis-ec.crt')), - key: File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'redis-ec.key')) - } - } - storage = StorageAsync::Client.send :new, Storage::Helpers.config_with(config_obj) - assert_client_config(storage, **config_obj, test_cert_type: :ec) + [:rsa, :dsa, :ec].each do |alg| + define_method "test_tls_client_cert_#{alg}" do + ca_file = create_cert + key = create_key alg + cert = create_cert key + FakeFS.with_fresh do + ca_file_path = File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'ca-root-cert.pem')) + key_path = File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'redis-dsa.pem')) + cert_path = File.expand_path(File.join(__FILE__, '..', '..', '..', 'script', 'config', 'redis-dsa.crt')) + FileUtils.mkdir_p(File.dirname(ca_file_path)) + File.write(ca_file_path, ca_file.to_pem) + File.write(key_path, key.to_pem) + File.write(cert_path, cert.to_pem) + config_obj = { + url: 'rediss://localhost:46379/0', + ssl_params: { + ca_file: ca_file_path, + cert: cert_path, + key: key_path + } + } + storage = StorageAsync::Client.send :new, Storage::Helpers.config_with(config_obj) + assert_client_config(storage, **config_obj, test_cert_type: alg) + end + end end def test_acl @@ -271,7 +262,7 @@ def assert_client_config(conn, **conf) assert_equal url.port, port unless conf[:username].to_s.strip.empty? && conf[:password].to_s.strip.empty? - assert_instance_of ThreeScale::Backend::StorageAsync::Client::AuthenticatedRESP2, client.protocol + assert_instance_of Async::Redis::Protocol::AuthenticatedRESP2, client.protocol username, password = client.protocol.instance_variable_get(:@credentials) assert_equal conf[:username], username assert_equal conf[:password], password From 821f08cabf8825dd7aa853cefa86d127157d1f21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Mon, 29 Apr 2024 13:45:45 +0200 Subject: [PATCH 37/54] Async: Add support for logical databases --- lib/3scale/backend/storage_async/client.rb | 3 ++- lib/async/redis/protocol/authenticated_resp2.rb | 8 +++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/3scale/backend/storage_async/client.rb b/lib/3scale/backend/storage_async/client.rb index 466e9d86..911d7184 100644 --- a/lib/3scale/backend/storage_async/client.rb +++ b/lib/3scale/backend/storage_async/client.rb @@ -89,9 +89,10 @@ def init_sentinels_client(opts) # Authenticated RESP2 if credentials are provided, RESP2 otherwise def make_redis_protocol(opts) uri = URI(opts[:url] || "") + db = uri.path[1..-1].to_i if uri.path credentials = [ uri.user || opts[:username], uri.password || opts[:password]] - Async::Redis::Protocol::AuthenticatedRESP2.new(credentials) + Async::Redis::Protocol::AuthenticatedRESP2.new(db: db, credentials: credentials) end # SSL endpoint if scheme is `rediss:`, TCP endpoint otherwise. diff --git a/lib/async/redis/protocol/authenticated_resp2.rb b/lib/async/redis/protocol/authenticated_resp2.rb index affab039..febbd690 100644 --- a/lib/async/redis/protocol/authenticated_resp2.rb +++ b/lib/async/redis/protocol/authenticated_resp2.rb @@ -9,7 +9,8 @@ module Protocol # Custom Redis Protocol class which sends the AUTH command on every new connection # to authenticate before sending any other command. class AuthenticatedRESP2 - def initialize(credentials) + def initialize(db: 0, credentials: []) + @db = db @credentials = credentials end @@ -21,6 +22,11 @@ def client(stream) client.read_response # Ignore response. end + if @db + client.write_request(["SELECT", @db]) + client.read_response + end + client end end From 0970789c09446cfb20d0edf5787790e75798cb1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Thu, 2 May 2024 17:06:35 +0200 Subject: [PATCH 38/54] Redis protocol: Don't force DB 0 It fails for sentinels --- lib/async/redis/protocol/authenticated_resp2.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/async/redis/protocol/authenticated_resp2.rb b/lib/async/redis/protocol/authenticated_resp2.rb index febbd690..26992a8b 100644 --- a/lib/async/redis/protocol/authenticated_resp2.rb +++ b/lib/async/redis/protocol/authenticated_resp2.rb @@ -9,7 +9,7 @@ module Protocol # Custom Redis Protocol class which sends the AUTH command on every new connection # to authenticate before sending any other command. class AuthenticatedRESP2 - def initialize(db: 0, credentials: []) + def initialize(db: nil, credentials: []) @db = db @credentials = credentials end From 2bc318cb6cbc15024c5c3d09388755466977983e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Thu, 2 May 2024 17:07:40 +0200 Subject: [PATCH 39/54] Path `async-redis` `SentinelsClient` class To include support for sentinels with ACL and TLS --- lib/3scale/backend/storage_async/client.rb | 7 +- lib/async/redis/sentinels_client_acl_tls.rb | 84 +++++++++++++++++++++ 2 files changed, 88 insertions(+), 3 deletions(-) create mode 100644 lib/async/redis/sentinels_client_acl_tls.rb diff --git a/lib/3scale/backend/storage_async/client.rb b/lib/3scale/backend/storage_async/client.rb index 911d7184..16af7191 100644 --- a/lib/3scale/backend/storage_async/client.rb +++ b/lib/3scale/backend/storage_async/client.rb @@ -1,6 +1,6 @@ require 'async/io' require 'async/redis/client' -require 'async/redis/sentinels' +require 'async/redis/sentinels_client_acl_tls' require 'async/redis/protocol/authenticated_resp2' module ThreeScale @@ -82,14 +82,15 @@ def init_sentinels_client(opts) uri = URI(opts[:url] || '') name = uri.host role = opts[:role] || :master + protocol = make_redis_protocol(opts) - Async::Redis::SentinelsClient.new(name, opts[:sentinels], role) + Async::Redis::SentinelsClientACLTLS.new(name, opts[:sentinels], role, protocol, opts) end # Authenticated RESP2 if credentials are provided, RESP2 otherwise def make_redis_protocol(opts) uri = URI(opts[:url] || "") - db = uri.path[1..-1].to_i if uri.path + db = uri.path[1..-1] credentials = [ uri.user || opts[:username], uri.password || opts[:password]] Async::Redis::Protocol::AuthenticatedRESP2.new(db: db, credentials: credentials) diff --git a/lib/async/redis/sentinels_client_acl_tls.rb b/lib/async/redis/sentinels_client_acl_tls.rb new file mode 100644 index 00000000..f5f67ae7 --- /dev/null +++ b/lib/async/redis/sentinels_client_acl_tls.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +require 'openssl' +require 'async/redis/sentinels' + +module Async + module Redis + class SentinelsClientACLTLS < SentinelsClient + def initialize(master_name, sentinels, role = :master, protocol = Protocol::RESP2, config = {}, **options) + @master_name = master_name + @sentinel_endpoints = sentinels.map do |sentinel| + make_endpoint(sentinel[:host], sentinel[:port], config[:ssl], config[:ssl_params]) + end + @role = role + + @protocol = protocol + @config = config + @pool = connect(**options) + end + + private + + def resolve_master + @sentinel_endpoints.each do |sentinel_endpoint| + client = Client.new(sentinel_endpoint, protocol: @protocol) + + begin + address = client.call('sentinel', 'get-master-addr-by-name', @master_name) + rescue Errno::ECONNREFUSED + next + end + + return make_endpoint(address[0], address[1], @config[:ssl], @config[:ssl_params]) if address + end + + nil + end + + def resolve_slave + @sentinel_endpoints.each do |sentinel_endpoint| + client = Client.new(sentinel_endpoint, protocol: @protocol) + + begin + reply = client.call('sentinel', 'slaves', @master_name) + rescue Errno::ECONNREFUSED + next + end + + slaves = available_slaves(reply) + next if slaves.empty? + + slave = select_slave(slaves) + return make_endpoint(slave['ip'], slave['port'], @config[:ssl], @config[:ssl_params]) + end + + nil + end + + def make_endpoint(host, port, ssl, ssl_params) + tcp_endpoint = Async::IO::Endpoint.tcp(host, port) + + if ssl + ssl_context = OpenSSL::SSL::SSLContext.new + ssl_context.set_params(format_ssl_params(ssl_params)) + return Async::IO::SSLEndpoint.new(tcp_endpoint, ssl_context: ssl_context) + end + + tcp_endpoint + end + + def format_ssl_params(ssl_params) + cert = ssl_params[:cert].to_s.strip + key = ssl_params[:key].to_s.strip + return ssl_params if cert.empty? && key.empty? + + updated_ssl_params = ssl_params.dup + updated_ssl_params[:cert] = OpenSSL::X509::Certificate.new(File.read(cert)) + updated_ssl_params[:key] = OpenSSL::PKey.read(File.read(key)) + + updated_ssl_params + end + end + end +end From e64a1510599c5297c6dbd9e1b3335c0c42170be1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Thu, 2 May 2024 17:07:59 +0200 Subject: [PATCH 40/54] Add new Env variables For sentinels with ACL and TLS --- docs/configuration.md | 28 +++++++++++++++++++++++++++ lib/3scale/backend/configuration.rb | 10 ++++++---- lib/3scale/backend/storage_helpers.rb | 3 ++- openshift/3scale_backend.conf | 4 ++++ 4 files changed, 40 insertions(+), 5 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 3c8ffbf3..5917fb06 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -82,6 +82,20 @@ variables. - Applies to: listener, worker, cron. - Format: list of URLs separated by ",". +### CONFIG_REDIS_SENTINEL_USERNAME + +- Sentinels ACL user name +- Optional. Defaults to empty. +- Applies to: listener, worker, cron. +- Format: string. + +### CONFIG_REDIS_SENTINEL_PASSWORD + +- Sentinels ACL password +- Optional. Defaults to empty. +- Applies to: listener, worker, cron. +- Format: string. + ### CONFIG_REDIS_SENTINEL_ROLE - Asks the sentinel for the URL of the master or a slave. @@ -185,6 +199,20 @@ sentinels. - Applies to: listener, worker, cron. - Format: list of URLs separated by ",". +### CONFIG_QUEUES_SENTINEL_USERNAME + +- Sentinels ACL user name +- Optional. Defaults to empty. +- Applies to: listener, worker, cron. +- Format: string. + +### CONFIG_QUEUES_SENTINEL_PASSWORD + +- Sentinels ACL password +- Optional. Defaults to empty. +- Applies to: listener, worker, cron. +- Format: string. + ### CONFIG_QUEUES_SENTINEL_ROLE - Asks the sentinel for the URL of the master or a slave. diff --git a/lib/3scale/backend/configuration.rb b/lib/3scale/backend/configuration.rb index f28bc8ca..3ef4ef39 100644 --- a/lib/3scale/backend/configuration.rb +++ b/lib/3scale/backend/configuration.rb @@ -39,10 +39,12 @@ def parse_int(value, default) config.workers_logger_formatter = :text # Add configuration sections - config.add_section(:queues, :master_name, :username, :password, :ssl, :ssl_params, :sentinels, :role, - :connect_timeout, :read_timeout, :write_timeout, :max_connections) - config.add_section(:redis, :url, :proxy, :username, :password, :ssl, :ssl_params, :sentinels, :role, - :connect_timeout, :read_timeout, :write_timeout, :max_connections, :async) + config.add_section(:queues, :master_name, :username, :password, :ssl, :ssl_params, :sentinels, + :sentinel_username, :sentinel_password, :role, :connect_timeout, :read_timeout, :write_timeout, + :max_connections) + config.add_section(:redis, :url, :proxy, :username, :password, :ssl, :ssl_params, :sentinels, + :sentinel_username, :sentinel_password, :role, :connect_timeout, :read_timeout, :write_timeout, + :max_connections, :async) config.add_section(:hoptoad, :service, :api_key) config.add_section(:internal_api, :user, :password) config.add_section(:master, :metrics) diff --git a/lib/3scale/backend/storage_helpers.rb b/lib/3scale/backend/storage_helpers.rb index 07038ef3..2bf6a797 100644 --- a/lib/3scale/backend/storage_helpers.rb +++ b/lib/3scale/backend/storage_helpers.rb @@ -62,7 +62,8 @@ class << self # CONN_WHITELIST - Connection options that can be specified in config # Note: we don't expose reconnect_attempts until the bug above is fixed CONN_WHITELIST = [ - :connect_timeout, :read_timeout, :write_timeout, :max_connections, :username, :password, :ssl, :ssl_params + :connect_timeout, :read_timeout, :write_timeout, :max_connections, :username, :password, :sentinel_username, + :sentinel_password, :ssl, :ssl_params ].freeze private_constant :CONN_WHITELIST diff --git a/openshift/3scale_backend.conf b/openshift/3scale_backend.conf index 2d51020d..4f89c028 100644 --- a/openshift/3scale_backend.conf +++ b/openshift/3scale_backend.conf @@ -32,6 +32,8 @@ ThreeScale::Backend.configure do |config| key: "#{ENV['CONFIG_QUEUES_PRIVATE_KEY']}" } config.queues.sentinels = "#{ENV['CONFIG_QUEUES_SENTINEL_HOSTS'] && !ENV['CONFIG_QUEUES_SENTINEL_HOSTS'].empty? ? ENV['CONFIG_QUEUES_SENTINEL_HOSTS'] : ENV['SENTINEL_HOSTS']}" + config.queues.sentinel_username = "#{ENV['CONFIG_QUEUES_SENTINEL_USERNAME']}" + config.queues.sentinel_password = "#{ENV['CONFIG_QUEUES_SENTINEL_PASSWORD']}" config.queues.role = "#{ENV['CONFIG_QUEUES_SENTINEL_ROLE']}".to_sym config.queues.connect_timeout = parse_int_env('CONFIG_QUEUES_CONNECT_TIMEOUT') config.queues.read_timeout = parse_int_env('CONFIG_QUEUES_READ_TIMEOUT') @@ -48,6 +50,8 @@ ThreeScale::Backend.configure do |config| key: "#{ENV['CONFIG_REDIS_PRIVATE_KEY']}" } config.redis.sentinels = "#{ENV['CONFIG_REDIS_SENTINEL_HOSTS']}" + config.redis.sentinel_username = "#{ENV['CONFIG_REDIS_SENTINEL_USERNAME']}" + config.redis.sentinel_password = "#{ENV['CONFIG_REDIS_SENTINEL_PASSWORD']}" config.redis.role = "#{ENV['CONFIG_REDIS_SENTINEL_ROLE']}".to_sym config.redis.connect_timeout = parse_int_env('CONFIG_REDIS_CONNECT_TIMEOUT') config.redis.read_timeout = parse_int_env('CONFIG_REDIS_READ_TIMEOUT') From 64d0f228346d6df7aa75520d6a57cbc216d29117 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Thu, 2 May 2024 17:17:57 +0200 Subject: [PATCH 41/54] Fix tests --- spec/unit/queue_storage_spec.rb | 2 +- test/unit/storage_async_test.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/unit/queue_storage_spec.rb b/spec/unit/queue_storage_spec.rb index 7d04b412..21b9ce75 100644 --- a/spec/unit/queue_storage_spec.rb +++ b/spec/unit/queue_storage_spec.rb @@ -45,7 +45,7 @@ module Backend def is_sentinel?(connection) if ThreeScale::Backend.configuration.redis.async connector = connection.instance_variable_get(:@redis_async) - connector.instance_of?(Async::Redis::SentinelsClient) + connector.instance_of?(Async::Redis::SentinelsClientACLTLS) else connector = connection.instance_variable_get(:@client) .instance_variable_get(:@config) diff --git a/test/unit/storage_async_test.rb b/test/unit/storage_async_test.rb index f4b707dd..c5ce467a 100644 --- a/test/unit/storage_async_test.rb +++ b/test/unit/storage_async_test.rb @@ -293,7 +293,7 @@ def assert_sentinel_config(conn, url:, **conf) role = conf[:role] || :master password = client.instance_variable_get(:@protocol).instance_variable_get(:@password) - assert_instance_of Async::Redis::SentinelsClient, client + assert_instance_of Async::Redis::SentinelsClientACLTLS, client assert_equal name, client.instance_variable_get(:@master_name) assert_equal role, client.instance_variable_get(:@role) From da556bfd0a6a379664c52ef2c87d8e9253040c5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Fri, 3 May 2024 09:25:41 +0200 Subject: [PATCH 42/54] Generate strongest keys for tests A test fails in CircleCI due to this --- test/test_helpers/certificates.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_helpers/certificates.rb b/test/test_helpers/certificates.rb index 6749e9b6..513888fd 100644 --- a/test/test_helpers/certificates.rb +++ b/test/test_helpers/certificates.rb @@ -20,9 +20,9 @@ module Certificates def create_key(alg) case alg when :rsa - OpenSSL::PKey::RSA.new(1024) + OpenSSL::PKey::RSA.new(2048) when :dsa - OpenSSL::PKey::DSA.new(1024) + OpenSSL::PKey::DSA.new(2048) when :ec OpenSSL::PKey::EC.generate("prime256v1") end From f9bb6fab8f7728bc0c201eb64602540b9db2ea2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Fri, 3 May 2024 14:20:24 +0200 Subject: [PATCH 43/54] Integrate with the operator --- docs/configuration.md | 14 -------------- lib/3scale/backend/storage_helpers.rb | 15 --------------- openshift/3scale_backend.conf | 2 -- test/unit/storage_async_test.rb | 4 +--- test/unit/storage_sync_test.rb | 2 +- 5 files changed, 2 insertions(+), 35 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 5917fb06..d75e50ac 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -54,13 +54,6 @@ variables. - Applies to: listener, worker, cron. - Format: path to file as string. -### CONFIG_REDIS_CA_PATH - -- System certificates location path, `/etc/ssl/certs` for RHEL -- Optional. Defaults to empty. -- Applies to: listener, worker, cron. -- Format: path to file as string. - ### CONFIG_REDIS_CERT - The path to the client SSL certificate @@ -171,13 +164,6 @@ sentinels. - Applies to: listener, worker, cron. - Format: path to file as string. -### CONFIG_QUEUES_CA_PATH - -- System certificates location path, `/etc/ssl/certs` for RHEL -- Optional. Defaults to empty. -- Applies to: listener, worker, cron. -- Format: path to file as string. - ### CONFIG_QUEUES_CERT - User certificate to connect to Redis through TLS diff --git a/lib/3scale/backend/storage_helpers.rb b/lib/3scale/backend/storage_helpers.rb index 2bf6a797..0507fdef 100644 --- a/lib/3scale/backend/storage_helpers.rb +++ b/lib/3scale/backend/storage_helpers.rb @@ -267,20 +267,6 @@ def cfg_unix_path_handler(options) options end - # If no CA cert given, look for the default one at `config/ca_cert.pem` - def cfg_ca_cert_handler(options) - return options if options[:ssl_params]&.key?(:ca_file) || options[:ssl_params]&.key?(:ca_path) - - cert_path = "#{Backend::Util.root_dir}/config/ca_cert.pem" - - return options unless File.exist?(cert_path) - - options[:ssl_params] ||= {} - options[:ssl_params][:ca_file] = cert_path - - options - end - # This ensures some default values are valid for the redis client. # In particular: # @@ -290,7 +276,6 @@ def cfg_ca_cert_handler(options) def cfg_defaults_handler(options, defaults) cfg_with_defaults = defaults.merge(ensure_url_param(options)) cfg_with_defaults = cfg_unix_path_handler(cfg_with_defaults) - cfg_with_defaults = cfg_ca_cert_handler(cfg_with_defaults) cfg_with_defaults.delete(:max_connections) unless options[:async] cfg_with_defaults end diff --git a/openshift/3scale_backend.conf b/openshift/3scale_backend.conf index 4f89c028..590b0e78 100644 --- a/openshift/3scale_backend.conf +++ b/openshift/3scale_backend.conf @@ -27,7 +27,6 @@ ThreeScale::Backend.configure do |config| config.queues.ssl = parse_boolean_env('CONFIG_QUEUES_SSL') config.queues.ssl_params = { ca_file: "#{ENV['CONFIG_QUEUES_CA_FILE']}", - ca_path: "#{ENV['CONFIG_QUEUES_CA_PATH']}", cert: "#{ENV['CONFIG_QUEUES_CERT']}", key: "#{ENV['CONFIG_QUEUES_PRIVATE_KEY']}" } @@ -45,7 +44,6 @@ ThreeScale::Backend.configure do |config| config.redis.ssl = parse_boolean_env('CONFIG_REDIS_SSL') config.redis.ssl_params = { ca_file: "#{ENV['CONFIG_REDIS_CA_FILE']}", - ca_path: "#{ENV['CONFIG_REDIS_CA_PATH']}", cert: "#{ENV['CONFIG_REDIS_CERT']}", key: "#{ENV['CONFIG_REDIS_PRIVATE_KEY']}" } diff --git a/test/unit/storage_async_test.rb b/test/unit/storage_async_test.rb index c5ce467a..76d73085 100644 --- a/test/unit/storage_async_test.rb +++ b/test/unit/storage_async_test.rb @@ -270,9 +270,7 @@ def assert_client_config(conn, **conf) unless conf[:ssl_params].to_s.strip.empty? assert_instance_of Async::IO::SSLEndpoint, client.endpoint - %i[ca_file ca_path].each do |key| - assert_equal conf[:ssl_params][key], client.endpoint.options[:ssl_context].send(key) - end + assert_equal conf[:ssl_params][:ca_file], client.endpoint.options[:ssl_context].send(:ca_file) assert_instance_of(OpenSSL::X509::Certificate, client.endpoint.options[:ssl_context].cert) unless conf[:ssl_params][:cert].to_s.strip.empty? unless conf[:test_cert_type].to_s.strip.empty? diff --git a/test/unit/storage_sync_test.rb b/test/unit/storage_sync_test.rb index 0040b726..1fd79815 100644 --- a/test/unit/storage_sync_test.rb +++ b/test/unit/storage_sync_test.rb @@ -293,7 +293,7 @@ def assert_client_config(conn, **conf) assert_equal conf[:password], config.password unless conf[:ssl_params].to_s.strip.empty? - %i[ca_file ca_path cert key].each do |key| + %i[ca_file cert key].each do |key| assert_equal conf[:ssl_params][key], config.ssl_params[key] end end From 40d99327dc09c093955442bfe696bf529ca9b76a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Mon, 6 May 2024 09:58:13 +0200 Subject: [PATCH 44/54] Fix typo --- lib/3scale/backend.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/3scale/backend.rb b/lib/3scale/backend.rb index 4df1ad1c..b989b66c 100644 --- a/lib/3scale/backend.rb +++ b/lib/3scale/backend.rb @@ -57,18 +57,18 @@ module ThreeScale module Backend class << self - def new_rescue_redis + def new_resque_redis QueueStorage.connection( environment, configuration, ) end - def set_rescue_redis - ::Resque.redis = new_rescue_redis + def set_resque_redis + ::Resque.redis = new_resque_redis end end end end -ThreeScale::Backend.set_rescue_redis +ThreeScale::Backend.set_resque_redis From 6c45bcaf2cfa6df768c81c42422424e666de004a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Mon, 6 May 2024 12:49:30 +0200 Subject: [PATCH 45/54] Don't crash the fetcher on timeouts They are expected. --- lib/3scale/backend/job_fetcher.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/3scale/backend/job_fetcher.rb b/lib/3scale/backend/job_fetcher.rb index 57d1ee1a..b742726d 100644 --- a/lib/3scale/backend/job_fetcher.rb +++ b/lib/3scale/backend/job_fetcher.rb @@ -85,6 +85,8 @@ def try_pop_from_queue(max=nil) def wait_pop_from_queue queue, encoded = @redis.blpop(*@queues, timeout: @fetch_timeout) [queue, [encoded]] if encoded + rescue RedisClient::ReadTimeoutError => _e + # Do nothing, we expect frequent timeouts from the sync client when the queue is empty end def decode_job(queue, encoded_job) From 34dfc4fcd8198038b8d02ef7d2d7ee4084c73c6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Mon, 6 May 2024 13:29:30 +0200 Subject: [PATCH 46/54] Update comments --- lib/3scale/backend/storage_helpers.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/3scale/backend/storage_helpers.rb b/lib/3scale/backend/storage_helpers.rb index 0507fdef..cf44abc6 100644 --- a/lib/3scale/backend/storage_helpers.rb +++ b/lib/3scale/backend/storage_helpers.rb @@ -45,13 +45,13 @@ class << self read_timeout: 3, write_timeout: 3, # Note that we can set reconnect_attempts to >= 0 because we use - # our redis-rb fork which implements a workaround for this issue - # that shows that when there might be duplicated transactions when + # a monkey patch which implements a workaround for this issue + # that shows that there might be duplicated transactions when # there's a timeout: https://github.com/redis/redis-rb/issues/668 # We should investigate if there are edge cases that can lead to # duplicated commands because of this setting. reconnect_attempts: 1, - # use by default the C extension client + # applies only to async mode. Use the C extension client by default driver: :hiredis, # applies only to async mode. The sync library opens 1 connection # per process. From 6bbf375d3e285b6f1ef419f96f4da2d84e2c047a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Wed, 8 May 2024 12:23:02 +0200 Subject: [PATCH 47/54] Fix comment --- lib/3scale/backend/storage_async/methods.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/3scale/backend/storage_async/methods.rb b/lib/3scale/backend/storage_async/methods.rb index 0a9b3399..65523d84 100644 --- a/lib/3scale/backend/storage_async/methods.rb +++ b/lib/3scale/backend/storage_async/methods.rb @@ -9,7 +9,7 @@ module Methods # # These are the different cases: # 1) Methods that can be called directly. For example SET: - # @redis_async.call('SET', some_key) + # call('SET', some_key) # 2) Methods that need to be "boolified". These are methods for which # redis-rb returns a boolean, but redis just returns an integer. # For example, Redis returns 0 or 1 for the EXISTS command, but From 7093a8cccf89d60e208dbaa5cdcb7170315bc8fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Wed, 8 May 2024 12:37:08 +0200 Subject: [PATCH 48/54] Add empty line at EOF --- test/test_helpers/certificates.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_helpers/certificates.rb b/test/test_helpers/certificates.rb index 513888fd..3f0ac6b5 100644 --- a/test/test_helpers/certificates.rb +++ b/test/test_helpers/certificates.rb @@ -64,4 +64,4 @@ def get_public_key(key) key.public_key end end -end \ No newline at end of file +end From 4b6e8cf534503de5a82a1a9c4f98b7bd326d844b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Mon, 20 May 2024 13:49:47 +0200 Subject: [PATCH 49/54] Small fixes after merging from master --- lib/3scale/backend/service.rb | 2 +- test/unit/storage_async_test.rb | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/3scale/backend/service.rb b/lib/3scale/backend/service.rb index 1a5877ad..177622a1 100644 --- a/lib/3scale/backend/service.rb +++ b/lib/3scale/backend/service.rb @@ -277,7 +277,7 @@ def persist_attribute(attribute, value, ignore_nils = false) def persist_sets storage.sadd storage_key_by_provider(:ids), id storage.sadd encode_key("services_set"), id - storage.sadd encode_key("provider_keys_set"), provider_key unless provider_key.nil? + storage.sadd encode_key("provider_keys_set"), provider_key end end diff --git a/test/unit/storage_async_test.rb b/test/unit/storage_async_test.rb index f063b785..c5003985 100644 --- a/test/unit/storage_async_test.rb +++ b/test/unit/storage_async_test.rb @@ -37,6 +37,17 @@ def test_redis_malformed_url end end + def test_redis_no_scheme + storage = StorageAsync::Client.send :new, url('backend-redis') + assert_client_config({ url: URI('redis://backend-redis:6379') }, storage) + end + + def test_redis_unknown_scheme + assert_raise ArgumentError do + StorageAsync::Client.send :new, url('myscheme://127.0.0.1:6379') + end + end + def test_sentinels_connection_string config_obj = { url: 'redis://master-group-name', @@ -185,17 +196,6 @@ def test_sentinels_empty end end - def test_redis_no_scheme - storage = StorageAsync::Client.send :new, url('backend-redis') - assert_client_config({ url: URI('redis://backend-redis:6379') }, storage) - end - - def test_redis_unknown_scheme - assert_raise ArgumentError do - StorageAsync::Client.send :new, url('myscheme://127.0.0.1:6379') - end - end - def test_tls_no_client_certificate config_obj = { url: 'rediss://localhost:46379/0', From f0d0c535224c5489954a1335ebccf9274ab43026 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Thu, 23 May 2024 09:57:30 +0200 Subject: [PATCH 50/54] Adapt to last changes to master --- .../async_redis/protocol/extended_resp2.rb | 12 ++- .../async_redis/sentinels_client_acl_tls.rb | 86 +++++++++++++++++++ lib/3scale/backend/storage_async/client.rb | 7 +- .../redis/protocol/authenticated_resp2.rb | 35 -------- lib/async/redis/sentinels_client_acl_tls.rb | 84 ------------------ spec/unit/queue_storage_spec.rb | 2 +- test/unit/storage_async_test.rb | 4 +- 7 files changed, 104 insertions(+), 126 deletions(-) create mode 100644 lib/3scale/backend/async_redis/sentinels_client_acl_tls.rb delete mode 100644 lib/async/redis/protocol/authenticated_resp2.rb delete mode 100644 lib/async/redis/sentinels_client_acl_tls.rb diff --git a/lib/3scale/backend/async_redis/protocol/extended_resp2.rb b/lib/3scale/backend/async_redis/protocol/extended_resp2.rb index 2e8a8031..631629db 100644 --- a/lib/3scale/backend/async_redis/protocol/extended_resp2.rb +++ b/lib/3scale/backend/async_redis/protocol/extended_resp2.rb @@ -8,14 +8,24 @@ module AsyncRedis module Protocol # Custom Redis Protocol supporting Redis logical DBs + # and ACL credentials class ExtendedRESP2 - def initialize(db: nil) + + attr_reader :credentials, :db + + def initialize(db: nil, credentials: []) @db = db + @credentials = credentials end def client(stream) client = Async::Redis::Protocol::RESP2.client(stream) + if @credentials.any? + client.write_request(["AUTH", *@credentials]) + client.read_response # Ignore response. + end + if @db client.write_request(["SELECT", @db]) client.read_response diff --git a/lib/3scale/backend/async_redis/sentinels_client_acl_tls.rb b/lib/3scale/backend/async_redis/sentinels_client_acl_tls.rb new file mode 100644 index 00000000..861e38e6 --- /dev/null +++ b/lib/3scale/backend/async_redis/sentinels_client_acl_tls.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +require 'openssl' +require 'async/redis/sentinels' + +module ThreeScale + module Backend + module AsyncRedis + class SentinelsClientACLTLS < Async::Redis::SentinelsClient + def initialize(master_name, sentinels, role = :master, protocol = Protocol::RESP2, config = {}, **options) + @master_name = master_name + @sentinel_endpoints = sentinels.map do |sentinel| + make_endpoint(sentinel[:host], sentinel[:port], config[:ssl], config[:ssl_params]) + end + @role = role + + @protocol = protocol + @config = config + @pool = connect(**options) + end + + private + + def resolve_master + @sentinel_endpoints.each do |sentinel_endpoint| + client = Async::Redis::Client.new(sentinel_endpoint, protocol: ThreeScale::Backend::AsyncRedis::Protocol::ExtendedRESP2.new(credentials: @protocol.credentials)) + + begin + address = client.call('sentinel', 'get-master-addr-by-name', @master_name) + rescue Errno::ECONNREFUSED + next + end + + return make_endpoint(address[0], address[1], @config[:ssl], @config[:ssl_params]) if address + end + + nil + end + + def resolve_slave + @sentinel_endpoints.each do |sentinel_endpoint| + client = Async::Redis::Client.new(sentinel_endpoint, protocol: ThreeScale::Backend::AsyncRedis::Protocol::ExtendedRESP2.new(credentials: @protocol.credentials)) + + begin + reply = client.call('sentinel', 'slaves', @master_name) + rescue Errno::ECONNREFUSED + next + end + + slaves = available_slaves(reply) + next if slaves.empty? + + slave = select_slave(slaves) + return make_endpoint(slave['ip'], slave['port'], @config[:ssl], @config[:ssl_params]) + end + + nil + end + + def make_endpoint(host, port, ssl, ssl_params) + tcp_endpoint = Async::IO::Endpoint.tcp(host, port) + + if ssl + ssl_context = OpenSSL::SSL::SSLContext.new + ssl_context.set_params(format_ssl_params(ssl_params)) + return Async::IO::SSLEndpoint.new(tcp_endpoint, ssl_context: ssl_context) + end + + tcp_endpoint + end + + def format_ssl_params(ssl_params) + cert = ssl_params[:cert].to_s.strip + key = ssl_params[:key].to_s.strip + return ssl_params if cert.empty? && key.empty? + + updated_ssl_params = ssl_params.dup + updated_ssl_params[:cert] = OpenSSL::X509::Certificate.new(File.read(cert)) + updated_ssl_params[:key] = OpenSSL::PKey.read(File.read(key)) + + updated_ssl_params + end + end + end + end +end diff --git a/lib/3scale/backend/storage_async/client.rb b/lib/3scale/backend/storage_async/client.rb index 54808c9d..7a3d6408 100644 --- a/lib/3scale/backend/storage_async/client.rb +++ b/lib/3scale/backend/storage_async/client.rb @@ -1,6 +1,6 @@ require 'async/io' require 'async/redis/client' -require 'async/redis/sentinels_client_acl_tls' +require '3scale/backend/async_redis/sentinels_client_acl_tls' require '3scale/backend/async_redis/protocol/extended_resp2' module ThreeScale @@ -84,15 +84,16 @@ def init_sentinels_client(opts) role = opts[:role] || :master protocol = make_redis_protocol(opts) - Async::Redis::SentinelsClientACLTLS.new(name, opts[:sentinels], role, protocol) + ThreeScale::Backend::AsyncRedis::SentinelsClientACLTLS.new(name, opts[:sentinels], role, protocol, opts) end # RESP2 with support for logical DBs def make_redis_protocol(opts) uri = URI(opts[:url] || "") db = uri.path[1..-1] + credentials = [ uri.user || opts[:username], uri.password || opts[:password]] - ThreeScale::Backend::AsyncRedis::Protocol::ExtendedRESP2.new(db: db) + ThreeScale::Backend::AsyncRedis::Protocol::ExtendedRESP2.new(db: db, credentials: credentials) end # SSL endpoint if scheme is `rediss:`, TCP endpoint otherwise. diff --git a/lib/async/redis/protocol/authenticated_resp2.rb b/lib/async/redis/protocol/authenticated_resp2.rb deleted file mode 100644 index 26992a8b..00000000 --- a/lib/async/redis/protocol/authenticated_resp2.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -require 'async/redis/protocol/resp2' - -module Async - module Redis - module Protocol - - # Custom Redis Protocol class which sends the AUTH command on every new connection - # to authenticate before sending any other command. - class AuthenticatedRESP2 - def initialize(db: nil, credentials: []) - @db = db - @credentials = credentials - end - - def client(stream) - client = Async::Redis::Protocol::RESP2.client(stream) - - if @credentials.any? - client.write_request(["AUTH", *@credentials]) - client.read_response # Ignore response. - end - - if @db - client.write_request(["SELECT", @db]) - client.read_response - end - - client - end - end - end - end -end diff --git a/lib/async/redis/sentinels_client_acl_tls.rb b/lib/async/redis/sentinels_client_acl_tls.rb deleted file mode 100644 index f5f67ae7..00000000 --- a/lib/async/redis/sentinels_client_acl_tls.rb +++ /dev/null @@ -1,84 +0,0 @@ -# frozen_string_literal: true - -require 'openssl' -require 'async/redis/sentinels' - -module Async - module Redis - class SentinelsClientACLTLS < SentinelsClient - def initialize(master_name, sentinels, role = :master, protocol = Protocol::RESP2, config = {}, **options) - @master_name = master_name - @sentinel_endpoints = sentinels.map do |sentinel| - make_endpoint(sentinel[:host], sentinel[:port], config[:ssl], config[:ssl_params]) - end - @role = role - - @protocol = protocol - @config = config - @pool = connect(**options) - end - - private - - def resolve_master - @sentinel_endpoints.each do |sentinel_endpoint| - client = Client.new(sentinel_endpoint, protocol: @protocol) - - begin - address = client.call('sentinel', 'get-master-addr-by-name', @master_name) - rescue Errno::ECONNREFUSED - next - end - - return make_endpoint(address[0], address[1], @config[:ssl], @config[:ssl_params]) if address - end - - nil - end - - def resolve_slave - @sentinel_endpoints.each do |sentinel_endpoint| - client = Client.new(sentinel_endpoint, protocol: @protocol) - - begin - reply = client.call('sentinel', 'slaves', @master_name) - rescue Errno::ECONNREFUSED - next - end - - slaves = available_slaves(reply) - next if slaves.empty? - - slave = select_slave(slaves) - return make_endpoint(slave['ip'], slave['port'], @config[:ssl], @config[:ssl_params]) - end - - nil - end - - def make_endpoint(host, port, ssl, ssl_params) - tcp_endpoint = Async::IO::Endpoint.tcp(host, port) - - if ssl - ssl_context = OpenSSL::SSL::SSLContext.new - ssl_context.set_params(format_ssl_params(ssl_params)) - return Async::IO::SSLEndpoint.new(tcp_endpoint, ssl_context: ssl_context) - end - - tcp_endpoint - end - - def format_ssl_params(ssl_params) - cert = ssl_params[:cert].to_s.strip - key = ssl_params[:key].to_s.strip - return ssl_params if cert.empty? && key.empty? - - updated_ssl_params = ssl_params.dup - updated_ssl_params[:cert] = OpenSSL::X509::Certificate.new(File.read(cert)) - updated_ssl_params[:key] = OpenSSL::PKey.read(File.read(key)) - - updated_ssl_params - end - end - end -end diff --git a/spec/unit/queue_storage_spec.rb b/spec/unit/queue_storage_spec.rb index 21b9ce75..579ec718 100644 --- a/spec/unit/queue_storage_spec.rb +++ b/spec/unit/queue_storage_spec.rb @@ -45,7 +45,7 @@ module Backend def is_sentinel?(connection) if ThreeScale::Backend.configuration.redis.async connector = connection.instance_variable_get(:@redis_async) - connector.instance_of?(Async::Redis::SentinelsClientACLTLS) + connector.instance_of?(ThreeScale::Backend::AsyncRedis::SentinelsClientACLTLS) else connector = connection.instance_variable_get(:@client) .instance_variable_get(:@config) diff --git a/test/unit/storage_async_test.rb b/test/unit/storage_async_test.rb index c5003985..fc10b3f2 100644 --- a/test/unit/storage_async_test.rb +++ b/test/unit/storage_async_test.rb @@ -268,7 +268,7 @@ def assert_client_config(conf, conn, test_cert_type = nil) assert_equal url.port, port unless conf[:username].to_s.strip.empty? && conf[:password].to_s.strip.empty? - assert_instance_of Async::Redis::Protocol::AuthenticatedRESP2, client.protocol + assert_instance_of ThreeScale::Backend::AsyncRedis::Protocol::ExtendedRESP2, client.protocol username, password = client.protocol.instance_variable_get(:@credentials) assert_equal conf[:username], username assert_equal conf[:password], password @@ -297,7 +297,7 @@ def assert_sentinel_config(conf, conn) role = conf[:role] || :master password = client.instance_variable_get(:@protocol).instance_variable_get(:@password) - assert_instance_of Async::Redis::SentinelsClientACLTLS, client + assert_instance_of ThreeScale::Backend::AsyncRedis::SentinelsClientACLTLS, client assert_equal name, client.instance_variable_get(:@master_name) assert_equal role, client.instance_variable_get(:@role) From 1cc4c2c05c21b153371482ddeb54415443c74863 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Thu, 23 May 2024 12:46:08 +0200 Subject: [PATCH 51/54] Refactor Async client --- lib/3scale/backend/async_redis/client.rb | 38 +++++++++ .../backend/async_redis/endpoint_helpers.rb | 49 ++++++++++++ .../async_redis/sentinels_client_acl_tls.rb | 42 +++------- lib/3scale/backend/storage_async/client.rb | 79 +------------------ 4 files changed, 100 insertions(+), 108 deletions(-) create mode 100644 lib/3scale/backend/async_redis/client.rb create mode 100644 lib/3scale/backend/async_redis/endpoint_helpers.rb diff --git a/lib/3scale/backend/async_redis/client.rb b/lib/3scale/backend/async_redis/client.rb new file mode 100644 index 00000000..6d6afaf0 --- /dev/null +++ b/lib/3scale/backend/async_redis/client.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +# Based on https://github.com/socketry/async-redis/blob/v0.8.1/examples/auth/wrapper.rb + +require 'async/redis/client' +require '3scale/backend/async_redis/endpoint_helpers' +require '3scale/backend/async_redis/sentinels_client_acl_tls' +require '3scale/backend/async_redis/protocol/extended_resp2' + +module ThreeScale + module Backend + module AsyncRedis + # Friendly client wrapper that supports SSL, AUTH and db SELECT + class Client + class << self + # @param opts [Hash] Redis connection options + # @return [Async::Redis::Client] + def call(opts) + uri = URI(opts[:url]) + + credentials = [ uri.user || opts[:username], uri.password || opts[:password]] + db = uri.path[1..-1].to_i if uri.path + + protocol = Protocol::ExtendedRESP2.new(db: db, credentials: credentials) + + if opts.key? :sentinels + SentinelsClientACLTLS.new(uri, protocol, opts) + else + endpoint = EndpointHelpers.prepare_endpoint_uri(uri, opts[:ssl], opts[:ssl_params]) + Async::Redis::Client.new(endpoint, protocol: protocol, **opts) + end + end + alias :connect :call + end + end + end + end +end diff --git a/lib/3scale/backend/async_redis/endpoint_helpers.rb b/lib/3scale/backend/async_redis/endpoint_helpers.rb new file mode 100644 index 00000000..4552727d --- /dev/null +++ b/lib/3scale/backend/async_redis/endpoint_helpers.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +require 'async/io' + +module ThreeScale + module Backend + module AsyncRedis + module EndpointHelpers + class << self + # @param uri [URI] + # @param ssl_params [Hash] + # @return [Async::IO::Endpoint::Generic] + def prepare_endpoint_uri(uri, ssl = false, ssl_params = nil) + tcp_endpoint = Async::IO::Endpoint.tcp(uri.hostname, uri.port) + + if ssl + ssl_context = OpenSSL::SSL::SSLContext.new + ssl_context.set_params(format_ssl_params(ssl_params)) if ssl_params + return Async::IO::SSLEndpoint.new(tcp_endpoint, ssl_context: ssl_context) + end + + tcp_endpoint + end + + # @param host [String] + # @param port [String] + # @param ssl_params [Hash] + # @return [Async::IO::Endpoint::Generic] + def prepare_endpoint(host, port, ssl = false, ssl_params = nil) + uri = URI("redis://#{host}:#{port}") + prepare_endpoint_uri(uri, ssl, ssl_params) + end + + def format_ssl_params(ssl_params) + cert = ssl_params[:cert].to_s.strip + key = ssl_params[:key].to_s.strip + return ssl_params if cert.empty? && key.empty? + + updated_ssl_params = ssl_params.dup + updated_ssl_params[:cert] = OpenSSL::X509::Certificate.new(File.read(cert)) + updated_ssl_params[:key] = OpenSSL::PKey.read(File.read(key)) + + updated_ssl_params + end + end + end + end + end +end diff --git a/lib/3scale/backend/async_redis/sentinels_client_acl_tls.rb b/lib/3scale/backend/async_redis/sentinels_client_acl_tls.rb index 861e38e6..1cb88ea4 100644 --- a/lib/3scale/backend/async_redis/sentinels_client_acl_tls.rb +++ b/lib/3scale/backend/async_redis/sentinels_client_acl_tls.rb @@ -7,12 +7,12 @@ module ThreeScale module Backend module AsyncRedis class SentinelsClientACLTLS < Async::Redis::SentinelsClient - def initialize(master_name, sentinels, role = :master, protocol = Protocol::RESP2, config = {}, **options) - @master_name = master_name - @sentinel_endpoints = sentinels.map do |sentinel| - make_endpoint(sentinel[:host], sentinel[:port], config[:ssl], config[:ssl_params]) + def initialize(uri, protocol = Async::Redis::Protocol::RESP2, config, **options) + @master_name = uri.host + @sentinel_endpoints = config[:sentinels].map do |sentinel| + EndpointHelpers.prepare_endpoint(sentinel[:host], sentinel[:port], config[:ssl], config[:ssl_params]) end - @role = role + @role = config[:role] || :master @protocol = protocol @config = config @@ -23,7 +23,7 @@ def initialize(master_name, sentinels, role = :master, protocol = Protocol::RESP def resolve_master @sentinel_endpoints.each do |sentinel_endpoint| - client = Async::Redis::Client.new(sentinel_endpoint, protocol: ThreeScale::Backend::AsyncRedis::Protocol::ExtendedRESP2.new(credentials: @protocol.credentials)) + client = Async::Redis::Client.new(sentinel_endpoint, protocol: Protocol::ExtendedRESP2.new(credentials: @protocol.credentials)) begin address = client.call('sentinel', 'get-master-addr-by-name', @master_name) @@ -31,7 +31,7 @@ def resolve_master next end - return make_endpoint(address[0], address[1], @config[:ssl], @config[:ssl_params]) if address + return EndpointHelpers.prepare_endpoint(address[0], address[1], @config[:ssl], @config[:ssl_params]) if address end nil @@ -39,7 +39,7 @@ def resolve_master def resolve_slave @sentinel_endpoints.each do |sentinel_endpoint| - client = Async::Redis::Client.new(sentinel_endpoint, protocol: ThreeScale::Backend::AsyncRedis::Protocol::ExtendedRESP2.new(credentials: @protocol.credentials)) + client = Async::Redis::Client.new(sentinel_endpoint, protocol: Protocol::ExtendedRESP2.new(credentials: @protocol.credentials)) begin reply = client.call('sentinel', 'slaves', @master_name) @@ -51,35 +51,11 @@ def resolve_slave next if slaves.empty? slave = select_slave(slaves) - return make_endpoint(slave['ip'], slave['port'], @config[:ssl], @config[:ssl_params]) + return EndpointHelpers.prepare_endpoint(slave['ip'], slave['port'], @config[:ssl], @config[:ssl_params]) end nil end - - def make_endpoint(host, port, ssl, ssl_params) - tcp_endpoint = Async::IO::Endpoint.tcp(host, port) - - if ssl - ssl_context = OpenSSL::SSL::SSLContext.new - ssl_context.set_params(format_ssl_params(ssl_params)) - return Async::IO::SSLEndpoint.new(tcp_endpoint, ssl_context: ssl_context) - end - - tcp_endpoint - end - - def format_ssl_params(ssl_params) - cert = ssl_params[:cert].to_s.strip - key = ssl_params[:key].to_s.strip - return ssl_params if cert.empty? && key.empty? - - updated_ssl_params = ssl_params.dup - updated_ssl_params[:cert] = OpenSSL::X509::Certificate.new(File.read(cert)) - updated_ssl_params[:key] = OpenSSL::PKey.read(File.read(key)) - - updated_ssl_params - end end end end diff --git a/lib/3scale/backend/storage_async/client.rb b/lib/3scale/backend/storage_async/client.rb index 7a3d6408..267129f2 100644 --- a/lib/3scale/backend/storage_async/client.rb +++ b/lib/3scale/backend/storage_async/client.rb @@ -1,7 +1,6 @@ -require 'async/io' -require 'async/redis/client' -require '3scale/backend/async_redis/sentinels_client_acl_tls' -require '3scale/backend/async_redis/protocol/extended_resp2' +# frozen_string_literal: true + +require '3scale/backend/async_redis/client' module ThreeScale module Backend @@ -35,7 +34,7 @@ def instance(reset = false) end def initialize(opts) - @redis_async = initialize_client(opts) + @redis_async = AsyncRedis::Client.call(opts) end def call(*args) @@ -59,76 +58,6 @@ def pipelined(&block) def close @redis_async.close end - - private - - DEFAULT_SCHEME = 'redis' - DEFAULT_HOST = 'localhost'.freeze - DEFAULT_PORT = 6379 - - def initialize_client(opts) - return init_host_client(opts) unless opts.key? :sentinels - - init_sentinels_client(opts) - end - - def init_host_client(opts) - endpoint = make_redis_endpoint(opts) - protocol = make_redis_protocol(opts) - Async::Redis::Client.new(endpoint, protocol: protocol, limit: opts[:max_connections]) - end - - def init_sentinels_client(opts) - uri = URI(opts[:url] || '') - name = uri.host - role = opts[:role] || :master - protocol = make_redis_protocol(opts) - - ThreeScale::Backend::AsyncRedis::SentinelsClientACLTLS.new(name, opts[:sentinels], role, protocol, opts) - end - - # RESP2 with support for logical DBs - def make_redis_protocol(opts) - uri = URI(opts[:url] || "") - db = uri.path[1..-1] - credentials = [ uri.user || opts[:username], uri.password || opts[:password]] - - ThreeScale::Backend::AsyncRedis::Protocol::ExtendedRESP2.new(db: db, credentials: credentials) - end - - # SSL endpoint if scheme is `rediss:`, TCP endpoint otherwise. - # Note: Unix socket endpoint is not supported in async mode - def make_redis_endpoint(opts) - uri = URI(opts[:url] || "") - scheme = uri.scheme || DEFAULT_SCHEME - host = uri.host || DEFAULT_HOST - port = uri.port || DEFAULT_PORT - - tcp_endpoint = Async::IO::Endpoint.tcp(host, port) - - case scheme - when 'redis' - tcp_endpoint - when 'rediss' - ssl_context = OpenSSL::SSL::SSLContext.new - ssl_context.set_params(format_ssl_params(opts[:ssl_params])) - Async::IO::SSLEndpoint.new(tcp_endpoint, ssl_context: ssl_context) - else - raise ArgumentError - end - end - - def format_ssl_params(ssl_params) - cert = ssl_params[:cert].to_s.strip - key = ssl_params[:key].to_s.strip - return ssl_params if cert.empty? && key.empty? - - updated_ssl_params = ssl_params.dup - updated_ssl_params[:cert] = OpenSSL::X509::Certificate.new(File.read(cert)) - updated_ssl_params[:key] = OpenSSL::PKey.read(File.read(key)) - - updated_ssl_params - end end end end From 470e9d5ddd5e9b2ad7b6c163284ba1f5d5137c2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Thu, 23 May 2024 12:56:21 +0200 Subject: [PATCH 52/54] Move macros to helper --- lib/3scale/backend/async_redis/endpoint_helpers.rb | 4 ++++ lib/3scale/backend/storage_async/client.rb | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/3scale/backend/async_redis/endpoint_helpers.rb b/lib/3scale/backend/async_redis/endpoint_helpers.rb index 4552727d..de1fa2c3 100644 --- a/lib/3scale/backend/async_redis/endpoint_helpers.rb +++ b/lib/3scale/backend/async_redis/endpoint_helpers.rb @@ -6,6 +6,10 @@ module ThreeScale module Backend module AsyncRedis module EndpointHelpers + + DEFAULT_HOST = 'localhost'.freeze + DEFAULT_PORT = 6379 + class << self # @param uri [URI] # @param ssl_params [Hash] diff --git a/lib/3scale/backend/storage_async/client.rb b/lib/3scale/backend/storage_async/client.rb index 267129f2..749af3bf 100644 --- a/lib/3scale/backend/storage_async/client.rb +++ b/lib/3scale/backend/storage_async/client.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +require '3scale/backend/async_redis/endpoint_helpers' require '3scale/backend/async_redis/client' module ThreeScale @@ -24,7 +25,7 @@ def instance(reset = false) @instance = new( Storage::Helpers.config_with( configuration.redis, - options: { default_url: "#{DEFAULT_HOST}:#{DEFAULT_PORT}" } + options: { default_url: "#{AsyncRedis::EndpointHelpers::DEFAULT_HOST}:#{AsyncRedis::EndpointHelpers::DEFAULT_PORT}" } ) ) else From 1e1e25756d3b888c9601d21f8549b4ef9c8e3c51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Thu, 23 May 2024 13:11:54 +0200 Subject: [PATCH 53/54] Simplify helper --- lib/3scale/backend/async_redis/client.rb | 4 ++-- .../backend/async_redis/endpoint_helpers.rb | 17 +++++------------ 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/lib/3scale/backend/async_redis/client.rb b/lib/3scale/backend/async_redis/client.rb index 6d6afaf0..878a9851 100644 --- a/lib/3scale/backend/async_redis/client.rb +++ b/lib/3scale/backend/async_redis/client.rb @@ -26,8 +26,8 @@ def call(opts) if opts.key? :sentinels SentinelsClientACLTLS.new(uri, protocol, opts) else - endpoint = EndpointHelpers.prepare_endpoint_uri(uri, opts[:ssl], opts[:ssl_params]) - Async::Redis::Client.new(endpoint, protocol: protocol, **opts) + endpoint = EndpointHelpers.prepare_endpoint(uri.host, uri.port, opts[:ssl], opts[:ssl_params]) + Async::Redis::Client.new(endpoint, protocol: protocol, limit: opts[:max_connections]) end end alias :connect :call diff --git a/lib/3scale/backend/async_redis/endpoint_helpers.rb b/lib/3scale/backend/async_redis/endpoint_helpers.rb index de1fa2c3..edcb8c38 100644 --- a/lib/3scale/backend/async_redis/endpoint_helpers.rb +++ b/lib/3scale/backend/async_redis/endpoint_helpers.rb @@ -11,11 +11,13 @@ module EndpointHelpers DEFAULT_PORT = 6379 class << self - # @param uri [URI] + + # @param host [String] + # @param port [String] # @param ssl_params [Hash] # @return [Async::IO::Endpoint::Generic] - def prepare_endpoint_uri(uri, ssl = false, ssl_params = nil) - tcp_endpoint = Async::IO::Endpoint.tcp(uri.hostname, uri.port) + def prepare_endpoint(host, port, ssl = false, ssl_params = nil) + tcp_endpoint = Async::IO::Endpoint.tcp(host, port) if ssl ssl_context = OpenSSL::SSL::SSLContext.new @@ -26,15 +28,6 @@ def prepare_endpoint_uri(uri, ssl = false, ssl_params = nil) tcp_endpoint end - # @param host [String] - # @param port [String] - # @param ssl_params [Hash] - # @return [Async::IO::Endpoint::Generic] - def prepare_endpoint(host, port, ssl = false, ssl_params = nil) - uri = URI("redis://#{host}:#{port}") - prepare_endpoint_uri(uri, ssl, ssl_params) - end - def format_ssl_params(ssl_params) cert = ssl_params[:cert].to_s.strip key = ssl_params[:key].to_s.strip From cb28b7e3c6329fc22ccbeec422d701a8150415b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Thu, 23 May 2024 13:50:19 +0200 Subject: [PATCH 54/54] Set some default configs --- lib/3scale/backend/async_redis/client.rb | 4 +++- lib/3scale/backend/async_redis/endpoint_helpers.rb | 2 +- lib/3scale/backend/storage_helpers.rb | 1 + test/unit/storage_async_test.rb | 6 ------ 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/lib/3scale/backend/async_redis/client.rb b/lib/3scale/backend/async_redis/client.rb index 878a9851..72ee74b7 100644 --- a/lib/3scale/backend/async_redis/client.rb +++ b/lib/3scale/backend/async_redis/client.rb @@ -26,7 +26,9 @@ def call(opts) if opts.key? :sentinels SentinelsClientACLTLS.new(uri, protocol, opts) else - endpoint = EndpointHelpers.prepare_endpoint(uri.host, uri.port, opts[:ssl], opts[:ssl_params]) + host = uri.host || EndpointHelpers::DEFAULT_HOST + port = uri.port || EndpointHelpers::DEFAULT_PORT + endpoint = EndpointHelpers.prepare_endpoint(host, port, opts[:ssl], opts[:ssl_params]) Async::Redis::Client.new(endpoint, protocol: protocol, limit: opts[:max_connections]) end end diff --git a/lib/3scale/backend/async_redis/endpoint_helpers.rb b/lib/3scale/backend/async_redis/endpoint_helpers.rb index edcb8c38..0edcd30b 100644 --- a/lib/3scale/backend/async_redis/endpoint_helpers.rb +++ b/lib/3scale/backend/async_redis/endpoint_helpers.rb @@ -13,7 +13,7 @@ module EndpointHelpers class << self # @param host [String] - # @param port [String] + # @param port [Integer] # @param ssl_params [Hash] # @return [Async::IO::Endpoint::Generic] def prepare_endpoint(host, port, ssl = false, ssl_params = nil) diff --git a/lib/3scale/backend/storage_helpers.rb b/lib/3scale/backend/storage_helpers.rb index 7e96100b..4ca75c71 100644 --- a/lib/3scale/backend/storage_helpers.rb +++ b/lib/3scale/backend/storage_helpers.rb @@ -281,6 +281,7 @@ def cfg_defaults_handler(options, defaults) cfg_with_defaults = defaults.merge(ensure_url_param(options)) cfg_with_defaults = cfg_unix_path_handler(cfg_with_defaults) cfg_with_defaults.delete(:max_connections) unless options[:async] + cfg_with_defaults[:ssl] ||= true if URI(options[:url].to_s).scheme == 'rediss' cfg_with_defaults end diff --git a/test/unit/storage_async_test.rb b/test/unit/storage_async_test.rb index fc10b3f2..964361cd 100644 --- a/test/unit/storage_async_test.rb +++ b/test/unit/storage_async_test.rb @@ -42,12 +42,6 @@ def test_redis_no_scheme assert_client_config({ url: URI('redis://backend-redis:6379') }, storage) end - def test_redis_unknown_scheme - assert_raise ArgumentError do - StorageAsync::Client.send :new, url('myscheme://127.0.0.1:6379') - end - end - def test_sentinels_connection_string config_obj = { url: 'redis://master-group-name',