From 32d337d434305f7045915c545f1f5934ca1655a7 Mon Sep 17 00:00:00 2001 From: Alexander Zagaynov Date: Fri, 21 Sep 2018 12:44:16 +0200 Subject: [PATCH 1/9] update aws-sdk gem to v3 --- manageiq-providers-amazon.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manageiq-providers-amazon.gemspec b/manageiq-providers-amazon.gemspec index 9ef5aef6f..6bea1b83a 100644 --- a/manageiq-providers-amazon.gemspec +++ b/manageiq-providers-amazon.gemspec @@ -13,7 +13,7 @@ Gem::Specification.new do |s| s.files = Dir["{app,config,lib}/**/*"] - s.add_dependency("aws-sdk", ["~>2.9.7"]) + s.add_dependency "aws-sdk", "~> 3.0" s.add_development_dependency "codeclimate-test-reporter", "~> 1.0.0" s.add_development_dependency "simplecov" From 7cc638c78590882871dfb0f7fe0bb40372bff7f2 Mon Sep 17 00:00:00 2001 From: Alexander Zagaynov Date: Wed, 26 Sep 2018 12:49:08 +0200 Subject: [PATCH 2/9] Revert "Merge pull request #434 from NickLaMuro/aws-sdk-auth-monkey-patch-spec" This reverts commit a08595f9803581078ee19692a8bd2333fc318d7c, reversing changes made to ce647bcb3d4b13751f38d2f49a13fcf7edbb5e2e. --- .../seahorse_client_net_http_pool_patch.rb | 2 - ...eahorse_client_net_http_pool_patch_spec.rb | 58 ------------------- 2 files changed, 60 deletions(-) delete mode 100644 spec/lib/patches/aws-sdk-core/seahorse_client_net_http_pool_patch_spec.rb diff --git a/lib/patches/aws-sdk-core/seahorse_client_net_http_pool_patch.rb b/lib/patches/aws-sdk-core/seahorse_client_net_http_pool_patch.rb index e8910338a..3abca5a36 100644 --- a/lib/patches/aws-sdk-core/seahorse_client_net_http_pool_patch.rb +++ b/lib/patches/aws-sdk-core/seahorse_client_net_http_pool_patch.rb @@ -5,8 +5,6 @@ module Seahorse module Client module NetHttp class ConnectionPool - @_original_start_session_source_loc = self.new.method(:start_session).source_location - def start_session endpoint endpoint = URI.parse(endpoint) diff --git a/spec/lib/patches/aws-sdk-core/seahorse_client_net_http_pool_patch_spec.rb b/spec/lib/patches/aws-sdk-core/seahorse_client_net_http_pool_patch_spec.rb deleted file mode 100644 index 6aa8739cc..000000000 --- a/spec/lib/patches/aws-sdk-core/seahorse_client_net_http_pool_patch_spec.rb +++ /dev/null @@ -1,58 +0,0 @@ -require 'aws-sdk' -require 'patches/aws-sdk-core/seahorse_client_net_http_pool_patch' - -describe Seahorse::Client::NetHttp::ConnectionPool do - describe "#start_session (monkey patched)" do - let(:patch_file) { File.join(%w(lib patches aws-sdk-core seahorse_client_net_http_pool_patch.rb)) } - let(:patch_filepath) { ManageIQ::Providers::Amazon::Engine.root.join(patch_file).to_s } - let(:source_location) { subject.method(:start_session).source_location } - - it "is now defined in the patch file" do - expect(source_location.first).to eq(patch_filepath) - end - - # This spec exists to confirm that the monkey patch we are doing to - # Seahorse::Client::NetHttp::ConnectionPool#start_session is still valid by - # comparing the source code of origial method to the patch code. - # - # Patch code for this can be found here: - # - # lib/patches/aws-sdk-core/seahorse_client_net_http_pool_patch.rb - # - # If this spec fails, make sure to check the difference between the patched - # code and what is in the original gem, and make the necessary changes so - # the only changes are to the http_proxy.user and http_proxy.password - # lines. - it "only changes the necessary lines" do - # How this works: - # - # We can basically assume with our current patch that the line counts for - # the method will be the same, so don't try and lex the file to get the - # method definition, just calculate it from a known quantity (from the - # patch itself) - # - # * patch_code is determined by taking the source_location line and - # reading up to the 5th to last line in the file, since we know that - # is the `end` statement of the method definition. - # * original_last_lineno is determined by determining taking the line - # number from the @_original_start_session_source_loc, and reading - # the number of lines from the original gem source equal to the total - # lines from the `patch_code` - # - # If the number of lines is off between the patch_code and the original, - # then it will fail, and that is also acceptable since the changes we - # made in the patch were done "inline" when compared to the original. - patch_code = File.readlines(patch_filepath)[source_location[1] - 1..-5].join("") - - original_file, original_lineno = described_class.instance_variable_get(:@_original_start_session_source_loc) - original_last_lineno = original_lineno + patch_code.lines.count - 1 - original_code = File.readlines(original_file)[original_lineno - 1..original_last_lineno - 1].join("") - - # Expect lines 1 through 9 to be the same in both code paths - expect(patch_code.lines[0..8].join).to eq(original_code.lines[0..8].join) - - # Expect lines 12 onward to be the same in both code paths - expect(patch_code.lines[11..-1].join).to eq(original_code.lines[11..-1].join) - end - end -end From 1a5f22beb94f9d7b390b1798f1cff6d00ef9671d Mon Sep 17 00:00:00 2001 From: Alexander Zagaynov Date: Wed, 26 Sep 2018 12:51:17 +0200 Subject: [PATCH 3/9] Revert "Merge pull request #432 from NickLaMuro/monkey-patch-aws-sdk-core-for-proxy-auth-fix" This reverts commit 2b2c87f577ec429738404a0588c3ad1d8e0ed68c, reversing changes made to beed01f16b9db9db6c22cab93ee8812586fa0858. --- app/models/authenticator/amazon.rb | 1 - .../providers/amazon/manager_mixin.rb | 2 - .../seahorse_client_net_http_pool_patch.rb | 44 ------------------- 3 files changed, 47 deletions(-) delete mode 100644 lib/patches/aws-sdk-core/seahorse_client_net_http_pool_patch.rb diff --git a/app/models/authenticator/amazon.rb b/app/models/authenticator/amazon.rb index 6e4b1e835..b6d8a329d 100644 --- a/app/models/authenticator/amazon.rb +++ b/app/models/authenticator/amazon.rb @@ -130,7 +130,6 @@ def aws_connect(access_key_id, secret_access_key, service = :IAM, proxy_uri = ni proxy_uri ||= VMDB::Util.http_proxy_uri require 'aws-sdk' - require 'patches/aws-sdk-core/seahorse_client_net_http_pool_patch' Aws.const_get(service)::Resource.new( :access_key_id => access_key_id, :secret_access_key => secret_access_key, diff --git a/app/models/manageiq/providers/amazon/manager_mixin.rb b/app/models/manageiq/providers/amazon/manager_mixin.rb index 87d623e9e..4c8029ddd 100644 --- a/app/models/manageiq/providers/amazon/manager_mixin.rb +++ b/app/models/manageiq/providers/amazon/manager_mixin.rb @@ -80,7 +80,6 @@ def raw_connect(access_key_id, secret_access_key, service, region, proxy_uri = nil, validate = false, uri = nil, assume_role: nil) require 'aws-sdk' - require 'patches/aws-sdk-core/seahorse_client_net_http_pool_patch' log_formatter_pattern = Aws::Log::Formatter.default.pattern.chomp secret_access_key = ManageIQ::Password.try_decrypt(secret_access_key) @@ -129,7 +128,6 @@ def connection_rescue_block def translate_exception(err) require 'aws-sdk' - require 'patches/aws-sdk-core/seahorse_client_net_http_pool_patch' case err when Aws::EC2::Errors::SignatureDoesNotMatch MiqException::MiqHostError.new "SignatureMismatch - check your AWS Secret Access Key and signing method" diff --git a/lib/patches/aws-sdk-core/seahorse_client_net_http_pool_patch.rb b/lib/patches/aws-sdk-core/seahorse_client_net_http_pool_patch.rb deleted file mode 100644 index 3abca5a36..000000000 --- a/lib/patches/aws-sdk-core/seahorse_client_net_http_pool_patch.rb +++ /dev/null @@ -1,44 +0,0 @@ -# Autoload the connection pool -Seahorse::Client::NetHttp::ConnectionPool - -module Seahorse - module Client - module NetHttp - class ConnectionPool - def start_session endpoint - - endpoint = URI.parse(endpoint) - - args = [] - args << endpoint.host - args << endpoint.port - args << http_proxy.host - args << http_proxy.port - args << (http_proxy.user && CGI::unescape(http_proxy.user)) - args << (http_proxy.password && CGI::unescape(http_proxy.password)) - - http = ExtendedSession.new(Net::HTTP.new(*args.compact)) - http.set_debug_output(logger) if http_wire_trace? - http.open_timeout = http_open_timeout - - if endpoint.scheme == 'https' - http.use_ssl = true - if ssl_verify_peer? - http.verify_mode = OpenSSL::SSL::VERIFY_PEER - http.ca_file = ssl_ca_bundle if ssl_ca_bundle - http.ca_path = ssl_ca_directory if ssl_ca_directory - http.cert_store = ssl_ca_store if ssl_ca_store - else - http.verify_mode = OpenSSL::SSL::VERIFY_NONE - end - else - http.use_ssl = false - end - - http.start - http - end - end - end - end -end From 79c5d279f4e442f594a3e3795a0af3be7ba3e0cc Mon Sep 17 00:00:00 2001 From: Alexander Zagaynov Date: Wed, 3 Oct 2018 15:51:21 +0200 Subject: [PATCH 4/9] fix 'invalid configuration option' issue --- app/models/manageiq/providers/amazon/manager_mixin.rb | 3 +-- .../amazon/cloud_manager/event_catcher/stream_spec.rb | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/app/models/manageiq/providers/amazon/manager_mixin.rb b/app/models/manageiq/providers/amazon/manager_mixin.rb index 4c8029ddd..1922ce55d 100644 --- a/app/models/manageiq/providers/amazon/manager_mixin.rb +++ b/app/models/manageiq/providers/amazon/manager_mixin.rb @@ -78,8 +78,7 @@ def verify_credentials(args) def raw_connect(access_key_id, secret_access_key, service, region, proxy_uri = nil, validate = false, uri = nil, assume_role: nil) - - require 'aws-sdk' + require "aws-sdk-#{service.to_s.downcase}" log_formatter_pattern = Aws::Log::Formatter.default.pattern.chomp secret_access_key = ManageIQ::Password.try_decrypt(secret_access_key) diff --git a/spec/models/manageiq/providers/amazon/cloud_manager/event_catcher/stream_spec.rb b/spec/models/manageiq/providers/amazon/cloud_manager/event_catcher/stream_spec.rb index 9ff826b5c..8d9cca348 100644 --- a/spec/models/manageiq/providers/amazon/cloud_manager/event_catcher/stream_spec.rb +++ b/spec/models/manageiq/providers/amazon/cloud_manager/event_catcher/stream_spec.rb @@ -1,5 +1,8 @@ require_relative "../../aws_helper" +require 'aws-sdk-sqs' +require 'aws-sdk-sns' + describe ManageIQ::Providers::Amazon::CloudManager::EventCatcher::Stream do subject do ems = FactoryBot.create(:ems_amazon_with_authentication) From e44fe91bc95c5420c76fa15b265944d879fcc1f8 Mon Sep 17 00:00:00 2001 From: Alexander Zagaynov Date: Fri, 5 Oct 2018 16:41:10 +0200 Subject: [PATCH 5/9] fix 'undefined method' error --- .../specific_stubbed_refresher_spec.rb | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/spec/models/manageiq/providers/amazon/cloud_manager/specific_stubbed_refresher_spec.rb b/spec/models/manageiq/providers/amazon/cloud_manager/specific_stubbed_refresher_spec.rb index 659874640..a3564f25b 100644 --- a/spec/models/manageiq/providers/amazon/cloud_manager/specific_stubbed_refresher_spec.rb +++ b/spec/models/manageiq/providers/amazon/cloud_manager/specific_stubbed_refresher_spec.rb @@ -2,6 +2,8 @@ require_relative '../aws_stubs' require_relative '../aws_refresher_spec_common' +require 'aws-sdk-cloudformation' + describe ManageIQ::Providers::Amazon::NetworkManager::Refresher do include AwsRefresherSpecCommon include AwsStubs @@ -29,9 +31,9 @@ context "OrchestrationStack refresh" do context "with all empty relations" do - let(:mocked_stack_parameters) { nil } - let(:mocked_stack_resources) { nil } - let(:mocked_stack_outputs) { nil } + let(:mocked_stack_parameters) { [] } + let(:mocked_stack_resources) { [] } + let(:mocked_stack_outputs) { [] } it "tests refresh passes" do with_aws_stubbed(stub_responses) do @@ -47,7 +49,7 @@ end context "with empty parameters relation" do - let(:mocked_stack_parameters) { nil } + let(:mocked_stack_parameters) { [] } it "tests refresh passes" do with_aws_stubbed(stub_responses) do @@ -62,7 +64,7 @@ end context "with empty outputs relation" do - let(:mocked_stack_outputs) { nil } + let(:mocked_stack_outputs) { [] } it "tests refresh passes" do with_aws_stubbed(stub_responses) do @@ -77,7 +79,7 @@ end context "with empty resources relation" do - let(:mocked_stack_resources) { nil } + let(:mocked_stack_resources) { [] } it "tests refresh passes" do with_aws_stubbed(stub_responses) do From ffc67d41bfd0e9f25176812f74728a83cd46dfe1 Mon Sep 17 00:00:00 2001 From: Alexander Zagaynov Date: Wed, 10 Oct 2018 13:02:15 +0200 Subject: [PATCH 6/9] fix counts assertion errors --- .../providers/amazon/network_manager/stubbed_refresher_spec.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spec/models/manageiq/providers/amazon/network_manager/stubbed_refresher_spec.rb b/spec/models/manageiq/providers/amazon/network_manager/stubbed_refresher_spec.rb index 03cab1e5c..0438cd777 100644 --- a/spec/models/manageiq/providers/amazon/network_manager/stubbed_refresher_spec.rb +++ b/spec/models/manageiq/providers/amazon/network_manager/stubbed_refresher_spec.rb @@ -2,6 +2,8 @@ require_relative '../aws_stubs' require_relative '../aws_refresher_spec_common' +require 'aws-sdk-elasticloadbalancing' + describe ManageIQ::Providers::Amazon::NetworkManager::Refresher do include AwsRefresherSpecCommon include AwsStubs From e5904f0e896cd1863199eb09e86d4b445d0eb1c2 Mon Sep 17 00:00:00 2001 From: Alexander Zagaynov Date: Wed, 10 Oct 2018 14:19:24 +0200 Subject: [PATCH 7/9] better approach to avoid 'invalid configuration option' error --- app/models/authenticator/amazon.rb | 2 +- spec/models/manageiq/providers/amazon/aws_helper.rb | 11 +++++++---- .../amazon/cloud_manager/event_catcher/stream_spec.rb | 3 --- .../cloud_manager/specific_stubbed_refresher_spec.rb | 2 -- .../amazon/network_manager/stubbed_refresher_spec.rb | 2 -- 5 files changed, 8 insertions(+), 12 deletions(-) diff --git a/app/models/authenticator/amazon.rb b/app/models/authenticator/amazon.rb index b6d8a329d..5d8691abf 100644 --- a/app/models/authenticator/amazon.rb +++ b/app/models/authenticator/amazon.rb @@ -128,8 +128,8 @@ def verify_credentials(access_key_id, secret_access_key) def aws_connect(access_key_id, secret_access_key, service = :IAM, proxy_uri = nil) service ||= :EC2 proxy_uri ||= VMDB::Util.http_proxy_uri + require "aws-sdk-#{service.to_s.downcase}" - require 'aws-sdk' Aws.const_get(service)::Resource.new( :access_key_id => access_key_id, :secret_access_key => secret_access_key, diff --git a/spec/models/manageiq/providers/amazon/aws_helper.rb b/spec/models/manageiq/providers/amazon/aws_helper.rb index 78393e459..bd8be8a5d 100644 --- a/spec/models/manageiq/providers/amazon/aws_helper.rb +++ b/spec/models/manageiq/providers/amazon/aws_helper.rb @@ -1,10 +1,13 @@ -require 'aws-sdk' +# frozen_string_literal: true def with_aws_stubbed(stub_responses_per_service) stub_responses_per_service.each do |service, stub_responses| - raise "Aws.config[#{service}][:stub_responses] already set" if Aws.config.fetch(service, {})[:stub_responses] - Aws.config[service] ||= {} - Aws.config[service][:stub_responses] = stub_responses + if Aws.config.dig(service, :stub_responses).present? + raise "Aws.config[#{service}][:stub_responses] already set" + else + require "aws-sdk-#{service.to_s.downcase}" + (Aws.config[service] ||= {})[:stub_responses] = stub_responses + end end yield ensure diff --git a/spec/models/manageiq/providers/amazon/cloud_manager/event_catcher/stream_spec.rb b/spec/models/manageiq/providers/amazon/cloud_manager/event_catcher/stream_spec.rb index 8d9cca348..9ff826b5c 100644 --- a/spec/models/manageiq/providers/amazon/cloud_manager/event_catcher/stream_spec.rb +++ b/spec/models/manageiq/providers/amazon/cloud_manager/event_catcher/stream_spec.rb @@ -1,8 +1,5 @@ require_relative "../../aws_helper" -require 'aws-sdk-sqs' -require 'aws-sdk-sns' - describe ManageIQ::Providers::Amazon::CloudManager::EventCatcher::Stream do subject do ems = FactoryBot.create(:ems_amazon_with_authentication) diff --git a/spec/models/manageiq/providers/amazon/cloud_manager/specific_stubbed_refresher_spec.rb b/spec/models/manageiq/providers/amazon/cloud_manager/specific_stubbed_refresher_spec.rb index a3564f25b..5b4dae51f 100644 --- a/spec/models/manageiq/providers/amazon/cloud_manager/specific_stubbed_refresher_spec.rb +++ b/spec/models/manageiq/providers/amazon/cloud_manager/specific_stubbed_refresher_spec.rb @@ -2,8 +2,6 @@ require_relative '../aws_stubs' require_relative '../aws_refresher_spec_common' -require 'aws-sdk-cloudformation' - describe ManageIQ::Providers::Amazon::NetworkManager::Refresher do include AwsRefresherSpecCommon include AwsStubs diff --git a/spec/models/manageiq/providers/amazon/network_manager/stubbed_refresher_spec.rb b/spec/models/manageiq/providers/amazon/network_manager/stubbed_refresher_spec.rb index 0438cd777..03cab1e5c 100644 --- a/spec/models/manageiq/providers/amazon/network_manager/stubbed_refresher_spec.rb +++ b/spec/models/manageiq/providers/amazon/network_manager/stubbed_refresher_spec.rb @@ -2,8 +2,6 @@ require_relative '../aws_stubs' require_relative '../aws_refresher_spec_common' -require 'aws-sdk-elasticloadbalancing' - describe ManageIQ::Providers::Amazon::NetworkManager::Refresher do include AwsRefresherSpecCommon include AwsStubs From 3cfcf0f2c40b1c9dab7a30c71a4e2d367cb4ceb1 Mon Sep 17 00:00:00 2001 From: Alexander Zagaynov Date: Thu, 17 Oct 2019 15:16:48 +0200 Subject: [PATCH 8/9] require specific aws gems --- app/models/authenticator/amazon.rb | 1 + app/models/manageiq/providers/amazon/manager_mixin.rb | 2 +- manageiq-providers-amazon.gemspec | 2 +- spec/models/authenticator/amazon_spec.rb | 3 ++- spec/models/manageiq/providers/amazon/aws_helper.rb | 2 -- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/models/authenticator/amazon.rb b/app/models/authenticator/amazon.rb index 5d8691abf..2320f82d2 100644 --- a/app/models/authenticator/amazon.rb +++ b/app/models/authenticator/amazon.rb @@ -128,6 +128,7 @@ def verify_credentials(access_key_id, secret_access_key) def aws_connect(access_key_id, secret_access_key, service = :IAM, proxy_uri = nil) service ||= :EC2 proxy_uri ||= VMDB::Util.http_proxy_uri + require "aws-sdk-#{service.to_s.downcase}" Aws.const_get(service)::Resource.new( diff --git a/app/models/manageiq/providers/amazon/manager_mixin.rb b/app/models/manageiq/providers/amazon/manager_mixin.rb index 1922ce55d..a06b3f929 100644 --- a/app/models/manageiq/providers/amazon/manager_mixin.rb +++ b/app/models/manageiq/providers/amazon/manager_mixin.rb @@ -126,7 +126,7 @@ def connection_rescue_block end def translate_exception(err) - require 'aws-sdk' + require 'aws-sdk-ec2' case err when Aws::EC2::Errors::SignatureDoesNotMatch MiqException::MiqHostError.new "SignatureMismatch - check your AWS Secret Access Key and signing method" diff --git a/manageiq-providers-amazon.gemspec b/manageiq-providers-amazon.gemspec index 6bea1b83a..112ed464c 100644 --- a/manageiq-providers-amazon.gemspec +++ b/manageiq-providers-amazon.gemspec @@ -13,7 +13,7 @@ Gem::Specification.new do |s| s.files = Dir["{app,config,lib}/**/*"] - s.add_dependency "aws-sdk", "~> 3.0" + s.add_dependency "aws-sdk", "~> 3.0.1" s.add_development_dependency "codeclimate-test-reporter", "~> 1.0.0" s.add_development_dependency "simplecov" diff --git a/spec/models/authenticator/amazon_spec.rb b/spec/models/authenticator/amazon_spec.rb index 0eaed1073..fd571f9ca 100644 --- a/spec/models/authenticator/amazon_spec.rb +++ b/spec/models/authenticator/amazon_spec.rb @@ -1,4 +1,5 @@ -require 'aws-sdk' +require 'aws-sdk-iam' +require 'aws-sdk-ec2' describe Authenticator::Amazon do AWS_ROOT_USER_KEY = 'aws_root_key'.freeze diff --git a/spec/models/manageiq/providers/amazon/aws_helper.rb b/spec/models/manageiq/providers/amazon/aws_helper.rb index bd8be8a5d..737117731 100644 --- a/spec/models/manageiq/providers/amazon/aws_helper.rb +++ b/spec/models/manageiq/providers/amazon/aws_helper.rb @@ -1,5 +1,3 @@ -# frozen_string_literal: true - def with_aws_stubbed(stub_responses_per_service) stub_responses_per_service.each do |service, stub_responses| if Aws.config.dig(service, :stub_responses).present? From 8be7e04e164aad04341958c807bfeb6dc05c5082 Mon Sep 17 00:00:00 2001 From: Alexander Zagaynov Date: Thu, 17 Oct 2019 15:35:27 +0200 Subject: [PATCH 9/9] fix failing spec --- .../providers/amazon/cloud_manager/event_catcher/stream_spec.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spec/models/manageiq/providers/amazon/cloud_manager/event_catcher/stream_spec.rb b/spec/models/manageiq/providers/amazon/cloud_manager/event_catcher/stream_spec.rb index 9ff826b5c..4ea16e722 100644 --- a/spec/models/manageiq/providers/amazon/cloud_manager/event_catcher/stream_spec.rb +++ b/spec/models/manageiq/providers/amazon/cloud_manager/event_catcher/stream_spec.rb @@ -1,5 +1,7 @@ require_relative "../../aws_helper" +require 'aws-sdk-sns' + describe ManageIQ::Providers::Amazon::CloudManager::EventCatcher::Stream do subject do ems = FactoryBot.create(:ems_amazon_with_authentication)