From 5086ae5a11aaa5778ed939642684b749a5c823cd Mon Sep 17 00:00:00 2001 From: Alexander Zagaynov Date: Fri, 21 Sep 2018 12:43:07 +0200 Subject: [PATCH 1/4] update aws-sdk gem to v3 --- manageiq-gems-pending.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manageiq-gems-pending.gemspec b/manageiq-gems-pending.gemspec index d11bc0789..3cf2efbb2 100644 --- a/manageiq-gems-pending.gemspec +++ b/manageiq-gems-pending.gemspec @@ -23,7 +23,7 @@ Gem::Specification.new do |s| s.add_runtime_dependency "activerecord", "< 5.3" s.add_runtime_dependency "activesupport", "< 5.3" s.add_runtime_dependency "awesome_spawn", "~> 1.4" - s.add_runtime_dependency "aws-sdk", "~> 2.9.7" + s.add_runtime_dependency "aws-sdk", "~> 3.0" s.add_runtime_dependency "binary_struct", "~> 2.1" s.add_runtime_dependency "bundler", ">= 1.8.4" # rails-assets requires bundler >= 1.8.4, see: https://rails-assets.org/ s.add_runtime_dependency "fog-openstack", "~> 0.3" From af2ff93237e86c3ff6a448712793288a739c68fe Mon Sep 17 00:00:00 2001 From: Alexander Zagaynov Date: Thu, 11 Oct 2018 15:21:46 +0200 Subject: [PATCH 2/4] remove obsolete patch --- .../aws-sdk/s3_upload_stream_patch.rb | 281 ------------------ 1 file changed, 281 deletions(-) delete mode 100644 lib/gems/pending/util/extensions/aws-sdk/s3_upload_stream_patch.rb diff --git a/lib/gems/pending/util/extensions/aws-sdk/s3_upload_stream_patch.rb b/lib/gems/pending/util/extensions/aws-sdk/s3_upload_stream_patch.rb deleted file mode 100644 index 4ecbb30ee..000000000 --- a/lib/gems/pending/util/extensions/aws-sdk/s3_upload_stream_patch.rb +++ /dev/null @@ -1,281 +0,0 @@ -# This is a backport of the `Aws::S3::Object#upload_stream` method that is -# introduced in the V3 of the 'aws-sdk' -# -# To my knowledge, the changes from this patch should be completely backwards -# compatible, and the API should easily support this change. -# -# Once upgraded, this file should be removed. - -require 'aws-sdk' - -if defined? Aws::S3::GEM_VERSION # This constant doesn't exist in v2 - raise <<~ERROR - The 'aws-sdk' gem has been updated to v3!!! - - Remove this file! - - #{File.expand_path(__FILE__)} - - And delete any references to it, starting with this one: - - #{caller(1..1).first} - - This functionality was backported and should now exist normally in the - aws-sdk library, so no monkey patching is necessary. - - Horray! - ERROR -else - # make sure the client is autoloaded before we continue - Aws::S3::Client -end - -# The code below comes directly from the aws-sdk, so ignore rubocop warnings -# for the remainder of this file. -# -# rubocop:disable all - -### Aws::S3::MultipartStreamUploader -# -# Ripped from: -# -# https://github.com/aws/aws-sdk-ruby/blob/708140b8/gems/aws-sdk-s3/lib/aws-sdk-s3/multipart_stream_uploader.rb - -require 'thread' -require 'set' -require 'tempfile' -require 'stringio' - -module Aws - module S3 - # @api private - class MultipartStreamUploader - # api private - PART_SIZE = 5 * 1024 * 1024 # 5MB - - # api private - THREAD_COUNT = 10 - - # api private - TEMPFILE_PREIX = 'aws-sdk-s3-upload_stream'.freeze - - # @api private - CREATE_OPTIONS = - Set.new(Client.api.operation(:create_multipart_upload).input.shape.member_names) - - # @api private - UPLOAD_PART_OPTIONS = - Set.new(Client.api.operation(:upload_part).input.shape.member_names) - - # @option options [Client] :client - def initialize(options = {}) - @client = options[:client] || Client.new - @tempfile = options[:tempfile] - @part_size = options[:part_size] || PART_SIZE - @thread_count = options[:thread_count] || THREAD_COUNT - end - - # @return [Client] - attr_reader :client - - # @option options [required,String] :bucket - # @option options [required,String] :key - # @return [void] - def upload(options = {}, &block) - upload_id = initiate_upload(options) - parts = upload_parts(upload_id, options, &block) - complete_upload(upload_id, parts, options) - end - - private - - def initiate_upload(options) - @client.create_multipart_upload(create_opts(options)).upload_id - end - - def complete_upload(upload_id, parts, options) - @client.complete_multipart_upload( - bucket: options[:bucket], - key: options[:key], - upload_id: upload_id, - multipart_upload: { parts: parts }) - end - - def upload_parts(upload_id, options, &block) - completed = Queue.new - errors = IO.pipe do |read_pipe, write_pipe| - threads = upload_in_threads(read_pipe, completed, upload_part_opts(options).merge(upload_id: upload_id)) - block.call(write_pipe) - write_pipe.close - threads.map(&:value).compact - end - if errors.empty? - Array.new(completed.size) { completed.pop }.sort_by { |part| part[:part_number] } - else - abort_upload(upload_id, options, errors) - end - end - - def abort_upload(upload_id, options, errors) - @client.abort_multipart_upload( - bucket: options[:bucket], - key: options[:key], - upload_id: upload_id - ) - msg = "multipart upload failed: #{errors.map(&:message).join("; ")}" - raise MultipartUploadError.new(msg, errors) - rescue MultipartUploadError => error - raise error - rescue => error - msg = "failed to abort multipart upload: #{error.message}" - raise MultipartUploadError.new(msg, errors + [error]) - end - - def create_opts(options) - CREATE_OPTIONS.inject({}) do |hash, key| - hash[key] = options[key] if options.key?(key) - hash - end - end - - def upload_part_opts(options) - UPLOAD_PART_OPTIONS.inject({}) do |hash, key| - hash[key] = options[key] if options.key?(key) - hash - end - end - - def read_to_part_body(read_pipe) - return if read_pipe.closed? - temp_io = @tempfile ? Tempfile.new(TEMPFILE_PREIX) : StringIO.new - temp_io.binmode - bytes_copied = IO.copy_stream(read_pipe, temp_io, @part_size) - temp_io.rewind - if bytes_copied == 0 - if Tempfile === temp_io - temp_io.close - temp_io.unlink - end - nil - else - temp_io - end - end - - def upload_in_threads(read_pipe, completed, options) - mutex = Mutex.new - part_number = 0 - @thread_count.times.map do - thread = Thread.new do - begin - loop do - body, thread_part_number = mutex.synchronize do - [read_to_part_body(read_pipe), part_number += 1] - end - break unless body - begin - part = options.merge( - body: body, - part_number: thread_part_number, - ) - resp = @client.upload_part(part) - completed << {etag: resp.etag, part_number: part[:part_number]} - ensure - if Tempfile === body - body.close - body.unlink - end - end - end - nil - rescue => error - # keep other threads from uploading other parts - mutex.synchronize { read_pipe.close_read } - error - end - end - thread.abort_on_exception = true - thread - end - end - end - end -end - -### Aws::S3::Object patches -# -# Customizations ripped from: -# -# https://github.com/aws/aws-sdk-ruby/blob/708140b8/gems/aws-sdk-s3/lib/aws-sdk-s3/multipart_stream_uploader.rb -# -# Only the Aws::S3::Object#upload_stream method is backported, which is a -# wrapper method around the above. This is an attempt to reduce the monkey -# patching surface area. -# - -module Aws - module S3 - class Object - # Unsure if this might have been used by the uploaded above, so I kept it - # in for now. - alias size content_length - - # Uploads a stream in a streaming fashion to the current object in S3. - # - # # Passed chunks automatically split into multipart upload parts - # # and the parts are uploaded in parallel. This allows for streaming uploads - # # that never touch the disk. - # - # Note that this is known to have issues in JRuby until jruby-9.1.15.0, so avoid using this with older versions of JRuby. - # - # @example Streaming chunks of data - # obj.upload_stream do |write_stream| - # 10.times { write_stream << 'foo' } - # end - # @example Streaming chunks of data - # obj.upload_stream do |write_stream| - # IO.copy_stream(IO.popen('ls'), write_stream) - # end - # @example Streaming chunks of data - # obj.upload_stream do |write_stream| - # IO.copy_stream(STDIN, write_stream) - # end - # - # @option options [Integer] :thread_count - # The number of parallel multipart uploads - # Default `:thread_count` is `10`. - # - # @option options [Boolean] :tempfile - # Normally read data is stored in memory when building the parts in order to complete - # the underlying multipart upload. By passing `:tempfile => true` data read will be - # temporarily stored on disk reducing the memory footprint vastly. - # Default `:tempfile` is `false`. - # - # @option options [Integer] :part_size - # Define how big each part size but the last should be. - # Default `:part_size` is `5 * 1024 * 1024`. - # - # @raise [MultipartUploadError] If an object is being uploaded in - # parts, and the upload can not be completed, then the upload is - # aborted and this error is raised. The raised error has a `#errors` - # method that returns the failures that caused the upload to be - # aborted. - # - # @return [Boolean] Returns `true` when the object is uploaded - # without any errors. - # - def upload_stream(options = {}, &block) - uploading_options = options.dup - uploader = MultipartStreamUploader.new( - client: client, - thread_count: uploading_options.delete(:thread_count), - tempfile: uploading_options.delete(:tempfile), - part_size: uploading_options.delete(:part_size), - ) - uploader.upload(uploading_options.merge(bucket: bucket_name, key: key), &block) - true - end - end - end -end -# rubocop:enable all From 2b213ca6b68c694f0534080ef9f125e5bb5344d7 Mon Sep 17 00:00:00 2001 From: Alexander Zagaynov Date: Fri, 11 Oct 2019 13:35:56 +0200 Subject: [PATCH 3/4] more precise aws sdk version --- manageiq-gems-pending.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manageiq-gems-pending.gemspec b/manageiq-gems-pending.gemspec index 3cf2efbb2..7bb6d050b 100644 --- a/manageiq-gems-pending.gemspec +++ b/manageiq-gems-pending.gemspec @@ -23,7 +23,7 @@ Gem::Specification.new do |s| s.add_runtime_dependency "activerecord", "< 5.3" s.add_runtime_dependency "activesupport", "< 5.3" s.add_runtime_dependency "awesome_spawn", "~> 1.4" - s.add_runtime_dependency "aws-sdk", "~> 3.0" + s.add_runtime_dependency "aws-sdk", "~> 3.0.1" s.add_runtime_dependency "binary_struct", "~> 2.1" s.add_runtime_dependency "bundler", ">= 1.8.4" # rails-assets requires bundler >= 1.8.4, see: https://rails-assets.org/ s.add_runtime_dependency "fog-openstack", "~> 0.3" From f0a5d5a8dbfe889ad4d3a0cec56d8180cf7c61cf Mon Sep 17 00:00:00 2001 From: Alexander Zagaynov Date: Thu, 17 Oct 2019 14:49:04 +0200 Subject: [PATCH 4/4] fix require statements --- lib/gems/pending/util/object_storage/miq_s3_storage.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/gems/pending/util/object_storage/miq_s3_storage.rb b/lib/gems/pending/util/object_storage/miq_s3_storage.rb index be43f9662..63696d4c6 100644 --- a/lib/gems/pending/util/object_storage/miq_s3_storage.rb +++ b/lib/gems/pending/util/object_storage/miq_s3_storage.rb @@ -84,8 +84,8 @@ def bucket private def s3 - require 'aws-sdk' - require 'util/extensions/aws-sdk/s3_upload_stream_patch' + require 'aws-sdk-s3' + @s3 ||= Aws::S3::Resource.new(:region => @settings[:region], :access_key_id => @settings[:username], :secret_access_key => @settings[:password])