DB Backups to AWS S3

app/models/database_backup.rb

@@ -1,7 +1,8 @@
 class DatabaseBackup < ApplicationRecord
   SUPPORTED_DEPOTS = {
     'smb' => 'Samba',
-    'nfs' => 'Network File System'
+    'nfs' => 'Network File System',
+    's3'  => 'AWS S3'
   }.freeze
 
   def self.supported_depots
@@ -32,7 +33,7 @@ def backup(options)
     options[:userid] ||= "system"
 
     depot = FileDepot.find_by(:id => options[:file_depot_id])
-    _backup(:uri => depot.uri, :username => depot.authentication_userid, :password => depot.authentication_password, :remote_file_name => backup_file_name)
+    _backup(:uri => depot.uri, :username => depot.authentication_userid, :password => depot.authentication_password, :remote_file_name => backup_file_name, :region => depot.aws_region)
 
     if @sch && @sch.adhoc == true
       _log.info("Removing adhoc schedule: [#{@sch.id}] [#{@sch.name}]")
@@ -46,7 +47,7 @@ def backup(options)
   def _backup(options)
     # add the metadata about this backup to this instance: (region, source hostname, db version, md5, status, etc.)
 
-    connect_opts = options.slice(:uri, :username, :password)
+    connect_opts = options.slice(:uri, :username, :password, :region)
     connect_opts[:remote_file_name] = options[:remote_file_name] if options[:remote_file_name]
     EvmDatabaseOps.backup(current_db_opts, connect_opts)
   end

app/models/file_depot_s3.rb

@@ -0,0 +1,78 @@
+class FileDepotS3 < FileDepot
+  attr_accessor :s3
+
+  def self.uri_prefix
+    "s3"
+  end
+
+  def self.validate_settings(settings)
+    new(:uri => settings[:uri]).verify_credentials(nil, settings.slice(:username, :password))
+  end
+
+  def connect(options = {})
+    require 'aws-sdk'
+    username = options[:username] || authentication_userid(options[:auth_type])
+    password = options[:password] || authentication_password(options[:auth_type])
+    # Note: The hard-coded aws_region will be removed after manageiq-ui-class implements region selection
+    aws_region = options[:region] || "us-east-1"
+
+    Aws::S3::Resource.new(
+      :access_key_id     => username,
+      :secret_access_key => MiqPassword.try_decrypt(password),
+      :region            => aws_region,
+      :logger            => $aws_log,
+      :log_level         => :debug,
+      :log_formatter     => Aws::Log::Formatter.new(Aws::Log::Formatter.default.pattern.chomp)
+    )
+  end
+
+  def with_depot_connection(options = {})
+    raise _("no block given") unless block_given?
+    _log.info("Connecting through #{self.class.name}: [#{name}]")
+    yield connect(options)
+  end
+
+  def verify_credentials(auth_type = nil, options = {})
+    _log.debug("verifying credentials for auth_type [#{auth_type}] with options [#{options}]")
+
+    connection_rescue_block do
+      # EC2 does Lazy Connections, so call a cheap function
+      with_depot_connection(options.merge(:auth_type => auth_type)) do |ec2|
+        validate_connection(ec2)
+      end
+    end
+
+    true
+  end
+
+  def validate_connection(connection)
+    connection_rescue_block do
+      connection.client.list_buckets
+    end
+  end
+
+  def connection_rescue_block
+    yield
+  rescue => err
+    miq_exception = translate_exception(err)
+    raise unless miq_exception
+
+    _log.log_backtrace(err)
+    _log.error("Error Class=#{err.class.name}, Message=#{err.message}")
+    raise miq_exception
+  end
+
+  def translate_exception(err)
+    require 'aws-sdk'
+    case err
+    when Aws::EC2::Errors::SignatureDoesNotMatch
+      MiqException::MiqHostError.new("SignatureMismatch - check your AWS Secret Access Key and signing method")
+    when Aws::EC2::Errors::AuthFailure
+      MiqException::MiqHostError.new("Login failed due to a bad username or password.")
+    when Aws::Errors::MissingCredentialsError
+      MiqException::MiqHostError.new("Missing credentials")
+    else
+      MiqException::MiqHostError.new("Unexpected response returned from system: #{err.message}")
+    end
+  end
+end

app/models/mixins/file_depot_mixin.rb

@@ -5,7 +5,8 @@ module FileDepotMixin
   extend ActiveSupport::Concern
   SUPPORTED_DEPOTS = {
     'smb' => 'Samba',
-    'nfs' => 'Network File System'
+    'nfs' => 'Network File System',
+    's3'  => 'Amazon Web Services'
   }.freeze
 
   included do

lib/evm_database_ops.rb

@@ -53,9 +53,11 @@ def self.backup(db_opts, connect_opts = {})
     #   :password => 'Zug-drep5s',
     #   :remote_file_name => "backup_1",     - Provide a base file name for the uploaded file
 
-    uri = with_mount_session(:backup, db_opts, connect_opts) do |database_opts|
+    uri = with_mount_session(:backup, db_opts, connect_opts) do |database_opts, session, remote_file_uri|
       validate_free_space(database_opts)
-      PostgresAdmin.backup(database_opts)
+      backup_result = PostgresAdmin.backup(database_opts)
+      session&.add(database_opts[:local_file], remote_file_uri)
+      backup_result
     end
     _log.info("[#{merged_db_opts(db_opts)[:dbname]}] database has been backed up to file: [#{uri}]")
     uri
@@ -64,7 +66,7 @@ def self.backup(db_opts, connect_opts = {})
   def self.dump(db_opts, connect_opts = {})
     # db_opts and connect_opts similar to .backup
 
-    uri = with_mount_session(:dump, db_opts, connect_opts) do |database_opts|
+    uri = with_mount_session(:dump, db_opts, connect_opts) do |database_opts, _session, _remote_file_uri|
       # For database dumps, this isn't going to be as accurate (since the dump
       # size will probably be larger than the calculated BD size), but it still
       # won't hurt to do as a generic way to get a rough idea if we have enough
@@ -87,7 +89,7 @@ def self.restore(db_opts, connect_opts = {})
     #   :username => 'samba_one',
     #   :password => 'Zug-drep5s',
 
-    uri = with_mount_session(:restore, db_opts, connect_opts) do |database_opts|
+    uri = with_mount_session(:restore, db_opts, connect_opts) do |database_opts, _session, _remote_file_uri|
       prepare_for_restore(database_opts[:local_file])
 
       # remove all the connections before we restore; AR will reconnect on the next query
@@ -119,7 +121,7 @@ def self.restore(db_opts, connect_opts = {})
       db_opts[:local_file] = session.uri_to_local_path(uri)
     end
 
-    block_result = yield db_opts if block_given?
+    block_result = yield(db_opts, session, uri) if block_given?
     uri || block_result
   ensure
     session.disconnect if session

locale/en.yml

@@ -729,6 +729,7 @@ en:
       FileDepotFtp:             FTP
       FileDepotFtpAnonymous:    Anonymous FTP
       FileDepotNfs:             NFS
+      FileDepotS3:              AWS S3
       FileDepotSmb:             Samba
       Filesystem:               File
       Flavor:                   Flavor

spec/lib/evm_database_ops_spec.rb

@@ -1,12 +1,13 @@
 require 'util/runcmd'
 describe EvmDatabaseOps do
   context "#backup" do
+    let(:session) { double("MiqSmbSession", :disconnect => nil) }
     before do
       @connect_opts = {:username => 'blah', :password => 'blahblah', :uri => "smb://myserver.com/share"}
       @db_opts =      {:dbname => 'vmdb_production', :username => 'root'}
-      allow(MiqSmbSession).to receive(:runcmd)
-      allow_any_instance_of(MiqSmbSession).to receive(:settings_mount_point).and_return(Rails.root.join("tmp"))
-      allow(MiqUtil).to receive(:runcmd)
+      allow(MiqGenericMountSession).to receive(:new_session).and_return(session)
+      allow(session).to receive(:settings_mount_point).and_return(Rails.root.join("tmp").to_s)
+      allow(session).to receive(:uri_to_local_path).and_return(Rails.root.join("tmp/share").to_s)
       allow(PostgresAdmin).to receive(:runcmd_with_logging)
       allow(FileUtils).to receive(:mv).and_return(true)
       allow(EvmDatabaseOps).to receive(:backup_destination_free_space).and_return(200.megabytes)
@@ -36,12 +37,14 @@
     it "remotely" do
       @db_opts[:local_file] = nil
       @connect_opts[:remote_file_name] = "custom_backup"
+      expect(session).to receive(:add).and_return("smb://myserver.com/share/db_backup/custom_backup")
       expect(EvmDatabaseOps.backup(@db_opts, @connect_opts)).to eq("smb://myserver.com/share/db_backup/custom_backup")
     end
 
     it "remotely without a remote file name" do
       @db_opts[:local_file] = nil
       @connect_opts[:remote_file_name] = nil
+      expect(session).to receive(:add)
       expect(EvmDatabaseOps.backup(@db_opts, @connect_opts)).to match(/smb:\/\/myserver.com\/share\/db_backup\/miq_backup_.*/)
     end
 
@@ -55,6 +58,7 @@
       expect(described_class).to receive(:_log).twice.and_return(log_stub)
       expect(log_stub).to        receive(:info).with(any_args)
       expect(log_stub).to        receive(:info).with("[vmdb_production] database has been backed up to file: [smb://myserver.com/share/db_backup/miq_backup]")
+      expect(session).to receive(:add).and_return("smb://myserver.com/share/db_backup/miq_backup")
 
       EvmDatabaseOps.backup(@db_opts, @connect_opts)
     end
@@ -158,7 +162,7 @@
 
     # convenience_wrapper for private method
     def execute_with_mount_session(action = :backup)
-      described_class.send(:with_mount_session, action, db_opts, connect_opts) do |dbopts|
+      described_class.send(:with_mount_session, action, db_opts, connect_opts) do |dbopts, _session, _remote_file_uri|
         yield dbopts if block_given?
       end
     end