Make db restore more reliable

lib/evm_database_ops.rb

@@ -92,16 +92,60 @@ def self.restore(db_opts, connect_opts = {})
         db_opts[:local_file] = session.uri_to_local_path(uri)
       end
 
-      backup = PostgresAdmin.restore(db_opts)
+      prepare_for_restore(db_opts[:local_file])
+
+      # remove all the connections before we restore; AR will reconnect on the next query
+      ActiveRecord::Base.connection_pool.disconnect!
+      backup_file = PostgresAdmin.restore(db_opts)
     ensure
       session.disconnect if session
     end
 
-    uri ||= backup
+    uri ||= backup_file
     _log.info("[#{db_opts[:dbname]}] database has been restored from file: [#{uri}]")
     uri
   end
 
+  private_class_method def self.prepare_for_restore(filename)
+    backup_type = validate_backup_file_type(filename)
+
+    if application_connections?
+      message = "Database restore failed. Shut down all evmserverd processes before attempting a database restore"
+      _log.error(message)
+      raise message
+    end
+
+    MiqRegion.replication_type = :none
+    60.times do
+      break if VmdbDatabaseConnection.where("application_name LIKE 'pglogical manager%'").count.zero?
+      _log.info("Waiting for pglogical connections to close...")
+      sleep 5
+    end
+
+    connection_count = backup_type == :basebackup ? VmdbDatabaseConnection.unscoped.count : VmdbDatabaseConnection.count
+    if connection_count > 1
+      message = "Database restore failed. #{connection_count - 1} connections remain to the database."
+      _log.error(message)
+      raise message
+    end
+  end
+
+  private_class_method def self.validate_backup_file_type(filename)
+    if PostgresAdmin.base_backup_file?(filename)
+      :basebackup
+    elsif PostgresAdmin.pg_dump_file?(filename)
+      :pgdump
+    else
+      message = "#{filename} is not in a recognized database backup format"
+      _log.error(message)
+      raise message
+    end
+  end
+
+  private_class_method def self.application_connections?
+    VmdbDatabaseConnection.all.map(&:application_name).any? { |app_name| app_name.start_with?("MIQ") }
+  end
+
   def self.gc(options = {})
     PostgresAdmin.gc(options)
   end

lib/tasks/evm_dba.rake

@@ -194,7 +194,7 @@ namespace :evm do
 
     namespace :restore do
       desc 'Restore the local ManageIQ EVM Database (VMDB) from a local backup file'
-      task :local do
+      task :local => :environment do
         require 'trollop'
         opts = Trollop.options(EvmRakeHelper.extract_command_options) do
           opt :local_file, "Destination file", :type => :string, :required => true
@@ -215,7 +215,7 @@ namespace :evm do
       end
 
       desc 'Restore the local ManageIQ EVM Database (VMDB) from a remote backup file'
-      task :remote do
+      task :remote => :environment do
         require 'trollop'
         opts = Trollop.options(EvmRakeHelper.extract_command_options) do
           opt :uri,              "Destination depot URI",       :type => :string, :required => true

spec/lib/evm_database_ops_spec.rb

@@ -55,6 +55,7 @@
       allow_any_instance_of(MiqSmbSession).to receive(:settings_mount_point).and_return(Rails.root.join("tmp"))
       allow(PostgresAdmin).to receive(:runcmd_with_logging)
       allow(PostgresAdmin).to receive(:pg_dump_file?).and_return(true)
+      allow(PostgresAdmin).to receive(:base_backup_file?).and_return(false)
     end
 
     it "from local backup" do