Add file splitting support to evm_dba tasks

Adds the `--byte-count` flag to evm:db:dump:* and evm:db:backup:* rake
tasks which passes the `:byte_count` option to the MiqFileStorage for
handling file splitting.

Works with all classes that conform to the MiqFileStorage interface.

lib/evm_database_ops.rb

@@ -104,7 +104,7 @@ def self.restore(db_opts, connect_opts = {})
     DEFAULT_OPTS.merge(db_opts)
   end
 
-  STORAGE_ACTIONS_TO_METHODS = { :backup => :upload, :dump => :upload, :restore => :download }.freeze
+  STORAGE_ACTIONS_TO_METHODS = { :backup => :add, :dump => :add, :restore => :download }.freeze
   private_class_method def self.with_file_storage(action, db_opts, connect_opts)
     db_opts      = merged_db_opts(db_opts)
     connect_opts = connect_opts.reverse_merge(:uri => "file://")
@@ -123,8 +123,8 @@ def self.restore(db_opts, connect_opts = {})
     end
 
     MiqFileStorage.with_interface_class(connect_opts) do |file_storage|
-      file_storage.connect
-      file_storage.send(STORAGE_ACTIONS_TO_METHODS[action], uri) do |input_path|
+      send_args = [uri, db_opts[:byte_count]].compact
+      file_storage.send(STORAGE_ACTIONS_TO_METHODS[action], *send_args) do |input_path|
         db_opts[:local_file] = input_path
         yield(db_opts)
       end

lib/tasks/evm_dba.rake

@@ -34,6 +34,8 @@ module EvmDba
           opt :local_file,         "Destination file",             :type => :string, :required => true
         when :remote_file
           opt :remote_file_name,   "Destination depot filename",   :type => :string
+        when :splitable
+          opt :byte_count,         "Size to split files into",     :type => :string
         when :remote_uri
           opt :uri,                "Destination depot URI",        :type => :string, :required => true
           opt :uri_username,       "Destination depot username",   :type => :string
@@ -48,7 +50,7 @@ module EvmDba
     end.delete_nils
   end
 
-  DB_OPT_KEYS = [:dbname, :username, :password, :hostname, :exclude_table_data].freeze
+  DB_OPT_KEYS = [:dbname, :username, :password, :hostname, :exclude_table_data, :byte_count].freeze
   def self.collect_db_opts(opts)
     db_opts = {}
     DB_OPT_KEYS.each { |k| db_opts[k] = opts[k] if opts[k] }
@@ -189,7 +191,7 @@ namespace :evm do
       require File.expand_path(File.join(Rails.root, "lib", "evm_database_ops"))
       desc 'Backup the local ManageIQ EVM Database (VMDB) to a local file'
       task :local do
-        opts = EvmDba.with_options(:local_file, :db_credentials)
+        opts = EvmDba.with_options(:local_file, :splitable, :db_credentials)
 
         EvmDatabaseOps.backup(opts)
 
@@ -198,7 +200,7 @@ namespace :evm do
 
       desc 'Backup the local ManageIQ EVM Database (VMDB) to a remote file'
       task :remote do
-        opts = EvmDba.with_options(:remote_uri, :aws, :remote_file, :db_credentials)
+        opts = EvmDba.with_options(:remote_uri, :aws, :remote_file, :splitable, :db_credentials)
 
         db_opts      = EvmDba.collect_db_opts(opts)
         connect_opts = EvmDba.collect_connect_opts(opts)
@@ -213,7 +215,7 @@ namespace :evm do
       require Rails.root.join("lib", "evm_database_ops").expand_path.to_s
       desc 'Dump the local ManageIQ EVM Database (VMDB) to a local file'
       task :local do
-        opts = EvmDba.with_options(:local_file, :db_credentials, :exclude_table_data)
+        opts = EvmDba.with_options(:local_file, :splitable, :db_credentials, :exclude_table_data)
 
         EvmDatabaseOps.dump(opts)
 
@@ -222,7 +224,7 @@ namespace :evm do
 
       desc 'Dump the local ManageIQ EVM Database (VMDB) to a remote file'
       task :remote do
-        opts = EvmDba.with_options(:remote_uri, :remote_file, :db_credentials, :exclude_table_data)
+        opts = EvmDba.with_options(:remote_uri, :remote_file, :splitable, :db_credentials, :exclude_table_data)
 
         db_opts      = EvmDba.collect_db_opts(opts)
         connect_opts = EvmDba.collect_connect_opts(opts)

spec/lib/evm_database_ops_spec.rb

@@ -1,6 +1,7 @@
 require 'util/runcmd'
 describe EvmDatabaseOps do
   let(:file_storage) { double("MiqSmbSession", :disconnect => nil) }
+  let(:local_backup) { "/tmp/backup_1" }
   let(:input_path)   { "foo/bar/mkfifo" }
   let(:run_db_ops)   { @db_opts.dup.merge(:local_file => input_path) }
   let(:tmpdir)       { Rails.root.join("tmp") }
@@ -23,19 +24,26 @@
     end
 
     it "locally" do
-      local_backup = "/tmp/backup_1"
       @db_opts[:local_file] = local_backup
       expect(PostgresAdmin).to receive(:backup).with(run_db_ops)
       expect(EvmDatabaseOps.backup(@db_opts, @connect_opts)).to eq(local_backup)
     end
 
     it "defaults" do
-      local_backup = "/tmp/backup_1"
       @db_opts[:local_file] = local_backup
       expect(PostgresAdmin).to receive(:backup).with(run_db_ops)
       expect(EvmDatabaseOps.backup(@db_opts, {})).to eq(local_backup)
     end
 
+    it "splits files with a local file" do
+      @db_opts[:local_file] = local_backup
+      @db_opts[:byte_count] = "200M"
+
+      allow(file_storage).to   receive(:send).with(:add, local_backup, "200M").and_yield(input_path)
+      expect(PostgresAdmin).to receive(:backup).with(run_db_ops)
+      EvmDatabaseOps.backup(@db_opts, {})
+    end
+
     it "without enough free space" do
       EvmSpecHelper.create_guid_miq_server_zone
       allow(EvmDatabaseOps).to receive(:backup_destination_free_space).and_return(100.megabytes)
@@ -76,6 +84,7 @@
   end
 
   context "#dump" do
+    let(:local_dump) { "/tmp/dump_1" }
     before do
       @connect_opts = {:username => 'blah', :password => 'blahblah', :uri => "smb://myserver.com/share"}
       @db_opts =      {:dbname => 'vmdb_production', :username => 'root'}
@@ -91,19 +100,26 @@
     end
 
     it "locally" do
-      local_dump = "/tmp/dump_1"
       @db_opts[:local_file] = local_dump
       expect(PostgresAdmin).to receive(:backup_pg_dump).with(run_db_ops)
       expect(EvmDatabaseOps.dump(@db_opts, @connect_opts)).to eq(local_dump)
     end
 
     it "defaults" do
-      local_dump = "/tmp/dump_1"
       @db_opts[:local_file] = local_dump
       expect(PostgresAdmin).to receive(:backup_pg_dump).with(run_db_ops)
       expect(EvmDatabaseOps.dump(@db_opts, {})).to eq(local_dump)
     end
 
+    it "splits files with a local file" do
+      @db_opts[:local_file] = local_dump
+      @db_opts[:byte_count] = "200M"
+
+      allow(file_storage).to   receive(:send).with(:add, local_dump, "200M").and_yield(input_path)
+      expect(PostgresAdmin).to receive(:backup_pg_dump).with(run_db_ops)
+      EvmDatabaseOps.dump(@db_opts, {})
+    end
+
     it "without enough free space" do
       EvmSpecHelper.create_guid_miq_server_zone
       allow(EvmDatabaseOps).to receive(:backup_destination_free_space).and_return(100.megabytes)
@@ -146,7 +162,6 @@
     end
 
     it "from local backup" do
-      local_backup = "/tmp/backup_1"
       @db_opts[:local_file] = local_backup
       expect(EvmDatabaseOps.restore(@db_opts, @connect_opts)).to eq(local_backup)
     end