From b087e1e9f9cb0e2ccf39ed7be06ecfb9306b4fc2 Mon Sep 17 00:00:00 2001 From: Nick LaMuro Date: Wed, 22 Aug 2018 00:04:09 -0500 Subject: [PATCH] Add file splitting support to evm_dba tasks Adds the `--byte-count` flag to evm:db:dump:* and evm:db:backup:* rake tasks which passes the `:byte_count` option to the MiqFileStorage for handling file splitting. Works with all classes that conform to the MiqFileStorage interface. --- lib/evm_database_ops.rb | 6 +++--- lib/tasks/evm_dba.rake | 12 +++++++----- spec/lib/evm_database_ops_spec.rb | 25 ++++++++++++++++++++----- 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/lib/evm_database_ops.rb b/lib/evm_database_ops.rb index d850c51b1854..063aea38c3f6 100644 --- a/lib/evm_database_ops.rb +++ b/lib/evm_database_ops.rb @@ -104,7 +104,7 @@ def self.restore(db_opts, connect_opts = {}) DEFAULT_OPTS.merge(db_opts) end - STORAGE_ACTIONS_TO_METHODS = { :backup => :upload, :dump => :upload, :restore => :download }.freeze + STORAGE_ACTIONS_TO_METHODS = { :backup => :add, :dump => :add, :restore => :download }.freeze private_class_method def self.with_file_storage(action, db_opts, connect_opts) db_opts = merged_db_opts(db_opts) connect_opts = connect_opts.reverse_merge(:uri => "file://") @@ -123,8 +123,8 @@ def self.restore(db_opts, connect_opts = {}) end MiqFileStorage.with_interface_class(connect_opts) do |file_storage| - file_storage.connect - file_storage.send(STORAGE_ACTIONS_TO_METHODS[action], uri) do |input_path| + send_args = [uri, db_opts[:byte_count]].compact + file_storage.send(STORAGE_ACTIONS_TO_METHODS[action], *send_args) do |input_path| db_opts[:local_file] = input_path yield(db_opts) end diff --git a/lib/tasks/evm_dba.rake b/lib/tasks/evm_dba.rake index 315726a4e24e..9193f7f0e846 100644 --- a/lib/tasks/evm_dba.rake +++ b/lib/tasks/evm_dba.rake @@ -34,6 +34,8 @@ module EvmDba opt :local_file, "Destination file", :type => :string, :required => true when :remote_file opt :remote_file_name, "Destination depot filename", :type => :string + when :splitable + opt :byte_count, "Size to split files into", :type => :string when :remote_uri opt :uri, "Destination depot URI", :type => :string, :required => true opt :uri_username, "Destination depot username", :type => :string @@ -48,7 +50,7 @@ module EvmDba end.delete_nils end - DB_OPT_KEYS = [:dbname, :username, :password, :hostname, :exclude_table_data].freeze + DB_OPT_KEYS = [:dbname, :username, :password, :hostname, :exclude_table_data, :byte_count].freeze def self.collect_db_opts(opts) db_opts = {} DB_OPT_KEYS.each { |k| db_opts[k] = opts[k] if opts[k] } @@ -189,7 +191,7 @@ namespace :evm do require File.expand_path(File.join(Rails.root, "lib", "evm_database_ops")) desc 'Backup the local ManageIQ EVM Database (VMDB) to a local file' task :local do - opts = EvmDba.with_options(:local_file, :db_credentials) + opts = EvmDba.with_options(:local_file, :splitable, :db_credentials) EvmDatabaseOps.backup(opts) @@ -198,7 +200,7 @@ namespace :evm do desc 'Backup the local ManageIQ EVM Database (VMDB) to a remote file' task :remote do - opts = EvmDba.with_options(:remote_uri, :aws, :remote_file, :db_credentials) + opts = EvmDba.with_options(:remote_uri, :aws, :remote_file, :splitable, :db_credentials) db_opts = EvmDba.collect_db_opts(opts) connect_opts = EvmDba.collect_connect_opts(opts) @@ -213,7 +215,7 @@ namespace :evm do require Rails.root.join("lib", "evm_database_ops").expand_path.to_s desc 'Dump the local ManageIQ EVM Database (VMDB) to a local file' task :local do - opts = EvmDba.with_options(:local_file, :db_credentials, :exclude_table_data) + opts = EvmDba.with_options(:local_file, :splitable, :db_credentials, :exclude_table_data) EvmDatabaseOps.dump(opts) @@ -222,7 +224,7 @@ namespace :evm do desc 'Dump the local ManageIQ EVM Database (VMDB) to a remote file' task :remote do - opts = EvmDba.with_options(:remote_uri, :remote_file, :db_credentials, :exclude_table_data) + opts = EvmDba.with_options(:remote_uri, :remote_file, :splitable, :db_credentials, :exclude_table_data) db_opts = EvmDba.collect_db_opts(opts) connect_opts = EvmDba.collect_connect_opts(opts) diff --git a/spec/lib/evm_database_ops_spec.rb b/spec/lib/evm_database_ops_spec.rb index fcbe240752a2..b6e933e3fbc4 100644 --- a/spec/lib/evm_database_ops_spec.rb +++ b/spec/lib/evm_database_ops_spec.rb @@ -1,6 +1,7 @@ require 'util/runcmd' describe EvmDatabaseOps do let(:file_storage) { double("MiqSmbSession", :disconnect => nil) } + let(:local_backup) { "/tmp/backup_1" } let(:input_path) { "foo/bar/mkfifo" } let(:run_db_ops) { @db_opts.dup.merge(:local_file => input_path) } let(:tmpdir) { Rails.root.join("tmp") } @@ -23,19 +24,26 @@ end it "locally" do - local_backup = "/tmp/backup_1" @db_opts[:local_file] = local_backup expect(PostgresAdmin).to receive(:backup).with(run_db_ops) expect(EvmDatabaseOps.backup(@db_opts, @connect_opts)).to eq(local_backup) end it "defaults" do - local_backup = "/tmp/backup_1" @db_opts[:local_file] = local_backup expect(PostgresAdmin).to receive(:backup).with(run_db_ops) expect(EvmDatabaseOps.backup(@db_opts, {})).to eq(local_backup) end + it "splits files with a local file" do + @db_opts[:local_file] = local_backup + @db_opts[:byte_count] = "200M" + + allow(file_storage).to receive(:send).with(:add, local_backup, "200M").and_yield(input_path) + expect(PostgresAdmin).to receive(:backup).with(run_db_ops) + EvmDatabaseOps.backup(@db_opts, {}) + end + it "without enough free space" do EvmSpecHelper.create_guid_miq_server_zone allow(EvmDatabaseOps).to receive(:backup_destination_free_space).and_return(100.megabytes) @@ -76,6 +84,7 @@ end context "#dump" do + let(:local_dump) { "/tmp/dump_1" } before do @connect_opts = {:username => 'blah', :password => 'blahblah', :uri => "smb://myserver.com/share"} @db_opts = {:dbname => 'vmdb_production', :username => 'root'} @@ -91,19 +100,26 @@ end it "locally" do - local_dump = "/tmp/dump_1" @db_opts[:local_file] = local_dump expect(PostgresAdmin).to receive(:backup_pg_dump).with(run_db_ops) expect(EvmDatabaseOps.dump(@db_opts, @connect_opts)).to eq(local_dump) end it "defaults" do - local_dump = "/tmp/dump_1" @db_opts[:local_file] = local_dump expect(PostgresAdmin).to receive(:backup_pg_dump).with(run_db_ops) expect(EvmDatabaseOps.dump(@db_opts, {})).to eq(local_dump) end + it "splits files with a local file" do + @db_opts[:local_file] = local_dump + @db_opts[:byte_count] = "200M" + + allow(file_storage).to receive(:send).with(:add, local_dump, "200M").and_yield(input_path) + expect(PostgresAdmin).to receive(:backup_pg_dump).with(run_db_ops) + EvmDatabaseOps.dump(@db_opts, {}) + end + it "without enough free space" do EvmSpecHelper.create_guid_miq_server_zone allow(EvmDatabaseOps).to receive(:backup_destination_free_space).and_return(100.megabytes) @@ -146,7 +162,6 @@ end it "from local backup" do - local_backup = "/tmp/backup_1" @db_opts[:local_file] = local_backup expect(EvmDatabaseOps.restore(@db_opts, @connect_opts)).to eq(local_backup) end