Skip to content

Commit

Permalink
Revert ManageIQ#19696
Browse files Browse the repository at this point in the history
  • Loading branch information
@fabiendupont
fabiendupont committed May 23, 2020
1 parent f92f692 commit 14cde9e
Show file tree
Hide file tree
Showing 7 changed files with 107 additions and 225 deletions.
103 changes: 20 additions & 83 deletions app/models/conversion_host.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,50 +159,14 @@ def ipaddress(family = 'ipv4')
# @raise [Net::SSH::HostKeyMismatch] if conversion host key has changed
# @raise [JSON::GeneratorError] if limits hash can't be converted to JSON
# @raise [StandardError] if any other problem happens
def apply_task_limits(task_id, limits = {})
connect_ssh do |ssu|
ssu.put_file("/tmp/#{task_id}-limits.json", limits.to_json)
command = AwesomeSpawn.build_command_line("mv", ["/tmp/#{task_id}-limits.json", "/var/lib/uci/#{task_id}/limits.json"])
ssu.shell_exec(command, nil, nil, nil)
end
def apply_task_limits(path, limits = {})
connect_ssh { |ssu| ssu.put_file(path, limits.to_json) }
rescue Net::SSH::AuthenticationFailed, Net::SSH::HostKeyMismatch => err
raise "Failed to connect and apply limits for task '#{task_id}' with [#{err.class}: #{err}]"
raise "Failed to connect and apply limits in file '#{path}' with [#{err.class}: #{err}]"
rescue JSON::GeneratorError => err
raise "Could not generate JSON from limits '#{limits}' with [#{err.class}: #{err}]"
rescue => err
raise "Could not apply the limits for task '#{task_id}' on '#{resource.name}' with [#{err.class}: #{err}]"
end

# Prepare the conversion assets for a specific task.
#
# @param [Integer] id of the task that needs the preparation
# @param [Hash] conversion options to write on the conversion host
#
# @return [Integer] length of data written to conversion options file
#
# @raise [Net::SSH::AuthenticationFailed] if conversion host credentials are invalid
# @raise [Net::SSH::HostKeyMismatch] if conversion host key has changed
# @raise [JSON::GeneratorError] if limits hash can't be converted to JSON
# @raise [StandardError] if any other problem happens
def prepare_conversion(task_id, conversion_options)
filtered_options = filter_options(conversion_options)

connect_ssh do |ssu|
# Prepare the conversion folders
command = AwesomeSpawn.build_command_line("mkdir", [:p, "/var/lib/uci/#{task_id}", "/var/log/uci/#{task_id}"])
ssu.shell_exec(command, nil, nil, nil)

# Write the conversion options file
ssu.put_file("/tmp/#{task_id}-input.json", conversion_options.to_json)
command = AwesomeSpawn.build_command_line("mv", ["/tmp/#{task_id}-input.json", "/var/lib/uci/#{task_id}/input.json"])
ssu.shell_exec(command, nil, nil, nil)
end
rescue Net::SSH::AuthenticationFailed, Net::SSH::HostKeyMismatch => err
raise "Failed to connect and prepare conversion for task '#{task_id}' with [#{err.class}: #{err}]"
rescue JSON::GeneratorError => err
raise "Could not generate JSON for task '#{task_id}' from options '#{filtered_options}' with [#{err.class}: #{err}]"
rescue => err
raise "Preparation of conversion for task '#{task_id}' failed on '#{resource.name}' with [#{err.class}: #{err}]"
raise "Could not apply the limits in file '#{path}' on '#{resource.name}' with [#{err.class}: #{err}]"
end

# Checks that LUKS keys vault exists and is valid JSON
Expand All @@ -225,55 +189,28 @@ def luks_keys_vault_valid?
false
end

# Build the podman command to execute conversion
#
# @param [Integer] id of the task that conversion applies to
#
# @return [String] podman command to be executed on conversion host
def build_podman_command(task_id, conversion_options)
uci_settings = Settings.transformation.uci.container
uci_image = uci_settings.image
uci_image = "#{uci_settings.registry}/#{image}" if uci_settings.registry.present?

params = [
"run",
:detach,
:privileged,
[:name, "conversion-#{task_id}"],
[:network, "host"],
[:volume, "/dev:/dev"],
[:volume, "/etc/pki/ca-trust:/etc/pki/ca-trust"],
[:volume, "/var/tmp:/var/tmp"],
[:volume, "/var/lib/uci/#{task_id}:/var/lib/uci"],
[:volume, "/var/log/uci/#{task_id}:/var/log/uci"],
[:volume, "/opt/vmware-vix-disklib-distrib:/opt/vmware-vix-disklib-distrib"]
]
params << [:volume, "/root/.ssh/id_rsa:/var/lib/uci/ssh_private_key"] if conversion_options[:transport_method] == 'ssh'
params << [:volume, "/root/.v2v_luks_keys_vault.json:/var/lib/uci/luks_keys_vault.json"] if luks_keys_vault_valid?
params << uci_image

AwesomeSpawn.build_command_line("/usr/bin/podman", params)
end

# Run the virt-v2v-wrapper script on the remote host and return a hash
# result from the parsed JSON output.
#
# Certain sensitive fields are filtered in the error messages to prevent
# that information from showing up in the UI or logs.
#
# @param [Integer] id of the task that conversion applies to
def run_conversion(task_id, conversion_options)
def run_conversion(conversion_options)
filtered_options = filter_options(conversion_options)
prepare_conversion(task_id, conversion_options)
connect_ssh { |ssu| ssu.shell_exec(build_podman_command(task_id, conversion_options), nil, nil, nil) }
command = AwesomeSpawn.build_command_line("/usr/bin/virt-v2v-wrapper")
result = connect_ssh { |ssu| ssu.shell_exec(command, nil, nil, conversion_options.to_json) }
JSON.parse(result)
rescue Net::SSH::AuthenticationFailed, Net::SSH::HostKeyMismatch => err
raise "Failed to connect and run conversion using options #{filtered_options} with [#{err.class}: #{err}]"
rescue JSON::ParserError
raise "Could not parse result data after running virt-v2v-wrapper using options: #{filtered_options}. Result was: #{result}"
rescue => err
raise "Starting conversion for task '#{task_id}' failed on '#{resource.name}' with [#{err.class}: #{err}]"
raise "Starting conversion failed on '#{resource.name}' with [#{err.class}: #{err}]"
end

def create_cutover_file(task_id)
command = AwesomeSpawn.build_command_line("touch", ["/var/lib/uci/#{task_id}/cutover"])
def create_cutover_file(path)
command = AwesomeSpawn.build_command_line("touch", [path])
connect_ssh { |ssu| ssu.shell_exec(command, nil, nil, nil) }
true
rescue
Expand All @@ -283,8 +220,8 @@ def create_cutover_file(task_id)
# Kill a specific remote process over ssh, sending the specified +signal+, or 'TERM'
# if no signal is specified.
#
def kill_virtv2v(task_id, signal)
command = AwesomeSpawn.build_command_line("/usr/bin/podman", ["exec", "conversion-#{task_id}", "/usr/bin/killall", :s, signal, "virt-v2v"])
def kill_virtv2v(pid, signal = 'TERM')
command = AwesomeSpawn.build_command_line("/bin/kill", [:s, signal, pid])
connect_ssh { |ssu| ssu.shell_exec(command, nil, nil, nil) }
true
rescue
Expand All @@ -294,15 +231,15 @@ def kill_virtv2v(task_id, signal)
# Retrieve the conversion state information from a remote file as a stream.
# Then parse and return the stream data as a hash using JSON.parse.
#
def get_conversion_state(task_id)
json_state = connect_ssh { |ssu| ssu.get_file("/var/lib/uci/#{task_id}/state.json", nil) }
def get_conversion_state(path)
json_state = connect_ssh { |ssu| ssu.get_file(path, nil) }
JSON.parse(json_state)
rescue Net::SSH::AuthenticationFailed, Net::SSH::HostKeyMismatch => err
raise "Failed to connect and retrieve conversion state data from file '/var/lib/uci/#{task_id}/state.json' with [#{err.class}: #{err}]"
raise "Failed to connect and retrieve conversion state data from file '#{path}' with [#{err.class}: #{err}]"
rescue JSON::ParserError
raise "Could not parse conversion state data from file '/var/lib/uci/#{task_id}/state.json': #{json_state}"
raise "Could not parse conversion state data from file '#{path}': #{json_state}"
rescue => err
raise "Error retrieving and parsing conversion state file '/var/lib/uci/#{task_id}/state.json' from '#{resource.name}' with [#{err.class}: #{err}"
raise "Error retrieving and parsing conversion state file '#{path}' from '#{resource.name}' with [#{err.class}: #{err}"
end

# Get and return the contents of the remote conversion log at +path+.
Expand Down
36 changes: 17 additions & 19 deletions app/models/service_template_transformation_plan_task.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -237,14 +237,7 @@ def update_options(opts)
def run_conversion
start_timestamp = Time.now.utc.strftime('%Y-%m-%d %H:%M:%S')
updates = {}
conversion_host.run_conversion(id, conversion_options)
updates[:virtv2v_wrapper] = {
"state_file" => "/var/lib/uci/#{id}/state.json",
"throttling_file" => "/var/lib/uci/#{id}/limits.json",
"cutover_file" => "/var/lib/uci/#{id}/cutover",
"v2v_log" => "/var/log/uci/#{id}/virt-v2v.log",
"wrapper_log" => "/var/log/uci/#{id}/virt-v2v-wrapper.log"
}
updates[:virtv2v_wrapper] = conversion_host.run_conversion(conversion_options)
updates[:virtv2v_started_on] = start_timestamp
updates[:virtv2v_status] = 'active'
_log.info("InfraConversionJob run_conversion to update_options: #{updates}")
Expand All @@ -253,7 +246,7 @@ def run_conversion

def get_conversion_state
updates = {}
virtv2v_state = conversion_host.get_conversion_state(id)
virtv2v_state = conversion_host.get_conversion_state(options[:virtv2v_wrapper]['state_file'])
updated_disks = virtv2v_disks
updates[:virtv2v_pid] = virtv2v_state['pid'] if virtv2v_state['pid'].present?
updates[:virtv2v_message] = virtv2v_state['last_message']['message'] if virtv2v_state['last_message'].present?
Expand Down Expand Up @@ -287,23 +280,30 @@ def get_conversion_state
end

def cutover
unless conversion_host.create_cutover_file(id)
raise _("Couldn't create cutover file for #{source.name} on #{conversion_host.name}")
if options[:virtv2v_wrapper]['cutover_file'].present?
unless conversion_host.create_cutover_file(options[:virtv2v_wrapper]['cutover_file'])
raise _("Couldn't create cutover file for #{source.name} on #{conversion_host.name}")
end
end
end

def kill_virtv2v
def kill_virtv2v(signal = 'TERM')
get_conversion_state

unless virtv2v_running?
_log.info("virt-v2v is not running, so there is nothing to do.")
return false
end

_log.info("Killing conversion pod for task '#{id}'.")
conversion_host.kill_virtv2v(id)
unless options[:virtv2v_pid]
_log.info("No PID has been reported by virt-v2v-wrapper, so we can't kill it.")
return false
end

_log.info("Killing virt-v2v (PID: #{options[:virtv2v_pid]}) with #{signal} signal.")
conversion_host.kill_virtv2v(options[:virtv2v_pid], signal)
rescue => err
_log.error("Couldn't kill conversion pod for task '#{id}': #{err.message}")
_log.error("Couldn't kill virt-v2v (PID: #{options[:virtv2v_pid]}) with #{signal} signal: #{err.message}")
update_options(:virtv2v_finished_on => Time.now.utc.strftime('%Y-%m-%d %H:%M:%S'))
false
end
Expand Down Expand Up @@ -372,8 +372,7 @@ def conversion_options_source_provider_vmwarews_vddk(_storage)
).to_s,
:vmware_password => source.host.authentication_password,
:two_phase => true,
:warm => warm_migration?,
:daemonize => false
:warm => warm_migration?
}
end

Expand All @@ -387,8 +386,7 @@ def conversion_options_source_provider_vmwarews_ssh(storage)
).to_s,
:vm_uuid => source.uid_ems,
:conversion_host_uuid => conversion_host.resource.ems_ref,
:transport_method => 'ssh',
:daemonize => false
:transport_method => 'ssh'
}
end

Expand Down
4 changes: 0 additions & 4 deletions config/settings.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1047,10 +1047,6 @@
:max_concurrent_tasks_per_conversion_host: 10
:cpu_limit_per_host: unlimited
:network_limit_per_host: unlimited
:uci:
:container:
:registry:
:image: manageiq/v2v-conversion-host:latest
:ui:
:mark_translated_strings: false
:url:
Expand Down
3 changes: 2 additions & 1 deletion lib/infra_conversion_throttler.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,13 +97,14 @@ def self.apply_limits
jobs.each do |job|
migration_task = job.migration_task
next unless migration_task.virtv2v_running?
next unless migration_task.options.fetch_path(:virtv2v_wrapper, 'throttling_file')

limits = {
:cpu => cpu_limit,
:network => network_limit
}
unless migration_task.options[:virtv2v_limits] == limits
ch.apply_task_limits(migration_task.id, limits)
ch.apply_task_limits(migration_task.options.fetch_path(:virtv2v_wrapper, 'throttling_file'), limits)
migration_task.update_options(:virtv2v_limits => limits)
end
end
Expand Down
16 changes: 11 additions & 5 deletions spec/lib/infra_conversion_throttler_spec.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,12 +120,18 @@
}
task_running_1.update_options(
:virtv2v_started_on => Time.now.utc,
:virtv2v_wrapper => {'state_file' => "/var/lib/uci/#{task_running_1.id}/state.json"}
:virtv2v_wrapper => {
'state_file' => "/tmp/state_1.json",
'throttling_file' => '/tmp/throttling_1.json'
}
)
task_running_2.options[:virtv2v_started_on] = Time.now.utc
task_running_2.options[:virtv2v_wrapper] = {'state_file' => "/var/lib/uci/#{task_running_2.id}/state.json"}
expect(conversion_host).to receive(:apply_task_limits).with(task_running_1.id, limits)
expect(conversion_host).not_to receive(:apply_task_limits).with(task_running_2.id, limits)
task_running_2.update_options(
:virtv2v_started_on => Time.now.utc,
:virtv2v_wrapper => {
'state_file' => "/tmp/state_2.json"
}
)
expect(conversion_host).to receive(:apply_task_limits).with('/tmp/throttling_1.json', limits)
described_class.apply_limits
expect(task_running_1.reload.options[:virtv2v_limits]).to eq(limits)
expect(task_running_2.reload.options[:virtv2v_limits]).to be_nil
Expand Down
Loading

0 comments on commit 14cde9e

Please sign in to comment.