From d5df5a12728d3b0f2d681574fdd6c975de581545 Mon Sep 17 00:00:00 2001 From: nofaralfasi Date: Wed, 23 Aug 2023 12:14:26 +0300 Subject: [PATCH] Fixes #36341 - Refactor ansible runner artifacts processing --- .../artifacts_processor.rb | 42 +++++++++++++++++++ .../configuration_loader.rb | 1 + .../runner/ansible_runner.rb | 38 +++++++---------- 3 files changed, 57 insertions(+), 24 deletions(-) create mode 100644 lib/smart_proxy_ansible/artifacts_processor.rb diff --git a/lib/smart_proxy_ansible/artifacts_processor.rb b/lib/smart_proxy_ansible/artifacts_processor.rb new file mode 100644 index 0000000..700e958 --- /dev/null +++ b/lib/smart_proxy_ansible/artifacts_processor.rb @@ -0,0 +1,42 @@ +module Proxy + module Ansible + # Helper for Artifacts Reader + class ArtifactsProcessor + class << self + ARTIFACTS_DIR = 'artifacts'.freeze + def artifacts_reader(root) + @counter ||= 1 + uuid ||= if (f = Dir["#{root}/#{ARTIFACTS_DIR}/*"].first) + File.basename(f) + end + return unless uuid + + job_event_dir = File.join(root, ARTIFACTS_DIR, uuid, 'job_events') + all_files_with_nums = [] + + loop do + files = Dir["#{job_event_dir}/*.json"].map do |file| + num = File.basename(file)[/\A\d+/].to_i unless file.include?('partial') + [file, num] + end + files_with_nums = files.select { |(_, num)| num && num >= @counter }.sort_by(&:last) + break if files_with_nums.empty? + + all_files_with_nums.concat(files_with_nums) + @counter = files_with_nums.last.last + 1 + end + all_files_with_nums + end + + def hostname_for_event(event) + hostname = event.dig('event_data', 'host') || event.dig('event_data', 'remote_addr') + hostname.to_s unless hostname.nil? || hostname.empty? + end + + def get_event_data(event, data_type) + event.dig('event_data', data_type) || {} + end + end + end + end +end diff --git a/lib/smart_proxy_ansible/configuration_loader.rb b/lib/smart_proxy_ansible/configuration_loader.rb index 354b955..b5acc44 100644 --- a/lib/smart_proxy_ansible/configuration_loader.rb +++ b/lib/smart_proxy_ansible/configuration_loader.rb @@ -4,6 +4,7 @@ def load_classes require 'smart_proxy_dynflow' require 'smart_proxy_dynflow/continuous_output' require 'smart_proxy_ansible/task_launcher/ansible_runner' + require 'smart_proxy_ansible/artifacts_processor' require 'smart_proxy_ansible/runner/ansible_runner' Proxy::Dynflow::TaskLauncherRegistry.register('ansible-runner', diff --git a/lib/smart_proxy_ansible/runner/ansible_runner.rb b/lib/smart_proxy_ansible/runner/ansible_runner.rb index bf11172..a9373f7 100644 --- a/lib/smart_proxy_ansible/runner/ansible_runner.rb +++ b/lib/smart_proxy_ansible/runner/ansible_runner.rb @@ -78,24 +78,14 @@ def initialize_command(*command) private + def handle_json_parse_error(event_file, error) + logger.error("[foreman_ansible] - Error parsing runner event at #{event_file}: #{error.class}: #{error.message}") + logger.debug(error.backtrace.join("\n")) + end + def process_artifacts - @counter ||= 1 - @uuid ||= if (f = Dir["#{@root}/artifacts/*"].first) - File.basename(f) - end - return unless @uuid - job_event_dir = File.join(@root, 'artifacts', @uuid, 'job_events') - loop do - files = Dir["#{job_event_dir}/*.json"].map do |file| - num = File.basename(file)[/\A\d+/].to_i unless file.include?('partial') - [file, num] - end - files_with_nums = files.select { |(_, num)| num && num >= @counter }.sort_by(&:last) - break if files_with_nums.empty? - logger.debug("[foreman_ansible] - processing event files: #{files_with_nums.map(&:first).inspect}}") - files_with_nums.map(&:first).each { |event_file| handle_event_file(event_file) } - @counter = files_with_nums.last.last + 1 - end + all_files_with_nums = ArtifactsProcessor.artifacts_reader(@root) + all_files_with_nums.map(&:first).each { |event_file| handle_event_file(event_file) } end def handle_event_file(event_file) @@ -115,8 +105,7 @@ def handle_event_file(event_file) end def hostname_for_event(event) - hostname = event.dig('event_data', 'host') || event.dig('event_data', 'remote_addr') - return nil if hostname.nil? || hostname.empty? + hostname = ArtifactsProcessor.hostname_for_event(event) unless @targets.key?(hostname) logger.warn("handle_host_event: unknown host #{hostname} for event '#{event['event']}', broadcasting") @@ -140,10 +129,11 @@ def handle_host_event(hostname, event) def handle_broadcast_data(event) log_event("broadcast", event) - if event['event'] == 'playbook_on_stats' - failures = event.dig('event_data', 'failures') || {} - unreachable = event.dig('event_data', 'dark') || {} - rescued = event.dig('event_data', 'rescued') || {} + event_type = event['event'] + if event_type == 'playbook_on_stats' + failures = ArtifactsProcessor.get_event_data(event, 'failures') + unreachable = ArtifactsProcessor.get_event_data(event, 'dark') + rescued = ArtifactsProcessor.get_event_data(event, 'rescued') header, *rows = event['stdout'].strip.lines.map(&:chomp) @outputs.keys.select { |key| key.is_a? String }.each do |host| line = rows.find { |row| row =~ /#{host}/ } @@ -159,7 +149,7 @@ def handle_broadcast_data(event) end # If the run ends early due to an error - fail all other tasks - if event['event'] == 'error' + if event_type == 'error' @outputs.keys.select { |key| key.is_a? String }.each do |host| @exit_statuses[host] = 4 if @exit_statuses[host].to_i == 0 end