Skip to content

Commit

Permalink
Revert "Fix loganalyzer.py UnicodeDecodeError (#6524)" (#6577)
Browse files Browse the repository at this point in the history 
This reverts commit 94ac4a0.
  • Loading branch information
@ZhaohuiS
ZhaohuiS authored Oct 20, 2022
1 parent 9bb1e74 commit 61acf83
Show file tree
Hide file tree
Showing 2 changed files with 52 additions and 59 deletions.
10 changes: 5 additions & 5 deletions ansible/library/extract_log.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,9 +92,9 @@ def extract_lines(directory, filename, target_string):
path = os.path.join(directory, filename)
file = None
if 'gz' in path:
file = gzip.open(path, mode='rt', errors='ignore')
file = gzip.open(path, mode='rt')
else:
file = open(path, errors='ignore')
file = open(path)
result = None
with file:
# This might be a gunzip file or logrotate issue, there has
Expand Down Expand Up @@ -226,17 +226,17 @@ def combine_logs_and_save(directory, filenames, start_string, target_string, tar
do_copy = False
line_processed = 0
line_copied = 0
with open(target_filename, 'w', errors='ignore') as fp:
with open(target_filename, 'w') as fp:
for filename in reversed(filenames):
path = os.path.join(directory, filename)
dt = datetime.datetime.fromtimestamp(os.path.getctime(path))
sz = os.path.getsize(path)
logger.debug("extract_log combine_logs from file {} create time {}, size {}".format(path, dt, sz))
file = None
if 'gz' in path:
file = gzip.open(path, mode='rt', errors='ignore')
file = gzip.open(path, mode='rt')
else:
file = open(path, errors='ignore')
file = open(path)

with file:
for line in file:
Expand Down
101 changes: 47 additions & 54 deletions ansible/roles/test/files/tools/loganalyzer/loganalyzer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@
sudo python loganalyzer.py --out_dir /home/hrachya/projects/loganalyzer/log.analyzer.results --action analyze --run_id myTest114 --logs file3.log -m /home/hrachya/projects/loganalyzer/match.file.1.log,/home/hrachya/projects/loganalyzer/match.file.2.log -i ignore.file.1.log,ignore.file.2.log -v
'''

# ---------------------------------------------------------------------
#---------------------------------------------------------------------
# Global imports
# ---------------------------------------------------------------------
#---------------------------------------------------------------------
from __future__ import print_function
import sys
import getopt
Expand All @@ -30,15 +30,15 @@
import subprocess
from datetime import datetime

# ---------------------------------------------------------------------
#---------------------------------------------------------------------
# Global variables
# ---------------------------------------------------------------------
#---------------------------------------------------------------------
tokenizer = ','
comment_key = '#'
system_log_file = '/var/log/syslog'
re_rsyslog_pid = re.compile("PID:\s+(\d+)")

# -- List of ERROR codes to be returned by AnsibleLogAnalyzer
#-- List of ERROR codes to be returned by AnsibleLogAnalyzer
err_duplicate_start_marker = -1
err_duplicate_end_marker = -2
err_no_end_marker = -3
Expand All @@ -48,7 +48,6 @@
err_end_ignore_marker = -7
err_start_ignore_marker = -8


class AnsibleLogAnalyzer:
'''
@summary: Overview of functionality
Expand Down Expand Up @@ -85,36 +84,36 @@ class AnsibleLogAnalyzer:
def init_sys_logger(self):
logger = logging.getLogger('LogAnalyzer')
logger.setLevel(logging.DEBUG)
handler = logging.handlers.SysLogHandler(address='/dev/log')
handler = logging.handlers.SysLogHandler(address = '/dev/log')
logger.addHandler(handler)
return logger
# ---------------------------------------------------------------------
#---------------------------------------------------------------------

def __init__(self, run_id, verbose, start_marker=None):
def __init__(self, run_id, verbose, start_marker = None):
self.run_id = run_id
self.verbose = verbose
self.start_marker = start_marker
# ---------------------------------------------------------------------
#---------------------------------------------------------------------

def print_diagnostic_message(self, message):
if (not self.verbose):
return

print('[LogAnalyzer][diagnostic]:%s' % message)
# ---------------------------------------------------------------------
#---------------------------------------------------------------------

def create_start_marker(self):
if (self.start_marker is None) or (len(self.start_marker) == 0):
return self.start_marker_prefix + "-" + self.run_id
else:
return self.start_marker

# ---------------------------------------------------------------------
#---------------------------------------------------------------------

def is_filename_stdin(self, file_name):
return file_name == "-"

# ---------------------------------------------------------------------
#---------------------------------------------------------------------

def require_marker_check(self, file_path):
'''
Expand All @@ -128,19 +127,19 @@ def require_marker_check(self, file_path):
files_to_skip = ["sairedis.rec", "bgpd.log"]
return not any([target in file_path for target in files_to_skip])

# ---------------------------------------------------------------------
#---------------------------------------------------------------------

def create_end_marker(self):
return self.end_marker_prefix + "-" + self.run_id
# ---------------------------------------------------------------------
#---------------------------------------------------------------------

def create_start_ignore_marker(self):
return self.start_ignore_marker_prefix + "-" + self.run_id
# ---------------------------------------------------------------------
#---------------------------------------------------------------------

def create_end_ignore_marker(self):
return self.end_ignore_marker_prefix + "-" + self.run_id
# ---------------------------------------------------------------------
#---------------------------------------------------------------------

def flush_rsyslogd(self):
'''
Expand Down Expand Up @@ -209,11 +208,11 @@ def wait_for_marker(self, marker, timeout=120, polling_interval=10):
prev_syslog_file = "/var/log/syslog.1"
last_dt = os.path.getctime(syslog_file)
while wait_time <= timeout:
with open(syslog_file, 'r', errors='ignore') as fp:
with open(syslog_file, 'r') as fp:
dt = os.path.getctime(syslog_file)
if last_dt != dt:
try:
with open(prev_syslog_file, 'r', errors='ignore') as pfp:
with open(prev_syslog_file, 'r') as pfp:
pfp.seek(last_check_pos)
for l in fp:
if marker in l:
Expand Down Expand Up @@ -252,7 +251,7 @@ def place_marker(self, log_file_list, marker, wait_for_marker=False):
raise RuntimeError("cannot find marker {} in /var/log/syslog".format(marker))

return
# ---------------------------------------------------------------------
#---------------------------------------------------------------------

def error_to_regx(self, error_string):
'''
Expand All @@ -272,11 +271,11 @@ def error_to_regx(self, error_string):
original_string = error_string
#-- Escapes out of all the meta characters --#
error_string = re.escape(error_string)
# -- Replaces a white space with the white space regular expression
#-- Replaces a white space with the white space regular expression
error_string = re.sub(r"(\\\s+)+", "\\\\s+", error_string)
# -- Replaces a digit number with the digit regular expression
#-- Replaces a digit number with the digit regular expression
error_string = re.sub(r"\b\d+\b", "\\\\d+", error_string)
# -- Replaces a hex number with the hex regular expression
#-- Replaces a hex number with the hex regular expression
error_string = re.sub(r"0x[0-9a-fA-F]+", "0x[0-9a-fA-F]+", error_string)
self.print_diagnostic_message('Built error string: %s' % error_string)

Expand All @@ -285,7 +284,7 @@ def error_to_regx(self, error_string):
error_string = '|'.join(map(self.error_to_regx, error_string))

return error_string
# ---------------------------------------------------------------------
#---------------------------------------------------------------------

def create_msg_regex(self, file_lsit):
'''
Expand All @@ -311,17 +310,17 @@ def create_msg_regex(self, file_lsit):
for index, row in enumerate(csvreader):
row = [item for item in row if item != ""]
self.print_diagnostic_message('[diagnostic]:processing row:%d' % index)
self.print_diagnostic_message('row:%s' % row)
self.print_diagnostic_message('row:%s'% row)
try:
# -- Ignore Empty Lines
#-- Ignore Empty Lines
if not row:
continue
# -- Ignore commented Lines
#-- Ignore commented Lines
if row[0].startswith(comment_key):
self.print_diagnostic_message('[diagnostic]:skipping row[0]:%s' % row[0])
continue

# -- ('s' | 'r') = (Raw String | Regular Expression)
#-- ('s' | 'r') = (Raw String | Regular Expression)
is_regex = row[0]
if ('s' == row[0]):
is_regex = False
Expand All @@ -330,7 +329,7 @@ def create_msg_regex(self, file_lsit):
else:
raise Exception('file:%s, malformed line:%d. '
'must be \'s\'(string) or \'r\'(regex)'
% (filename, index))
%(filename,index))

if (is_regex):
messages_regex.extend(row[1:])
Expand All @@ -347,7 +346,7 @@ def create_msg_regex(self, file_lsit):
else:
regex = None
return regex, messages_regex
# ---------------------------------------------------------------------
#---------------------------------------------------------------------

def line_matches(self, str, match_messages_regex, ignore_messages_regex):
'''
Expand Down Expand Up @@ -379,7 +378,7 @@ def line_matches(self, str, match_messages_regex, ignore_messages_regex):
ret_code = True

return ret_code
# ---------------------------------------------------------------------
#---------------------------------------------------------------------

def line_is_expected(self, str, expect_messages_regex):
'''
Expand Down Expand Up @@ -414,10 +413,11 @@ def analyze_file(self, log_file_path, match_messages_regex, ignore_messages_rege
@return: List of strings match search criteria.
'''

self.print_diagnostic_message('analyzing file: %s' % log_file_path)

# -- indicates whether log analyzer currently is in the log range between start
# -- and end marker. see analyze_file method.
self.print_diagnostic_message('analyzing file: %s'% log_file_path)

#-- indicates whether log analyzer currently is in the log range between start
#-- and end marker. see analyze_file method.
check_marker = self.require_marker_check(log_file_path)
in_analysis_range = not check_marker
stdin_as_input = self.is_filename_stdin(log_file_path)
Expand Down Expand Up @@ -503,7 +503,7 @@ def analyze_file(self, log_file_path, match_messages_regex, ignore_messages_rege
sys.exit(err_no_end_marker)

return matching_lines, expected_lines
# ---------------------------------------------------------------------
#---------------------------------------------------------------------

def analyze_file_list(self, log_file_list, match_messages_regex, ignore_messages_regex, expect_messages_regex):
'''
Expand All @@ -528,16 +528,14 @@ def analyze_file_list(self, log_file_list, match_messages_regex, ignore_messages
for log_file in log_file_list:
if not len(log_file):
continue
match_strings, expect_strings = self.analyze_file(
log_file, match_messages_regex, ignore_messages_regex, expect_messages_regex)
match_strings, expect_strings = self.analyze_file(log_file, match_messages_regex, ignore_messages_regex, expect_messages_regex)

match_strings.reverse()
expect_strings.reverse()
res[log_file] = [match_strings, expect_strings]
res[log_file] = [ match_strings, expect_strings ]

return res
# ---------------------------------------------------------------------

#---------------------------------------------------------------------

def usage():
print('loganalyzer input parameters:')
Expand Down Expand Up @@ -567,8 +565,7 @@ def usage():
print(' in one of specified log files during the analysis. Must be present')
print(' when action == analyze.')

# ---------------------------------------------------------------------

#---------------------------------------------------------------------

def check_action(action, log_files_in, out_dir, match_files_in, ignore_files_in, expect_files_in):
'''
Expand All @@ -591,13 +588,13 @@ def check_action(action, log_files_in, out_dir, match_files_in, ignore_files_in,
print('ERROR: missing required match_files_in for analyze action')
ret_code = False


else:
ret_code = False
print('ERROR: invalid action:%s specified' % action)

return ret_code
# ---------------------------------------------------------------------

#---------------------------------------------------------------------

def check_run_id(run_id):
'''
Expand All @@ -615,8 +612,7 @@ def check_run_id(run_id):
ret_code = False

return ret_code
# ---------------------------------------------------------------------

#---------------------------------------------------------------------

def write_result_file(run_id, out_dir, analysis_result_per_file, messages_regex_e, unused_regex_messages):
'''
Expand Down Expand Up @@ -670,8 +666,7 @@ def write_result_file(run_id, out_dir, analysis_result_per_file, messages_regex_

out_file.write("\n-------------------------------------------------\n\n")
out_file.flush()
# ---------------------------------------------------------------------

#---------------------------------------------------------------------

def write_summary_file(run_id, out_dir, analysis_result_per_file, unused_regex_messages):
'''
Expand Down Expand Up @@ -708,8 +703,7 @@ def write_summary_file(run_id, out_dir, analysis_result_per_file, unused_regex_m
out_file.write("-----------------------------------\n")
out_file.flush()
out_file.close()
# ---------------------------------------------------------------------

#---------------------------------------------------------------------

def main(argv):

Expand All @@ -724,8 +718,7 @@ def main(argv):
verbose = False

try:
opts, args = getopt.getopt(argv, "a:r:s:l:o:m:i:e:vh", [
"action=", "run_id=", "start_marker=", "logs=", "out_dir=", "match_files_in=", "ignore_files_in=", "expect_files_in=", "verbose", "help"])
opts, args = getopt.getopt(argv, "a:r:s:l:o:m:i:e:vh", ["action=", "run_id=", "start_marker=", "logs=", "out_dir=", "match_files_in=", "ignore_files_in=", "expect_files_in=", "verbose", "help"])

except getopt.GetoptError:
print("Invalid option specified")
Expand Down Expand Up @@ -806,11 +799,11 @@ def main(argv):
analyzer.place_marker(log_file_list, analyzer.create_end_ignore_marker(), wait_for_marker=True)
return 0


else:
print('Unknown action:%s specified' % action)
return len(result)
# ---------------------------------------------------------------------

#---------------------------------------------------------------------

if __name__ == "__main__":
main(sys.argv[1:])

0 comments on commit 61acf83

Please sign in to comment.