Skip to content

Commit

Permalink
Fix loganalyzer.py UnicodeDecodeError (#6524)
Browse files Browse the repository at this point in the history 
What is the motivation for this PR?
Fix "UnicodeDecodeError: 'utf-8' codec can't decode byte 0xc0 in position 2203: invalid start byte" when running loganalyzer.py on dut.
master and 202205 can hit this issue if syslog contains non-ascii characters.

How did you do it?
Add errors='ignore' parameter when open syslog file.

How did you verify/test it?
Run platform_tests/api/test_fan_drawer_fans.py::test_get_status and enable loganalyzer.

Signed-off-by: Zhaohui Sun <[email protected]>
  • Loading branch information
@ZhaohuiS @wangxin
ZhaohuiS authored and wangxin committed Oct 14, 2022
1 parent b24a2c7 commit 14c78ff
Show file tree
Hide file tree
Showing 2 changed files with 59 additions and 52 deletions.
10 changes: 5 additions & 5 deletions ansible/library/extract_log.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,9 +92,9 @@ def extract_lines(directory, filename, target_string):
path = os.path.join(directory, filename)
file = None
if 'gz' in path:
file = gzip.open(path, mode='rt')
file = gzip.open(path, mode='rt', errors='ignore')
else:
file = open(path)
file = open(path, errors='ignore')
result = None
with file:
# This might be a gunzip file or logrotate issue, there has
Expand Down Expand Up @@ -226,17 +226,17 @@ def combine_logs_and_save(directory, filenames, start_string, target_string, tar
do_copy = False
line_processed = 0
line_copied = 0
with open(target_filename, 'w') as fp:
with open(target_filename, 'w', errors='ignore') as fp:
for filename in reversed(filenames):
path = os.path.join(directory, filename)
dt = datetime.datetime.fromtimestamp(os.path.getctime(path))
sz = os.path.getsize(path)
logger.debug("extract_log combine_logs from file {} create time {}, size {}".format(path, dt, sz))
file = None
if 'gz' in path:
file = gzip.open(path, mode='rt')
file = gzip.open(path, mode='rt', errors='ignore')
else:
file = open(path)
file = open(path, errors='ignore')

with file:
for line in file:
Expand Down
101 changes: 54 additions & 47 deletions ansible/roles/test/files/tools/loganalyzer/loganalyzer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@
sudo python loganalyzer.py --out_dir /home/hrachya/projects/loganalyzer/log.analyzer.results --action analyze --run_id myTest114 --logs file3.log -m /home/hrachya/projects/loganalyzer/match.file.1.log,/home/hrachya/projects/loganalyzer/match.file.2.log -i ignore.file.1.log,ignore.file.2.log -v
'''

#---------------------------------------------------------------------
# ---------------------------------------------------------------------
# Global imports
#---------------------------------------------------------------------
# ---------------------------------------------------------------------
from __future__ import print_function
import sys
import getopt
Expand All @@ -30,15 +30,15 @@
import subprocess
from datetime import datetime

#---------------------------------------------------------------------
# ---------------------------------------------------------------------
# Global variables
#---------------------------------------------------------------------
# ---------------------------------------------------------------------
tokenizer = ','
comment_key = '#'
system_log_file = '/var/log/syslog'
re_rsyslog_pid = re.compile("PID:\s+(\d+)")

#-- List of ERROR codes to be returned by AnsibleLogAnalyzer
# -- List of ERROR codes to be returned by AnsibleLogAnalyzer
err_duplicate_start_marker = -1
err_duplicate_end_marker = -2
err_no_end_marker = -3
Expand All @@ -48,6 +48,7 @@
err_end_ignore_marker = -7
err_start_ignore_marker = -8


class AnsibleLogAnalyzer:
'''
@summary: Overview of functionality
Expand Down Expand Up @@ -84,36 +85,36 @@ class AnsibleLogAnalyzer:
def init_sys_logger(self):
logger = logging.getLogger('LogAnalyzer')
logger.setLevel(logging.DEBUG)
handler = logging.handlers.SysLogHandler(address = '/dev/log')
handler = logging.handlers.SysLogHandler(address='/dev/log')
logger.addHandler(handler)
return logger
#---------------------------------------------------------------------
# ---------------------------------------------------------------------

def __init__(self, run_id, verbose, start_marker = None):
def __init__(self, run_id, verbose, start_marker=None):
self.run_id = run_id
self.verbose = verbose
self.start_marker = start_marker
#---------------------------------------------------------------------
# ---------------------------------------------------------------------

def print_diagnostic_message(self, message):
if (not self.verbose):
return

print('[LogAnalyzer][diagnostic]:%s' % message)
#---------------------------------------------------------------------
# ---------------------------------------------------------------------

def create_start_marker(self):
if (self.start_marker is None) or (len(self.start_marker) == 0):
return self.start_marker_prefix + "-" + self.run_id
else:
return self.start_marker

#---------------------------------------------------------------------
# ---------------------------------------------------------------------

def is_filename_stdin(self, file_name):
return file_name == "-"

#---------------------------------------------------------------------
# ---------------------------------------------------------------------

def require_marker_check(self, file_path):
'''
Expand All @@ -127,19 +128,19 @@ def require_marker_check(self, file_path):
files_to_skip = ["sairedis.rec", "bgpd.log"]
return not any([target in file_path for target in files_to_skip])

#---------------------------------------------------------------------
# ---------------------------------------------------------------------

def create_end_marker(self):
return self.end_marker_prefix + "-" + self.run_id
#---------------------------------------------------------------------
# ---------------------------------------------------------------------

def create_start_ignore_marker(self):
return self.start_ignore_marker_prefix + "-" + self.run_id
#---------------------------------------------------------------------
# ---------------------------------------------------------------------

def create_end_ignore_marker(self):
return self.end_ignore_marker_prefix + "-" + self.run_id
#---------------------------------------------------------------------
# ---------------------------------------------------------------------

def flush_rsyslogd(self):
'''
Expand Down Expand Up @@ -208,11 +209,11 @@ def wait_for_marker(self, marker, timeout=120, polling_interval=10):
prev_syslog_file = "/var/log/syslog.1"
last_dt = os.path.getctime(syslog_file)
while wait_time <= timeout:
with open(syslog_file, 'r') as fp:
with open(syslog_file, 'r', errors='ignore') as fp:
dt = os.path.getctime(syslog_file)
if last_dt != dt:
try:
with open(prev_syslog_file, 'r') as pfp:
with open(prev_syslog_file, 'r', errors='ignore') as pfp:
pfp.seek(last_check_pos)
for l in fp:
if marker in l:
Expand Down Expand Up @@ -251,7 +252,7 @@ def place_marker(self, log_file_list, marker, wait_for_marker=False):
raise RuntimeError("cannot find marker {} in /var/log/syslog".format(marker))

return
#---------------------------------------------------------------------
# ---------------------------------------------------------------------

def error_to_regx(self, error_string):
'''
Expand All @@ -271,11 +272,11 @@ def error_to_regx(self, error_string):
original_string = error_string
#-- Escapes out of all the meta characters --#
error_string = re.escape(error_string)
#-- Replaces a white space with the white space regular expression
# -- Replaces a white space with the white space regular expression
error_string = re.sub(r"(\\\s+)+", "\\\\s+", error_string)
#-- Replaces a digit number with the digit regular expression
# -- Replaces a digit number with the digit regular expression
error_string = re.sub(r"\b\d+\b", "\\\\d+", error_string)
#-- Replaces a hex number with the hex regular expression
# -- Replaces a hex number with the hex regular expression
error_string = re.sub(r"0x[0-9a-fA-F]+", "0x[0-9a-fA-F]+", error_string)
self.print_diagnostic_message('Built error string: %s' % error_string)

Expand All @@ -284,7 +285,7 @@ def error_to_regx(self, error_string):
error_string = '|'.join(map(self.error_to_regx, error_string))

return error_string
#---------------------------------------------------------------------
# ---------------------------------------------------------------------

def create_msg_regex(self, file_lsit):
'''
Expand All @@ -310,17 +311,17 @@ def create_msg_regex(self, file_lsit):
for index, row in enumerate(csvreader):
row = [item for item in row if item != ""]
self.print_diagnostic_message('[diagnostic]:processing row:%d' % index)
self.print_diagnostic_message('row:%s'% row)
self.print_diagnostic_message('row:%s' % row)
try:
#-- Ignore Empty Lines
# -- Ignore Empty Lines
if not row:
continue
#-- Ignore commented Lines
# -- Ignore commented Lines
if row[0].startswith(comment_key):
self.print_diagnostic_message('[diagnostic]:skipping row[0]:%s' % row[0])
continue

#-- ('s' | 'r') = (Raw String | Regular Expression)
# -- ('s' | 'r') = (Raw String | Regular Expression)
is_regex = row[0]
if ('s' == row[0]):
is_regex = False
Expand All @@ -329,7 +330,7 @@ def create_msg_regex(self, file_lsit):
else:
raise Exception('file:%s, malformed line:%d. '
'must be \'s\'(string) or \'r\'(regex)'
%(filename,index))
% (filename, index))

if (is_regex):
messages_regex.extend(row[1:])
Expand All @@ -346,7 +347,7 @@ def create_msg_regex(self, file_lsit):
else:
regex = None
return regex, messages_regex
#---------------------------------------------------------------------
# ---------------------------------------------------------------------

def line_matches(self, str, match_messages_regex, ignore_messages_regex):
'''
Expand Down Expand Up @@ -378,7 +379,7 @@ def line_matches(self, str, match_messages_regex, ignore_messages_regex):
ret_code = True

return ret_code
#---------------------------------------------------------------------
# ---------------------------------------------------------------------

def line_is_expected(self, str, expect_messages_regex):
'''
Expand Down Expand Up @@ -413,11 +414,10 @@ def analyze_file(self, log_file_path, match_messages_regex, ignore_messages_rege
@return: List of strings match search criteria.
'''

self.print_diagnostic_message('analyzing file: %s' % log_file_path)

self.print_diagnostic_message('analyzing file: %s'% log_file_path)

#-- indicates whether log analyzer currently is in the log range between start
#-- and end marker. see analyze_file method.
# -- indicates whether log analyzer currently is in the log range between start
# -- and end marker. see analyze_file method.
check_marker = self.require_marker_check(log_file_path)
in_analysis_range = not check_marker
stdin_as_input = self.is_filename_stdin(log_file_path)
Expand Down Expand Up @@ -503,7 +503,7 @@ def analyze_file(self, log_file_path, match_messages_regex, ignore_messages_rege
sys.exit(err_no_end_marker)

return matching_lines, expected_lines
#---------------------------------------------------------------------
# ---------------------------------------------------------------------

def analyze_file_list(self, log_file_list, match_messages_regex, ignore_messages_regex, expect_messages_regex):
'''
Expand All @@ -528,14 +528,16 @@ def analyze_file_list(self, log_file_list, match_messages_regex, ignore_messages
for log_file in log_file_list:
if not len(log_file):
continue
match_strings, expect_strings = self.analyze_file(log_file, match_messages_regex, ignore_messages_regex, expect_messages_regex)
match_strings, expect_strings = self.analyze_file(
log_file, match_messages_regex, ignore_messages_regex, expect_messages_regex)

match_strings.reverse()
expect_strings.reverse()
res[log_file] = [ match_strings, expect_strings ]
res[log_file] = [match_strings, expect_strings]

return res
#---------------------------------------------------------------------
# ---------------------------------------------------------------------


def usage():
print('loganalyzer input parameters:')
Expand Down Expand Up @@ -565,7 +567,8 @@ def usage():
print(' in one of specified log files during the analysis. Must be present')
print(' when action == analyze.')

#---------------------------------------------------------------------
# ---------------------------------------------------------------------


def check_action(action, log_files_in, out_dir, match_files_in, ignore_files_in, expect_files_in):
'''
Expand All @@ -588,13 +591,13 @@ def check_action(action, log_files_in, out_dir, match_files_in, ignore_files_in,
print('ERROR: missing required match_files_in for analyze action')
ret_code = False


else:
ret_code = False
print('ERROR: invalid action:%s specified' % action)

return ret_code
#---------------------------------------------------------------------
# ---------------------------------------------------------------------


def check_run_id(run_id):
'''
Expand All @@ -612,7 +615,8 @@ def check_run_id(run_id):
ret_code = False

return ret_code
#---------------------------------------------------------------------
# ---------------------------------------------------------------------


def write_result_file(run_id, out_dir, analysis_result_per_file, messages_regex_e, unused_regex_messages):
'''
Expand Down Expand Up @@ -666,7 +670,8 @@ def write_result_file(run_id, out_dir, analysis_result_per_file, messages_regex_

out_file.write("\n-------------------------------------------------\n\n")
out_file.flush()
#---------------------------------------------------------------------
# ---------------------------------------------------------------------


def write_summary_file(run_id, out_dir, analysis_result_per_file, unused_regex_messages):
'''
Expand Down Expand Up @@ -703,7 +708,8 @@ def write_summary_file(run_id, out_dir, analysis_result_per_file, unused_regex_m
out_file.write("-----------------------------------\n")
out_file.flush()
out_file.close()
#---------------------------------------------------------------------
# ---------------------------------------------------------------------


def main(argv):

Expand All @@ -718,7 +724,8 @@ def main(argv):
verbose = False

try:
opts, args = getopt.getopt(argv, "a:r:s:l:o:m:i:e:vh", ["action=", "run_id=", "start_marker=", "logs=", "out_dir=", "match_files_in=", "ignore_files_in=", "expect_files_in=", "verbose", "help"])
opts, args = getopt.getopt(argv, "a:r:s:l:o:m:i:e:vh", [
"action=", "run_id=", "start_marker=", "logs=", "out_dir=", "match_files_in=", "ignore_files_in=", "expect_files_in=", "verbose", "help"])

except getopt.GetoptError:
print("Invalid option specified")
Expand Down Expand Up @@ -799,11 +806,11 @@ def main(argv):
analyzer.place_marker(log_file_list, analyzer.create_end_ignore_marker(), wait_for_marker=True)
return 0


else:
print('Unknown action:%s specified' % action)
return len(result)
#---------------------------------------------------------------------
# ---------------------------------------------------------------------


if __name__ == "__main__":
main(sys.argv[1:])

0 comments on commit 14c78ff

Please sign in to comment.