From 28a9796bf5cd478144d58c33effb620ee83417bf Mon Sep 17 00:00:00 2001 From: smarrable Date: Mon, 14 Jan 2019 16:41:23 +1000 Subject: [PATCH 01/20] adding basic foundations of debugger. HTML builder, plus ParseHistory --- textfsm/visual_debugger.py | 63 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 textfsm/visual_debugger.py diff --git a/textfsm/visual_debugger.py b/textfsm/visual_debugger.py new file mode 100644 index 0000000..8c1d0d6 --- /dev/null +++ b/textfsm/visual_debugger.py @@ -0,0 +1,63 @@ + +class VisualDebugger: + + def __init__(self, fsm, parse_history): + self.fsm = fsm + self.parse_history = parse_history + self.state_colormap = {} + + def add_prelude_boilerplate(self, html_file): + prelude_lines = [ + "", + "", + "", + "", + "visual debugger" + ] + + html_file.writelines(prelude_lines) + + def hsl_css(self, h, s, l): + return "background-color: hsl({},{}%,{}%);".format(h, s, l) + + def add_css_styling(self, html_file): + css_prelude_lines = [ + "\n" + ] + + html_file.writelines(css_closing_lines) + + def build_debug_html(self): + with open("debug.html", "w+") as f: + self.add_prelude_boilerplate(f) + self.build_state_colors() + self.add_css_styling(f) + + + diff --git a/textfsm/debugger/visual_debugger.py b/textfsm/debugger/visual_debugger.py deleted file mode 100644 index d217f9a..0000000 --- a/textfsm/debugger/visual_debugger.py +++ /dev/null @@ -1,48 +0,0 @@ -from collections import namedtuple -class VisualDebugger(object): - - def __init__(self, fsm, parse_history): - self.fsm = fsm - self.parse_history = parse_history - self.state_colormap = {} - - def add_prelude_boilerplate(self, html_file): - prelude_lines = [ - "", - "", - "", - "", - "visual debugger" - ] - - html_file.writelines(prelude_lines) - - def hsl_css(self, h, s, l): - return "background-color: hsl({},{}%,{}%);".format(h, s, l) - - def add_css_styling(self, html_file): - css_prelude_lines = [ - "\n" @@ -89,11 +106,44 @@ def add_css_styling(self, html_file): html_file.writelines(css_closing_lines) + def add_cli_text(self, html_file): + + end_head_start_body = [ + "\n", + "\n", + "
\n"
+        ]
+
+        html_file.writelines(end_head_start_body)
+
+        lines = self.cli_text.splitlines()
+        lines = [line + '\n' for line in lines]
+
+        l_count = 0
+        for line_history in self.fsm.parse_history:
+            lines[l_count] = ("".format(line_history.state) + lines[l_count] + "")
+            l_count += 1
+            # match_index_pairs = []
+            # for match in line_history.matches:
+            #     print(len(match.match_obj.groups()))
+            #     print(match.match_obj.groups())
+
+        end_body_end_html = [
+            "
\n", + "\n", + "\n" + ] + + html_file.writelines(lines) + + html_file.writelines(end_body_end_html) + def build_debug_html(self): with open("debug.html", "w+") as f: self.add_prelude_boilerplate(f) self.build_state_colors() self.add_css_styling(f) + self.add_cli_text(f) From 0620b1797505b17d5c9e20a65251f9f8b3d0e546 Mon Sep 17 00:00:00 2001 From: smarrable Date: Wed, 16 Jan 2019 14:11:48 +1000 Subject: [PATCH 06/20] wip, adding match highlighting. Still needs work --- textfsm/debugger/__init__.py | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/textfsm/debugger/__init__.py b/textfsm/debugger/__init__.py index b793417..e0891a9 100644 --- a/textfsm/debugger/__init__.py +++ b/textfsm/debugger/__init__.py @@ -3,7 +3,7 @@ LINE_SATURATION = 70 LINE_LIGHTNESS = 75 MATCH_SATURATION = 100 -MATCH_LIGHTNESS = 50 +MATCH_LIGHTNESS = 30 BORDER_RADIUS = 5 @@ -95,6 +95,8 @@ def add_css_styling(self, html_file): MATCH_LIGHTNESS ), "border-radius: {}px;\n".format(BORDER_RADIUS), + "font - weight: bold;\n" + "color: white;\n", "}\n" ] state_matches.add(line.state) @@ -121,12 +123,29 @@ def add_cli_text(self, html_file): l_count = 0 for line_history in self.fsm.parse_history: - lines[l_count] = ("".format(line_history.state) + lines[l_count] + "") + + match_index_pairs = [] + for match in line_history.matches: + if len(match.match_obj.groups()) > 0: + built_line = lines[l_count][:match.match_obj.start(1)] + for i in range(0, len(match.match_obj.groups())): + built_line += ( + "".format(line_history.state) + + lines[l_count][match.match_obj.start(i):match.match_obj.end(i)] + + "" + + ) + built_line += lines[l_count][match.match_obj.end(1):] + lines[l_count] = built_line + else: + print("ZERO") + print(match.match_obj.groups()) + + lines[l_count] = ("".format(line_history.state) + + lines[l_count] + "") l_count += 1 - # match_index_pairs = [] - # for match in line_history.matches: - # print(len(match.match_obj.groups())) - # print(match.match_obj.groups()) + + end_body_end_html = [ "\n", From 5ef1a0ef29900c6c553c91cf3842388b569cba0b Mon Sep 17 00:00:00 2001 From: smarrable Date: Wed, 16 Jan 2019 16:16:25 +1000 Subject: [PATCH 07/20] wip, highlighting working, need to add regex matches --- textfsm/debugger/__init__.py | 109 ++++++++++++++++++++++++++++++----- 1 file changed, 93 insertions(+), 16 deletions(-) diff --git a/textfsm/debugger/__init__.py b/textfsm/debugger/__init__.py index e0891a9..62591a4 100644 --- a/textfsm/debugger/__init__.py +++ b/textfsm/debugger/__init__.py @@ -1,7 +1,7 @@ from collections import namedtuple -LINE_SATURATION = 70 -LINE_LIGHTNESS = 75 +LINE_SATURATION = 40 +LINE_LIGHTNESS = 60 MATCH_SATURATION = 100 MATCH_LIGHTNESS = 30 @@ -16,8 +16,14 @@ class MatchedPair(namedtuple('MatchPair', ['match_obj', 'rule'])): pass -class StartStopIndex(namedtuple('StartStopIndex', ['start', 'stop'])): - pass +class StartStopIndex(namedtuple('StartStopIndex', ['start', 'end'])): + + def __eq__(self, other): + return self.start == other.start and self.end == other.end + + def __gt__(self, other): + return self.start > other.start + class VisualDebugger(object): @@ -63,6 +69,11 @@ def add_css_styling(self, html_file): "\n" + ] + + html_file.writelines(css_closing_lines) + + def merge_indexes(self, match_index_pairs): + + def overlapping(index_a, index_b): + if index_a.end > index_b.start and index_a.start < index_b.end: + return True + if index_a.start < index_b.end and index_b.start < index_a.end: + return True + if index_a.start < index_b.start and index_a.end > index_b.end: + return True + if index_b.start < index_a.start and index_b.end > index_a.end: + return True + + def merge_pairs(index_a, index_b): + start = 0 + if index_a.start < index_b.start: + start = index_a.start + else: + start = index_b.start + if index_a.end < index_b.end: + end = index_b.end + else: + end = index_a.end + return StartStopIndex(start, end, [index_a.value, index_b.value]) + + for pair in match_index_pairs: + overlap = False + match_index_pairs.remove(pair) + for check_pair in match_index_pairs: + if overlapping(pair, check_pair): + overlap = True + match_index_pairs.remove(check_pair) + match_index_pairs.append(merge_pairs(pair, check_pair)) + break + if not overlap: + match_index_pairs.append(pair) + + def add_cli_text(self, html_file): + cli_text_prelude = [ + "\n", + "", + "

States:

\n" + ] + + for state in self.state_colormap.keys(): + cli_text_prelude += [ + "\n".format(state, state) + ] + + cli_text_prelude += [ + "

CLI Text:

\n" + "
\n"
+        ]
+
+        html_file.writelines(cli_text_prelude)
+
+        lines = self.cli_text.splitlines()
+        lines = [line + '\n' for line in lines]
+
+        l_count = 0
+        for line_history in self.fsm.parse_history:
+            if line_history.match_index_pairs:
+                print("GOT PAIRS")
+                # Merge indexes if there is any overlap
+                built_line = ""
+                prev_end = 0
+                for index in line_history.match_index_pairs:
+                    built_line += (
+                          lines[l_count][prev_end:index.start]
+                          + "".format(line_history.state)
+                          + lines[l_count][index.start:index.end]
+                          + ""
+                    )
+                    prev_end = index.end
+
+                built_line += lines[l_count][line_history.match_index_pairs[-1].end:]
+                lines[l_count] = built_line
+                
+            lines[l_count] = ("".format(line_history.state)
+                              + lines[l_count] + "")
+            l_count += 1
+
+        end_body_end_html = [
+            "
\n", + "\n", + "\n" + ] + + html_file.writelines(lines) + + html_file.writelines(end_body_end_html) + + def build_debug_html(self): + with open("debug.html", "w+") as f: + self.add_prelude_boilerplate(f) + self.build_state_colors() + self.add_css_styling(f) + self.add_cli_text(f) + + + From a85b476304b5e01523f57bd78380078ad3e2ced4 Mon Sep 17 00:00:00 2001 From: marrable Date: Fri, 18 Jan 2019 07:30:00 +1000 Subject: [PATCH 11/20] wip regex on hover working --- textfsm/debugger.py | 34 +++-- textfsm/debugger/__init__.py | 242 ---------------------------------- textfsm/debugger/__init__.pyc | Bin 719 -> 0 bytes textfsm/parser.py | 2 +- 4 files changed, 25 insertions(+), 253 deletions(-) delete mode 100644 textfsm/debugger/__init__.py delete mode 100644 textfsm/debugger/__init__.pyc diff --git a/textfsm/debugger.py b/textfsm/debugger.py index 4514fa1..6d14032 100644 --- a/textfsm/debugger.py +++ b/textfsm/debugger.py @@ -77,6 +77,15 @@ def add_css_styling(self, html_file): font-family: Arial, Helvetica, sans-serif; color: white; } + .regex { + background-color: silver; + border: 2px; + border-style: solid; + border-color: black; + display: none; + border-radius: 5px; + padding: 0px 10px; + } ''') html_file.writelines(css_prelude_lines) @@ -97,8 +106,8 @@ def add_css_styling(self, html_file): html_file.writelines(state_block) # Build and write state match styling CSS - state_matches = set() new_parse_history = [] + l_count = 0 for line in self.fsm.parse_history: match_index_pairs = [] @@ -118,10 +127,11 @@ def add_css_styling(self, html_file): match_index_pairs.sort() line = line._replace(match_index_pairs=match_index_pairs) new_parse_history.append(line) - if line.matches: - if line.state not in state_matches: + if line.match_index_pairs: + match_count = 0 + for index_pair in line.match_index_pairs: match_block = [ - ".{}-match{{\n".format(line.state), + ".{}-match-{}-{}{{\n".format(line.state, l_count, match_count), self.hsl_css( self.state_colormap[line.state], MATCH_SATURATION, @@ -131,11 +141,14 @@ def add_css_styling(self, html_file): " font-weight: bold;\n" " color: white;\n", " padding: 0px 5px;\n", - "}\n" + "}\n", + ".{}-match-{}-{}:hover + .regex {{\n".format(line.state, l_count, match_count), + " display: inline;\n", + "}\n" ] - state_matches.add(line.state) html_file.writelines(match_block) - + match_count += 1 + l_count += 1 self.fsm.parse_history = new_parse_history css_closing_lines = [ @@ -205,18 +218,19 @@ def add_cli_text(self, html_file): l_count = 0 for line_history in self.fsm.parse_history: if line_history.match_index_pairs: - print("GOT PAIRS") # Merge indexes if there is any overlap built_line = "" prev_end = 0 + match_count = 0 for index in line_history.match_index_pairs: built_line += ( lines[l_count][prev_end:index.start] - + "".format(line_history.state) + + "".format(line_history.state, l_count, match_count) + lines[l_count][index.start:index.end] - + "" + + "{}".format(self.fsm.value_map[index.value]) ) prev_end = index.end + match_count += 1 built_line += lines[l_count][line_history.match_index_pairs[-1].end:] lines[l_count] = built_line diff --git a/textfsm/debugger/__init__.py b/textfsm/debugger/__init__.py deleted file mode 100644 index 0f8ec96..0000000 --- a/textfsm/debugger/__init__.py +++ /dev/null @@ -1,242 +0,0 @@ -from collections import namedtuple -from textwrap import dedent - -LINE_SATURATION = 40 -LINE_LIGHTNESS = 60 -MATCH_SATURATION = 100 -MATCH_LIGHTNESS = 30 - -BORDER_RADIUS = 5 - - -class LineHistory(namedtuple('LineHistory', ['line', 'state', 'matches'])): - pass - - -class MatchedPair(namedtuple('MatchPair', ['match_obj', 'rule'])): - pass - - -class StartStopIndex(namedtuple('StartStopIndex', ['start', 'end', 'value'])): - - def __eq__(self, other): - return self.start == other.start and self.end == other.end - - def __gt__(self, other): - return self.start > other.start - - - -class VisualDebugger(object): - - def __init__(self, fsm, cli_text): - self.fsm = fsm - self.cli_text = cli_text - self.state_colormap = {} - - @staticmethod - def add_prelude_boilerplate(html_file): - prelude_lines = dedent(''' - - - - - visual debugger - ''') - - html_file.write(prelude_lines) - - def build_state_colors(self): - h = 0 - separation = 30 - used_colors = set() - for state_name in self.fsm.states.keys(): - while h in used_colors: - h = (h + separation) % 360 - self.state_colormap[state_name] = h - used_colors.add(h) - h = (h + separation) % 360 - if h == 0 or h > 360: - h = 0 - separation -= 10 - if separation == 0: - separation = 30 - - @staticmethod - def hsl_css(h, s, l): - return " background-color: hsl({},{}%,{}%);\n".format(h, s, l) - - def add_css_styling(self, html_file): - css_prelude_lines = dedent(''' - \n" - ] - - html_file.writelines(css_closing_lines) - - def merge_indexes(self, match_index_pairs): - - def overlapping(index_a, index_b): - if index_a.end > index_b.start and index_a.start < index_b.end: - return True - if index_a.start < index_b.end and index_b.start < index_a.end: - return True - if index_a.start < index_b.start and index_a.end > index_b.end: - return True - if index_b.start < index_a.start and index_b.end > index_a.end: - return True - - def merge_pairs(index_a, index_b): - start = 0 - if index_a.start < index_b.start: - start = index_a.start - else: - start = index_b.start - if index_a.end < index_b.end: - end = index_b.end - else: - end = index_a.end - return StartStopIndex(start, end, [index_a.value, index_b.value]) - - for pair in match_index_pairs: - overlap = False - match_index_pairs.remove(pair) - for check_pair in match_index_pairs: - if overlapping(pair, check_pair): - overlap = True - match_index_pairs.remove(check_pair) - match_index_pairs.append(merge_pairs(pair, check_pair)) - break - if not overlap: - match_index_pairs.append(pair) - - def add_cli_text(self, html_file): - cli_text_prelude = [ - "\n", - "", - "

States:

\n" - ] - - for state in self.state_colormap.keys(): - cli_text_prelude += [ - "\n".format(state, state) - ] - - cli_text_prelude += [ - "

CLI Text:

\n" - "
\n"
-        ]
-
-        html_file.writelines(cli_text_prelude)
-
-        lines = self.cli_text.splitlines()
-        lines = [line + '\n' for line in lines]
-
-        l_count = 0
-        for line_history in self.fsm.parse_history:
-
-            match_index_pairs = []
-
-            print(l_count)
-            # Flatten match index structure
-            for match in line_history.matches:
-                for key in match.match_obj.groupdict().keys():
-                    match_index_pairs.append(
-                        StartStopIndex(
-                            match.match_obj.start(key),
-                            match.match_obj.end(key),
-                            key
-                        )
-                    )
-                    print(self.fsm.value_map[key])
-
-            if match_index_pairs:
-                # Merge indexes if there is any overlap
-                self.merge_indexes(match_index_pairs)
-                match_index_pairs.sort()
-                built_line = ""
-                prev_end = 0
-                for index in match_index_pairs:
-                    built_line += (
-                          lines[l_count][prev_end:index.start]
-                          + "".format(line_history.state)
-                          + lines[l_count][index.start:index.end]
-                          + ""
-                    )
-                    prev_end = index.end
-
-                built_line += lines[l_count][match_index_pairs[-1].end:]
-                lines[l_count] = built_line
-
-            lines[l_count] = ("".format(line_history.state)
-                              + lines[l_count] + "")
-            l_count += 1
-
-        end_body_end_html = [
-            "
\n", - "\n", - "\n" - ] - - html_file.writelines(lines) - - html_file.writelines(end_body_end_html) - - def build_debug_html(self): - with open("debug.html", "w+") as f: - self.add_prelude_boilerplate(f) - self.build_state_colors() - self.add_css_styling(f) - self.add_cli_text(f) - - - diff --git a/textfsm/debugger/__init__.pyc b/textfsm/debugger/__init__.pyc deleted file mode 100644 index 1a3db3349c9f167026069d89c532398d2dd7cf6e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 719 zcmcIhOHRWu5FID|rc~k>S&<9y6Jo&v5K6^@3du@iHyFin6;C9(F1O-X8~|qAk~N!J z$@9$k$@AVg{+=ga+l$u{4r@gFHT}LIQ1l6?0jz+f0HGq#0ti(MY5-~oRSdfb)>It9 zC%_KiWl6{5z?*=Yf=Zx@{z$W)L>HXEBArtg^Fu2Y6k$Tp6jm$jHnqkXrT3=McgCY@ zw>Y4@rW}V9cq~xkm?B$X`KEpEfv>t@#)LovOV1Z6#gIk>$Ym+HkCdUMl-8-PrgYB= zR0e-VkUe|d`pg$LyLDw}b%Q=@^~U)O^#`uK?RTYDT~+BelhQN>r99hgbC$!(JvSeT z;OZerR!@a#aZ2To_gOF`-ReC!Zb{jHz2!WaoFDNRPI%Znyn~a+lx+n+rK?FK8rS%o aY7k84%s+yT@POz0C%fe7de(6or-NTU!=G*d diff --git a/textfsm/parser.py b/textfsm/parser.py index fad81bf..408ce91 100755 --- a/textfsm/parser.py +++ b/textfsm/parser.py @@ -933,7 +933,7 @@ def _CheckLine(self, line): """ line_history = None if self.visual_debug: - line_history = LineHistory(line, self._cur_state_name, []) + line_history = LineHistory(line, self._cur_state_name, [], []) for rule in self._cur_state: matched = self._CheckRule(rule, line) From 847aae64bf30cd0aed311e16b09e651e30f4d881 Mon Sep 17 00:00:00 2001 From: smarrable Date: Fri, 18 Jan 2019 14:06:25 +1000 Subject: [PATCH 12/20] added handling for negative indices --- textfsm/debugger.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/textfsm/debugger.py b/textfsm/debugger.py index 6d14032..863d46f 100644 --- a/textfsm/debugger.py +++ b/textfsm/debugger.py @@ -47,7 +47,7 @@ def add_prelude_boilerplate(html_file): html_file.write(prelude_lines) def build_state_colors(self): - h = 0 + h = 60 separation = 30 used_colors = set() for state_name in self.fsm.states.keys(): @@ -223,15 +223,20 @@ def add_cli_text(self, html_file): prev_end = 0 match_count = 0 for index in line_history.match_index_pairs: + if index.start < 0 or index.end < 0: + continue built_line += ( lines[l_count][prev_end:index.start] + "".format(line_history.state, l_count, match_count) + lines[l_count][index.start:index.end] - + "{}".format(self.fsm.value_map[index.value]) + + "{} >> {}".format(self.fsm.value_map[index.value], index.value) ) prev_end = index.end match_count += 1 + if l_count == 4: + print(line_history.match_index_pairs) + print(built_line) built_line += lines[l_count][line_history.match_index_pairs[-1].end:] lines[l_count] = built_line From 854bfcbdb4f5b4fe8540623053671e52d0530326 Mon Sep 17 00:00:00 2001 From: smarrable Date: Fri, 18 Jan 2019 15:35:18 +1000 Subject: [PATCH 13/20] added shadowed header --- textfsm/debugger.py | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/textfsm/debugger.py b/textfsm/debugger.py index 863d46f..aec2687 100644 --- a/textfsm/debugger.py +++ b/textfsm/debugger.py @@ -47,20 +47,10 @@ def add_prelude_boilerplate(html_file): html_file.write(prelude_lines) def build_state_colors(self): - h = 60 - separation = 30 - used_colors = set() + cntr = 1 for state_name in self.fsm.states.keys(): - while h in used_colors: - h = (h + separation) % 360 - self.state_colormap[state_name] = h - used_colors.add(h) - h = (h + separation) % 360 - if h == 0 or h > 360: - h = 0 - separation -= 10 - if separation == 0: - separation = 30 + self.state_colormap[state_name] = (67 * cntr) % 360 + cntr += 1 @staticmethod def hsl_css(h, s, l): @@ -71,11 +61,14 @@ def add_css_styling(self, html_file): \n" - ] - - html_file.writelines(css_closing_lines) - - def merge_indexes(self, match_index_pairs): - - def overlapping(index_a, index_b): - if index_a.end > index_b.start and index_a.start < index_b.end: - return True - if index_a.start < index_b.end and index_b.start < index_a.end: - return True - if index_a.start < index_b.start and index_a.end > index_b.end: - return True - if index_b.start < index_a.start and index_b.end > index_a.end: - return True - - def merge_pairs(index_a, index_b): - start = 0 - if index_a.start < index_b.start: - start = index_a.start - else: - start = index_b.start - if index_a.end < index_b.end: - end = index_b.end - else: - end = index_a.end - return StartStopIndex(start, end, [index_a.value, index_b.value]) - - for pair in match_index_pairs: - overlap = False - match_index_pairs.remove(pair) - for check_pair in match_index_pairs: - if overlapping(pair, check_pair): - overlap = True - match_index_pairs.remove(check_pair) - match_index_pairs.append(merge_pairs(pair, check_pair)) - break - if not overlap: - match_index_pairs.append(pair) - - def add_cli_text(self, html_file): - cli_text_prelude = [ - "\n", - "
", - "

States:

\n" - ] - - for state in self.state_colormap.keys(): - cli_text_prelude += [ - "\n".format(state, state) - ] - - cli_text_prelude += [ - "\n", - "

CLI Text:

\n", - "
\n"
-        ]
-
-        html_file.writelines(cli_text_prelude)
-
-        lines = self.cli_text.splitlines()
-        lines = [line + '\n' for line in lines]
-
-        l_count = 0
-        for line_history in self.fsm.parse_history:
-            if line_history.match_index_pairs:
-                # Merge indexes if there is any overlap
-                built_line = ""
-                prev_end = 0
-                match_count = 0
-                for index in line_history.match_index_pairs:
-                    if index.start < 0 or index.end < 0:
-                        continue
-                    built_line += (
-                          lines[l_count][prev_end:index.start]
-                          + "".format(line_history.state, l_count, match_count)
-                          + lines[l_count][index.start:index.end]
-                          + "{} >> {}".format(self.fsm.value_map[index.value], index.value)
-                    )
-                    prev_end = index.end
-                    match_count += 1
-
-                if l_count == 4:
-                    print(line_history.match_index_pairs)
-                    print(built_line)
-                built_line += lines[l_count][line_history.match_index_pairs[-1].end:]
-                lines[l_count] = built_line
-                
-            lines[l_count] = ("".format(line_history.state)
-                              + lines[l_count] + "")
-            l_count += 1
-
-        end_body_end_html = [
-            "
\n", - "\n", - "\n" - ] - - html_file.writelines(lines) - - html_file.writelines(end_body_end_html) - - def build_debug_html(self): - with open("debug.html", "w+") as f: - self.add_prelude_boilerplate(f) - self.build_state_colors() - self.add_css_styling(f) - self.add_cli_text(f) - - - + html_file.writelines(css_prelude_lines) + + # Build and write state styling CSS + for state_name in self.fsm.states.keys(): + state_block = [ + ".{}{{\n".format(state_name), + self.hsl_css( + self.state_colormap[state_name], + LINE_SATURATION, + LINE_LIGHTNESS + ), + " border-radius: {}px;\n".format(BORDER_RADIUS), + " padding: 0 10px;\n", + "}\n" + ] + html_file.writelines(state_block) + + # Build and write state match styling CSS + new_parse_history = [] + l_count = 0 + for line in self.fsm.parse_history: + + match_index_pairs = [] + + # Flatten match index structure + for match in line.matches: + for key in match.match_obj.groupdict().keys(): + match_index_pairs.append( + StartStopIndex( + match.match_obj.start(key), + match.match_obj.end(key), + key + ) + ) + + self.merge_indexes(match_index_pairs) + match_index_pairs.sort() + line = line._replace(match_index_pairs=match_index_pairs) + new_parse_history.append(line) + if line.match_index_pairs: + match_count = 0 + for index_pair in line.match_index_pairs: + match_block = [ + ".{}-match-{}-{}{{\n".format(line.state, l_count, match_count), + self.hsl_css( + self.state_colormap[line.state], + MATCH_SATURATION, + MATCH_LIGHTNESS + ), + " border-radius: {}px;\n".format(BORDER_RADIUS), + " font-weight: bold;\n" + " color: white;\n", + " padding: 0 5px;\n", + "}\n", + ".{}-match-{}-{}:hover + .regex {{\n".format(line.state, l_count, match_count), + " display: inline;\n", + "}\n" + ] + html_file.writelines(match_block) + match_count += 1 + l_count += 1 + self.fsm.parse_history = new_parse_history + + css_closing_lines = [ + "\n" + ] + + html_file.writelines(css_closing_lines) + + def merge_indexes(self, match_index_pairs): + + def overlapping(index_a, index_b): + if index_a.end > index_b.start and index_a.start < index_b.end: + return True + if index_a.start < index_b.end and index_b.start < index_a.end: + return True + if index_a.start < index_b.start and index_a.end > index_b.end: + return True + if index_b.start < index_a.start and index_b.end > index_a.end: + return True + + def merge_pairs(index_a, index_b): + start = 0 + if index_a.start < index_b.start: + start = index_a.start + else: + start = index_b.start + if index_a.end < index_b.end: + end = index_b.end + else: + end = index_a.end + return StartStopIndex(start, end, [index_a.value, index_b.value]) + + for pair in match_index_pairs: + overlap = False + match_index_pairs.remove(pair) + for check_pair in match_index_pairs: + if overlapping(pair, check_pair): + overlap = True + match_index_pairs.remove(check_pair) + match_index_pairs.append(merge_pairs(pair, check_pair)) + break + if not overlap: + match_index_pairs.append(pair) + + def add_cli_text(self, html_file): + cli_text_prelude = [ + "\n", + "
", + "

States:

\n" + ] + + for state in self.state_colormap.keys(): + cli_text_prelude += [ + "\n".format(state, state) + ] + + cli_text_prelude += [ + "\n", + "

CLI Text:

\n", + "
\n"
+    ]
+
+    html_file.writelines(cli_text_prelude)
+
+    lines = self.cli_text.splitlines()
+    lines = [line + '\n' for line in lines]
+
+    l_count = 0
+    for line_history in self.fsm.parse_history:
+      if line_history.match_index_pairs:
+        # Merge indexes if there is any overlap
+        built_line = ""
+        prev_end = 0
+        match_count = 0
+        for index in line_history.match_index_pairs:
+          if index.start < 0 or index.end < 0:
+            continue
+          value_pattern = self.fsm.value_map[index.value]
+          regex = re.sub('^\(\?P<.*?>', '', value_pattern)[:-1]
+          built_line += (
+              lines[l_count][prev_end:index.start]
+              + "".format(line_history.state, l_count, match_count)
+              + lines[l_count][index.start:index.end]
+              + "{} >> {}".format(regex, index.value)
+          )
+          prev_end = index.end
+          match_count += 1
+
+        if l_count == 4:
+          print(line_history.match_index_pairs)
+          print(built_line)
+        built_line += lines[l_count][line_history.match_index_pairs[-1].end:]
+        lines[l_count] = built_line
+
+      lines[l_count] = ("".format(line_history.state)
+                        + lines[l_count] + "")
+      l_count += 1
+
+    end_body_end_html = [
+      "
\n", + "\n", + "\n" + ] + + html_file.writelines(lines) + + html_file.writelines(end_body_end_html) + + def build_debug_html(self): + with open("debug.html", "w+") as f: + self.add_prelude_boilerplate(f) + self.build_state_colors() + self.add_css_styling(f) + self.add_cli_text(f) From 3df9e1bfb09c486b6e71fea96da91f18d4877e4c Mon Sep 17 00:00:00 2001 From: smarrable Date: Wed, 30 Jan 2019 10:42:35 +1000 Subject: [PATCH 15/20] cleaned up html generation to contain less linting errors --- textfsm/debugger.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/textfsm/debugger.py b/textfsm/debugger.py index acab519..eeb4c2b 100644 --- a/textfsm/debugger.py +++ b/textfsm/debugger.py @@ -207,11 +207,11 @@ def add_cli_text(self, html_file): for state in self.state_colormap.keys(): cli_text_prelude += [ - "\n".format(state, state) + "\n".format(state, state) ] cli_text_prelude += [ - "\n", "\n", "

CLI Text:

\n", "
\n"
@@ -232,27 +232,33 @@ def add_cli_text(self, html_file):
         for index in line_history.match_index_pairs:
           if index.start < 0 or index.end < 0:
             continue
+
+          # Strip out useless pattern format characters
           value_pattern = self.fsm.value_map[index.value]
-          regex = re.sub('^\(\?P<.*?>', '', value_pattern)[:-1]
+          regex = re.sub('\?P<.*?>', '', value_pattern).replace('<', '<').replace('>', '>')
+
+          # Build section of match and escape HTML tag chevrons if present
           built_line += (
-              lines[l_count][prev_end:index.start]
+              lines[l_count][prev_end:index.start].replace('<', '<').replace('>', '>')
               + "".format(line_history.state, l_count, match_count)
-              + lines[l_count][index.start:index.end]
+              + lines[l_count][index.start:index.end].replace('<', '<').replace('>', '>')
               + "{} >> {}".format(regex, index.value)
           )
           prev_end = index.end
           match_count += 1
 
-        if l_count == 4:
-          print(line_history.match_index_pairs)
-          print(built_line)
-        built_line += lines[l_count][line_history.match_index_pairs[-1].end:]
+        built_line += lines[l_count][line_history.match_index_pairs[-1].end:].replace('<', '<').replace('>', '>')
         lines[l_count] = built_line
+      else:
+        # Escape HTML tag chevrons if present
+        lines[l_count] = lines[l_count].replace('<', '<').replace('>', '>')
 
+      # Add final span wrapping tag for line state color
       lines[l_count] = ("".format(line_history.state)
                         + lines[l_count] + "")
       l_count += 1
 
+    # Close off document
     end_body_end_html = [
       "
\n", "\n", From 71ba4d5a3c11f9eeafe561c79648ec330655769d Mon Sep 17 00:00:00 2001 From: smarrable Date: Wed, 30 Jan 2019 13:06:51 +1000 Subject: [PATCH 16/20] added command line argument control of v-debug mode --- textfsm/debugger.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/textfsm/debugger.py b/textfsm/debugger.py index eeb4c2b..d37d6cb 100644 --- a/textfsm/debugger.py +++ b/textfsm/debugger.py @@ -12,11 +12,15 @@ class LineHistory(namedtuple('LineHistory', ['line', 'state', 'matches', 'match_index_pairs'])): - pass + """" The match history for a given line when parsed using the FSM. + + Contains the regex match objects for that line, + which are converted to indices for highlighting + """ class MatchedPair(namedtuple('MatchPair', ['match_obj', 'rule'])): - pass + """" Stores the line history when parsed using the FSM.""" class StartStopIndex(namedtuple('StartStopIndex', ['start', 'end', 'value'])): @@ -233,7 +237,7 @@ def add_cli_text(self, html_file): if index.start < 0 or index.end < 0: continue - # Strip out useless pattern format characters + # Strip out useless pattern format characters and value value_pattern = self.fsm.value_map[index.value] regex = re.sub('\?P<.*?>', '', value_pattern).replace('<', '<').replace('>', '>') From e24da2bb28f41897fbb225d9680fa6157cf25a06 Mon Sep 17 00:00:00 2001 From: smarrable Date: Wed, 30 Jan 2019 14:37:53 +1000 Subject: [PATCH 17/20] added formating to be consistent with project style. Added comments --- textfsm/debugger.py | 74 ++++++++++++++++++++++++++++++++++++--------- textfsm/parser.py | 11 +++++-- 2 files changed, 67 insertions(+), 18 deletions(-) diff --git a/textfsm/debugger.py b/textfsm/debugger.py index d37d6cb..5d47a7b 100644 --- a/textfsm/debugger.py +++ b/textfsm/debugger.py @@ -1,3 +1,30 @@ +#!/usr/bin/python +# +# Copyright 2011 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" Visual Debugger + +Provides a HTML-based debugging tool that allows authors of templates +to view the behavior of templates when applied to some example CLI text. +State changes are represented with color coding such that state +transitions are clearly represented during parsing. + +Matches on lines are highlighted to show extracted values and hovering +over a match shows the value and corresponding regex that was matched. +""" from collections import namedtuple from textwrap import dedent @@ -8,8 +35,6 @@ MATCH_SATURATION = 100 MATCH_LIGHTNESS = 30 -BORDER_RADIUS = 5 - class LineHistory(namedtuple('LineHistory', ['line', 'state', 'matches', 'match_index_pairs'])): """" The match history for a given line when parsed using the FSM. @@ -24,7 +49,7 @@ class MatchedPair(namedtuple('MatchPair', ['match_obj', 'rule'])): class StartStopIndex(namedtuple('StartStopIndex', ['start', 'end', 'value'])): - + """Represents the start and stop indices of a match for a given template value.""" def __eq__(self, other): return self.start == other.start and self.end == other.end @@ -33,6 +58,7 @@ def __gt__(self, other): class VisualDebugger(object): + """Responsible for building the parse history of a TextFSM object into a visual html doc. """ def __init__(self, fsm, cli_text): self.fsm = fsm @@ -52,6 +78,7 @@ def add_prelude_boilerplate(html_file): html_file.write(prelude_lines) def build_state_colors(self): + """Basic colour wheel selection for state highlighting""" cntr = 1 for state_name in self.fsm.states.keys(): self.state_colormap[state_name] = (67 * cntr) % 360 @@ -59,6 +86,7 @@ def build_state_colors(self): @staticmethod def hsl_css(h, s, l): + """Return the CSS string for HSL background color.""" return " background-color: hsl({},{}%,{}%);\n".format(h, s, l) def add_css_styling(self, html_file): @@ -108,7 +136,7 @@ def add_css_styling(self, html_file): LINE_SATURATION, LINE_LIGHTNESS ), - " border-radius: {}px;\n".format(BORDER_RADIUS), + " border-radius: 5 px;\n", " padding: 0 10px;\n", "}\n" ] @@ -132,10 +160,15 @@ def add_css_styling(self, html_file): ) ) + # Merge indexes that overlap due to multiple rule matches for a single line. self.merge_indexes(match_index_pairs) match_index_pairs.sort() + + # Overwrite named tuple data member line = line._replace(match_index_pairs=match_index_pairs) new_parse_history.append(line) + + # Generate CSS for match highlighting and on-hover regex display if line.match_index_pairs: match_count = 0 for index_pair in line.match_index_pairs: @@ -146,7 +179,7 @@ def add_css_styling(self, html_file): MATCH_SATURATION, MATCH_LIGHTNESS ), - " border-radius: {}px;\n".format(BORDER_RADIUS), + " border-radius: 5 px;\n", " font-weight: bold;\n" " color: white;\n", " padding: 0 5px;\n", @@ -158,6 +191,8 @@ def add_css_styling(self, html_file): html_file.writelines(match_block) match_count += 1 l_count += 1 + + # Overwrite parse history from FSM with newly processed history self.fsm.parse_history = new_parse_history css_closing_lines = [ @@ -167,6 +202,7 @@ def add_css_styling(self, html_file): html_file.writelines(css_closing_lines) def merge_indexes(self, match_index_pairs): + """Merge overlapping index pairs that may occur due to multiple rule matches.""" def overlapping(index_a, index_b): if index_a.end > index_b.start and index_a.start < index_b.end: @@ -203,6 +239,11 @@ def merge_pairs(index_a, index_b): match_index_pairs.append(pair) def add_cli_text(self, html_file): + """Builds the HTML elements of the debug page including: + - Colored States Header Bar + - Highlighted CLI Text + """ + cli_text_prelude = [ "\n", "
", @@ -226,10 +267,11 @@ def add_cli_text(self, html_file): lines = self.cli_text.splitlines() lines = [line + '\n' for line in lines] + # Process each line history and add highlighting where matches occur. l_count = 0 for line_history in self.fsm.parse_history: + # Only process highlights where matches occur. if line_history.match_index_pairs: - # Merge indexes if there is any overlap built_line = "" prev_end = 0 match_count = 0 @@ -237,11 +279,12 @@ def add_cli_text(self, html_file): if index.start < 0 or index.end < 0: continue - # Strip out useless pattern format characters and value + # Strip out useless pattern format characters and value label. + # Escape chevrons in regex pattern. value_pattern = self.fsm.value_map[index.value] regex = re.sub('\?P<.*?>', '', value_pattern).replace('<', '<').replace('>', '>') - # Build section of match and escape HTML tag chevrons if present + # Build section of match and escape non HTML chevrons if present built_line += ( lines[l_count][prev_end:index.start].replace('<', '<').replace('>', '>') + "".format(line_history.state, l_count, match_count) @@ -254,7 +297,7 @@ def add_cli_text(self, html_file): built_line += lines[l_count][line_history.match_index_pairs[-1].end:].replace('<', '<').replace('>', '>') lines[l_count] = built_line else: - # Escape HTML tag chevrons if present + # Escape non HTML tag chevrons if present lines[l_count] = lines[l_count].replace('<', '<').replace('>', '>') # Add final span wrapping tag for line state color @@ -263,17 +306,18 @@ def add_cli_text(self, html_file): l_count += 1 # Close off document - end_body_end_html = [ - "\n", - "\n", - "\n" - ] + end_body_end_html = dedent(''' + + + + ''') html_file.writelines(lines) - html_file.writelines(end_body_end_html) + html_file.write(end_body_end_html) def build_debug_html(self): + """Calls HTML building procedures in sequence to create debug HTML doc.""" with open("debug.html", "w+") as f: self.add_prelude_boilerplate(f) self.build_state_colors() diff --git a/textfsm/parser.py b/textfsm/parser.py index 408ce91..3a61de2 100755 --- a/textfsm/parser.py +++ b/textfsm/parser.py @@ -573,7 +573,7 @@ def __init__(self, template, options_class=_DEFAULT_OPTIONS): # Name of the current state. self._cur_state_name = None # Visual debug mode flag - self.visual_debug = True + self.visual_debug = False # Keep track of parse history when in debug mode. self.parse_history = [] @@ -1066,7 +1066,7 @@ def main(argv=None): argv = sys.argv try: - opts, args = getopt.getopt(argv[1:], 'h', ['help']) + opts, args = getopt.getopt(argv[1:], 'h', ['help', 'visual-debug']) except getopt.error as msg: raise Usage(msg) @@ -1091,6 +1091,11 @@ def main(argv=None): with open(args[1], 'r') as f: cli_input = f.read() + for opt, _ in opts: + if opt == '--visual-debug': + print("visual debug mode, open 'debug.html' for template behaviour when parsing the CLI text.") + fsm.visual_debug = True + table = fsm.ParseText(cli_input) if fsm.visual_debug: @@ -1117,7 +1122,7 @@ def main(argv=None): if __name__ == '__main__': - help_msg = '%s [--help] template [input_file [output_file]]\n' % sys.argv[0] + help_msg = '%s [--help] [--visual-debug] template [input_file [output_file]]\n' % sys.argv[0] try: sys.exit(main()) except Usage as err: From 0459b81c233e4c06cb2335206278723f5b3a19b7 Mon Sep 17 00:00:00 2001 From: smarrable Date: Wed, 30 Jan 2019 14:47:50 +1000 Subject: [PATCH 18/20] fixed dropping of border-radius style --- textfsm/debugger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/textfsm/debugger.py b/textfsm/debugger.py index 5d47a7b..326d135 100644 --- a/textfsm/debugger.py +++ b/textfsm/debugger.py @@ -136,7 +136,7 @@ def add_css_styling(self, html_file): LINE_SATURATION, LINE_LIGHTNESS ), - " border-radius: 5 px;\n", + " border-radius: 5px;\n", " padding: 0 10px;\n", "}\n" ] From d6f7dc80a6e92a79be38c404a439a627d73ca721 Mon Sep 17 00:00:00 2001 From: smarrable Date: Thu, 31 Jan 2019 11:17:14 +1000 Subject: [PATCH 19/20] added fix for when 'continue' action causes overlapping matches --- textfsm/debugger.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/textfsm/debugger.py b/textfsm/debugger.py index 326d135..6cb8c44 100644 --- a/textfsm/debugger.py +++ b/textfsm/debugger.py @@ -275,21 +275,31 @@ def add_cli_text(self, html_file): built_line = "" prev_end = 0 match_count = 0 + for index in line_history.match_index_pairs: if index.start < 0 or index.end < 0: continue # Strip out useless pattern format characters and value label. # Escape chevrons in regex pattern. - value_pattern = self.fsm.value_map[index.value] - regex = re.sub('\?P<.*?>', '', value_pattern).replace('<', '<').replace('>', '>') + re_patterns = [] + values = [] + if type(index.value) is list: + values = index.value + for v in index.value: + value_pattern = self.fsm.value_map[v] + re_patterns.append(re.sub('\?P<.*?>', '', value_pattern).replace('<', '<').replace('>', '>')) + else: + values.append(index.value) + value_pattern = self.fsm.value_map[index.value] + re_patterns.append(re.sub('\?P<.*?>', '', value_pattern).replace('<', '<').replace('>', '>')) # Build section of match and escape non HTML chevrons if present built_line += ( lines[l_count][prev_end:index.start].replace('<', '<').replace('>', '>') + "".format(line_history.state, l_count, match_count) + lines[l_count][index.start:index.end].replace('<', '<').replace('>', '>') - + "{} >> {}".format(regex, index.value) + + "{} >> {}".format(re_patterns, values) ) prev_end = index.end match_count += 1 From 708ec9f55f62f252bbb4fee37f4b940592fcbce8 Mon Sep 17 00:00:00 2001 From: smarrable Date: Thu, 31 Jan 2019 15:00:06 +1000 Subject: [PATCH 20/20] fixed HTML double backslash rendering in list of values matching --- textfsm/debugger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/textfsm/debugger.py b/textfsm/debugger.py index 6cb8c44..2eb9997 100644 --- a/textfsm/debugger.py +++ b/textfsm/debugger.py @@ -299,7 +299,7 @@ def add_cli_text(self, html_file): lines[l_count][prev_end:index.start].replace('<', '<').replace('>', '>') + "".format(line_history.state, l_count, match_count) + lines[l_count][index.start:index.end].replace('<', '<').replace('>', '>') - + "{} >> {}".format(re_patterns, values) + + "{} >> {}".format(', '.join(re_patterns), ', '.join(values)) ) prev_end = index.end match_count += 1