From 8b2f6bb43370975b2ba7fc2418d7f8f096d651f2 Mon Sep 17 00:00:00 2001 From: Sinan Akkoyun Date: Mon, 2 Oct 2023 00:41:10 +0200 Subject: [PATCH 1/6] Implemented lang detection by specified block lang --- examples/chat_formatting.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/examples/chat_formatting.py b/examples/chat_formatting.py index 7d04ad19..aed4a71f 100644 --- a/examples/chat_formatting.py +++ b/examples/chat_formatting.py @@ -126,7 +126,11 @@ def print_code_block(self, chunk): self.code_block_text += chunk # Remove language after codeblock start - code_block_text = re.sub(r'```.*?$', '```', self.code_block_text, flags=re.MULTILINE) + code_block_text = '\n' + '\n'.join(self.code_block_text.split('\n')[1:]) # Removes output of specified language + specified_lang = self.code_block_text.split('\n', 1)[0] # Get 1st line (directly after delimiter, can be language) + + if specified_lang is not None: + print(specified_lang) # Split updated text into lines and find the longest line lines = code_block_text.split('\n') @@ -140,7 +144,7 @@ def print_code_block(self, chunk): # Try guessing the lexer for syntax highlighting, if we haven't guessed already try: - lexer = guess_lexer(padded_text) + lexer = guess_lexer(padded_text) if specified_lang is None else get_lexer_by_name(specified_lang) except ClassNotFound: lexer = get_lexer_by_name("text") # Fallback to plain text if language isn't supported by pygments @@ -153,3 +157,8 @@ def print_code_block(self, chunk): # Update the lines_printed counter self.lines_printed = len(lines) + + def write(self): + f = open("demofile3.txt", "w") + f.write(self.code_block_text) + f.close() From f7072fac7ec5e032eb5ac7e0e16912d1e1155336 Mon Sep 17 00:00:00 2001 From: Sinan Akkoyun Date: Mon, 2 Oct 2023 00:42:22 +0200 Subject: [PATCH 2/6] removed debug line --- examples/chat_formatting.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/examples/chat_formatting.py b/examples/chat_formatting.py index aed4a71f..cc184dcc 100644 --- a/examples/chat_formatting.py +++ b/examples/chat_formatting.py @@ -129,9 +129,6 @@ def print_code_block(self, chunk): code_block_text = '\n' + '\n'.join(self.code_block_text.split('\n')[1:]) # Removes output of specified language specified_lang = self.code_block_text.split('\n', 1)[0] # Get 1st line (directly after delimiter, can be language) - if specified_lang is not None: - print(specified_lang) - # Split updated text into lines and find the longest line lines = code_block_text.split('\n') max_length = max(len(line) for line in lines) From f58000eca67727742c451ff58329bb394b479476 Mon Sep 17 00:00:00 2001 From: Sinan Akkoyun Date: Mon, 2 Oct 2023 00:53:43 +0200 Subject: [PATCH 3/6] fixed line-deletion bug with lang replacement --- examples/chat_formatting.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/chat_formatting.py b/examples/chat_formatting.py index cc184dcc..f2bad205 100644 --- a/examples/chat_formatting.py +++ b/examples/chat_formatting.py @@ -126,7 +126,8 @@ def print_code_block(self, chunk): self.code_block_text += chunk # Remove language after codeblock start - code_block_text = '\n' + '\n'.join(self.code_block_text.split('\n')[1:]) # Removes output of specified language + code_block_text = '\n'.join([''] + self.code_block_text.split('\n')[1:]) + specified_lang = self.code_block_text.split('\n', 1)[0] # Get 1st line (directly after delimiter, can be language) # Split updated text into lines and find the longest line From c5283cb25dd5dc5d5fec3821dda10c8421dd135e Mon Sep 17 00:00:00 2001 From: Sinan Akkoyun Date: Mon, 2 Oct 2023 01:04:46 +0200 Subject: [PATCH 4/6] Offloaded lexguessing to every new line :) --- examples/chat_formatting.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/examples/chat_formatting.py b/examples/chat_formatting.py index f2bad205..fe81ee50 100644 --- a/examples/chat_formatting.py +++ b/examples/chat_formatting.py @@ -91,6 +91,7 @@ class CodeBlockFormatter: code_block_text: str lines_printed: int + last_lexer: str formatter = BlackBackgroundTerminalFormatter() @@ -100,6 +101,7 @@ def begin(self): self.code_block_text = "" self.lines_printed = 0 + self.last_lexer = get_lexer_by_name("text") self.formatter.begin() @@ -128,6 +130,7 @@ def print_code_block(self, chunk): # Remove language after codeblock start code_block_text = '\n'.join([''] + self.code_block_text.split('\n')[1:]) + # Get specified language specified_lang = self.code_block_text.split('\n', 1)[0] # Get 1st line (directly after delimiter, can be language) # Split updated text into lines and find the longest line @@ -142,9 +145,17 @@ def print_code_block(self, chunk): # Try guessing the lexer for syntax highlighting, if we haven't guessed already try: - lexer = guess_lexer(padded_text) if specified_lang is None else get_lexer_by_name(specified_lang) + if bool(specified_lang): + lexer = get_lexer_by_name(specified_lang) + self.last_lexer = lexer + elif '\n' in chunk: # Offload lexguessing to every newline + lexer = guess_lexer(padded_text) + self.last_lexer = lexer + else: + lexer = self.last_lexer except ClassNotFound: lexer = get_lexer_by_name("text") # Fallback to plain text if language isn't supported by pygments + self.last_lexer = lexer # Highlight highlighted_text = highlight(padded_text, lexer, self.formatter) From 8d5e02ce62888160cf3c10de83ddc17f7578085e Mon Sep 17 00:00:00 2001 From: Sinan Akkoyun Date: Mon, 2 Oct 2023 01:18:49 +0200 Subject: [PATCH 5/6] Removed debug code --- examples/chat_formatting.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/examples/chat_formatting.py b/examples/chat_formatting.py index fe81ee50..b38a93e3 100644 --- a/examples/chat_formatting.py +++ b/examples/chat_formatting.py @@ -166,8 +166,3 @@ def print_code_block(self, chunk): # Update the lines_printed counter self.lines_printed = len(lines) - - def write(self): - f = open("demofile3.txt", "w") - f.write(self.code_block_text) - f.close() From 2c9b122c129ffc6bb3d5e4ab7a51c4333359a8d5 Mon Sep 17 00:00:00 2001 From: SinanAkkoyun Date: Mon, 2 Oct 2023 23:31:09 +0200 Subject: [PATCH 6/6] Fixed Mistral 7B codeblock delim chunking (`` + `) --- examples/chat.py | 27 ++++++++++++++++++++++----- examples/chat_formatting.py | 9 ++++++++- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/examples/chat.py b/examples/chat.py index 61e443bc..72292d51 100644 --- a/examples/chat.py +++ b/examples/chat.py @@ -1,5 +1,5 @@ -import sys, os +import sys, os, re sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from exllamav2 import( @@ -181,6 +181,11 @@ def get_tokenized_context(max_len): codeblock_formatter = None if args.no_code_formatting else CodeBlockFormatter() in_code_block = False +delim_buffer_array = [] +delim_pattern = re.compile(r'(`{1,3})') + +delim_overflow = "" + # Main loop while True: @@ -219,10 +224,17 @@ def get_tokenized_context(max_len): response_text += chunk responses_ids[-1] = torch.cat([responses_ids[-1], tokens], dim = -1) - # Check for code block delimiters + # Append chunk to delimiter buffer if contains delimiters + if delim_pattern.search(chunk) and len(delim_buffer_array) < 2: # dirty fix for assumption that codeblock start is never smaller than `` + ` + # add chunk + delim_buffer_array.append(chunk) + else: + delim_overflow = "".join(delim_buffer_array) + delim_buffer_array = [] - codeblock_delimiter = chunk.startswith("```") and codeblock_formatter is not None - if codeblock_delimiter: chunk = chunk[3:] # Suppress delimiter in output + # Check for code block delimiters + # if delim_buffer_array contains a full delimiter (```), codeblock true + codeblock_delimiter = "".join(delim_buffer_array).find("```") != -1 and (codeblock_formatter is not None) # Print output @@ -233,9 +245,13 @@ def get_tokenized_context(max_len): codeblock_formatter.begin() print("\n") in_code_block = True + delim_buffer_array = [] # Print unformatted - print(chunk, end = "") + # if delim buffer is > 0 do not print for now + if len(delim_buffer_array) == 0: + print(chunk, end = "") + sys.stdout.flush() else: @@ -244,6 +260,7 @@ def get_tokenized_context(max_len): if codeblock_delimiter: print("\033[0m", end = "") # Reset block color to be certain in_code_block = False + delim_buffer_array = [] # Print formatted codeblock_formatter.print_code_block(chunk) diff --git a/examples/chat_formatting.py b/examples/chat_formatting.py index b38a93e3..229e86b0 100644 --- a/examples/chat_formatting.py +++ b/examples/chat_formatting.py @@ -15,11 +15,13 @@ # Code block formatter for black background + class BlackBackgroundTerminalFormatter(TerminalFormatter): code_pad: int = 2 block_pad_left: int = 1 + def __init__(self): super().__init__(style = "monokai") @@ -109,7 +111,7 @@ def begin(self): # Print a code block, updating the CLI in real-time def print_code_block(self, chunk): - + # Clear previously printed lines for _ in range(self.lines_printed): # -1 not needed? # Move cursor up one line @@ -130,6 +132,11 @@ def print_code_block(self, chunk): # Remove language after codeblock start code_block_text = '\n'.join([''] + self.code_block_text.split('\n')[1:]) + # Handle delim at end + if code_block_text.endswith("```"): + code_block_text = code_block_text[:-3] + + # Get specified language specified_lang = self.code_block_text.split('\n', 1)[0] # Get 1st line (directly after delimiter, can be language)