fix max token limit and tokenizer in tests

deepset-ai · Dec 28, 2023 · 5ad8ecf · 5ad8ecf
1 parent a02aafb
commit 5ad8ecf
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 4 deletions.
diff --git a/test/prompt/invocation_layer/test_openai.py b/test/prompt/invocation_layer/test_openai.py
@@ -53,7 +53,7 @@ def test_openai_token_limit_warning(mock_openai_tokenizer, caplog):
 @pytest.mark.parametrize(
     "model_name,max_tokens_limit",
     [
-        ("gpt-3.5-turbo-instruct", 4097),
+        ("gpt-3.5-turbo-instruct", 4096),
         ("gpt-3.5-turbo", 4096),
         ("gpt-3.5-turbo-16k", 16384),
         ("gpt-4-32k", 32768),
@@ -76,7 +76,7 @@ def test_openai_token_limit_warning_not_triggered(caplog, mock_openai_tokenizer,
 @pytest.mark.parametrize(
     "model_name,max_tokens_limit",
     [
-        ("gpt-3.5-turbo-instruct", 4097),
+        ("gpt-3.5-turbo-instruct", 4096),
         ("gpt-3.5-turbo", 4096),
         ("gpt-3.5-turbo-16k", 16384),
         ("gpt-4-32k", 32768),

diff --git a/test/utils/test_openai_utils.py b/test/utils/test_openai_utils.py
@@ -23,8 +23,8 @@ def test_openai_text_completion_tokenization_details_gpt_default():
 @pytest.mark.unit
 def test_openai_text_completion_tokenization_details_gpt_davinci():
     tokenizer_name, max_tokens_limit = _openai_text_completion_tokenization_details(model_name="gpt-3.5-turbo-instruct")
-    assert tokenizer_name == "p50k_base"
-    assert max_tokens_limit == 4097
+    assert tokenizer_name == "cl100k_base"
+    assert max_tokens_limit == 4096
 
 
 @pytest.mark.unit