diff --git a/test/prompt/invocation_layer/test_openai.py b/test/prompt/invocation_layer/test_openai.py index b068a9c9fb..63a47b31ad 100644 --- a/test/prompt/invocation_layer/test_openai.py +++ b/test/prompt/invocation_layer/test_openai.py @@ -53,7 +53,7 @@ def test_openai_token_limit_warning(mock_openai_tokenizer, caplog): @pytest.mark.parametrize( "model_name,max_tokens_limit", [ - ("gpt-3.5-turbo-instruct", 4097), + ("gpt-3.5-turbo-instruct", 4096), ("gpt-3.5-turbo", 4096), ("gpt-3.5-turbo-16k", 16384), ("gpt-4-32k", 32768), @@ -76,7 +76,7 @@ def test_openai_token_limit_warning_not_triggered(caplog, mock_openai_tokenizer, @pytest.mark.parametrize( "model_name,max_tokens_limit", [ - ("gpt-3.5-turbo-instruct", 4097), + ("gpt-3.5-turbo-instruct", 4096), ("gpt-3.5-turbo", 4096), ("gpt-3.5-turbo-16k", 16384), ("gpt-4-32k", 32768), diff --git a/test/utils/test_openai_utils.py b/test/utils/test_openai_utils.py index 4e62f3f265..92add5f219 100644 --- a/test/utils/test_openai_utils.py +++ b/test/utils/test_openai_utils.py @@ -23,8 +23,8 @@ def test_openai_text_completion_tokenization_details_gpt_default(): @pytest.mark.unit def test_openai_text_completion_tokenization_details_gpt_davinci(): tokenizer_name, max_tokens_limit = _openai_text_completion_tokenization_details(model_name="gpt-3.5-turbo-instruct") - assert tokenizer_name == "p50k_base" - assert max_tokens_limit == 4097 + assert tokenizer_name == "cl100k_base" + assert max_tokens_limit == 4096 @pytest.mark.unit