Azure Openai is not well configured

[alabrashJr]

I’ve tried to use the autoevals.Factuality module with my Azure endpoint. I noticed the engine argument in the documentation, so I assumed that it corresponds to the model name for Azure OpenAI. However, using engine results in the following error:

      6 output = "People's Republic of China"
      7 expected = "China"
----> 9 result = evaluator(output, expected, input=input)

File /opt/conda/envs/py11/lib/python3.11/site-packages/braintrust_core/score.py:50, in Scorer.__call__(self, output, expected, **kwargs)
     49 def __call__(self, output, expected=None, **kwargs):
---> 50     return self.eval(output, expected, **kwargs)

File /opt/conda/envs/py11/lib/python3.11/site-packages/braintrust_core/score.py:47, in Scorer.eval(self, output, expected, **kwargs)
     46 def eval(self, output, expected=None, **kwargs):
---> 47     return self._run_eval_sync(output, expected, **kwargs)

File /opt/conda/envs/py11/lib/python3.11/site-packages/autoevals/llm.py:204, in OpenAILLMClassifier._run_eval_sync(self, output, expected, **kwargs)
    203 def _run_eval_sync(self, output, expected, **kwargs):
--> 204     return self._postprocess_response(run_cached_request(**self._request_args(output, expected, **kwargs)))

File /opt/conda/envs/py11/lib/python3.11/site-packages/autoevals/oai.py:120, in run_cached_request(request_type, api_key, base_url, **kwargs)
    118 while retries < 100:
    119     try:
--> 120         resp = post_process_response(getattr(wrapper, request_type)(**kwargs))
    121         break
    122     except wrapper.RateLimitError:

File /opt/conda/envs/py11/lib/python3.11/site-packages/openai/_utils/_utils.py:274, in required_args.<locals>.inner.<locals>.wrapper(*args, **kwargs)
    272             msg = f"Missing required argument: {quote(missing[0])}"
    273     raise TypeError(msg)
--> 274 return func(*args, **kwargs)

TypeError: Completions.create() got an unexpected keyword argument 'engine'

As far as I remember, the OpenAI SDK initially used the engine argument, but they later transitioned to using model, even for Azure OpenAI. I don’t think this has been updated on your side.

I tried using model argument as well but it gave the following error:

NotFoundError                             Traceback (most recent call last)
Cell In[87], line 9
      6 output = "People's Republic of China"
      7 expected = "China"
----> 9 result = evaluator(output, expected, input=input)

File /opt/conda/envs/py11/lib/python3.11/site-packages/braintrust_core/score.py:50, in Scorer.__call__(self, output, expected, **kwargs)
     49 def __call__(self, output, expected=None, **kwargs):
---> 50     return self.eval(output, expected, **kwargs)

File /opt/conda/envs/py11/lib/python3.11/site-packages/braintrust_core/score.py:47, in Scorer.eval(self, output, expected, **kwargs)
     46 def eval(self, output, expected=None, **kwargs):
---> 47     return self._run_eval_sync(output, expected, **kwargs)

File /opt/conda/envs/py11/lib/python3.11/site-packages/autoevals/llm.py:204, in OpenAILLMClassifier._run_eval_sync(self, output, expected, **kwargs)
    203 def _run_eval_sync(self, output, expected, **kwargs):
--> 204     return self._postprocess_response(run_cached_request(**self._request_args(output, expected, **kwargs)))

File /opt/conda/envs/py11/lib/python3.11/site-packages/autoevals/oai.py:120, in run_cached_request(request_type, api_key, base_url, **kwargs)
    118 while retries < 100:
    119     try:
--> 120         resp = post_process_response(getattr(wrapper, request_type)(**kwargs))
    121         break
    122     except wrapper.RateLimitError:

File /opt/conda/envs/py11/lib/python3.11/site-packages/openai/_utils/_utils.py:274, in required_args.<locals>.inner.<locals>.wrapper(*args, **kwargs)
    272             msg = f"Missing required argument: {quote(missing[0])}"
    273     raise TypeError(msg)
--> 274 return func(*args, **kwargs)

File /opt/conda/envs/py11/lib/python3.11/site-packages/openai/resources/chat/completions.py:679, in Completions.create(self, messages, model, frequency_penalty, function_call, functions, logit_bias, logprobs, max_tokens, n, parallel_tool_calls, presence_penalty, response_format, seed, service_tier, stop, stream, stream_options, temperature, tool_choice, tools, top_logprobs, top_p, user, extra_headers, extra_query, extra_body, timeout)
    644 @required_args(["messages", "model"], ["messages", "model", "stream"])
    645 def create(
    646     self,
   (...)
    676     timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
    677 ) -> ChatCompletion | Stream[ChatCompletionChunk]:
    678     validate_response_format(response_format)
--> 679     return self._post(
    680         "/chat/completions",
    681         body=maybe_transform(
    682             {
    683                 "messages": messages,
    684                 "model": model,
    685                 "frequency_penalty": frequency_penalty,
    686                 "function_call": function_call,
    687                 "functions": functions,
    688                 "logit_bias": logit_bias,
    689                 "logprobs": logprobs,
    690                 "max_tokens": max_tokens,
    691                 "n": n,
    692                 "parallel_tool_calls": parallel_tool_calls,
    693                 "presence_penalty": presence_penalty,
    694                 "response_format": response_format,
    695                 "seed": seed,
    696                 "service_tier": service_tier,
    697                 "stop": stop,
    698                 "stream": stream,
    699                 "stream_options": stream_options,
    700                 "temperature": temperature,
    701                 "tool_choice": tool_choice,
    702                 "tools": tools,
    703                 "top_logprobs": top_logprobs,
    704                 "top_p": top_p,
    705                 "user": user,
    706             },
    707             completion_create_params.CompletionCreateParams,
    708         ),
    709         options=make_request_options(
    710             extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
    711         ),
    712         cast_to=ChatCompletion,
    713         stream=stream or False,
    714         stream_cls=Stream[ChatCompletionChunk],
    715     )

File /opt/conda/envs/py11/lib/python3.11/site-packages/openai/_base_client.py:1260, in SyncAPIClient.post(self, path, cast_to, body, options, files, stream, stream_cls)
   1246 def post(
   1247     self,
   1248     path: str,
   (...)
   1255     stream_cls: type[_StreamT] | None = None,
   1256 ) -> ResponseT | _StreamT:
   1257     opts = FinalRequestOptions.construct(
   1258         method="post", url=path, json_data=body, files=to_httpx_files(files), **options
   1259     )
-> 1260     return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))

File /opt/conda/envs/py11/lib/python3.11/site-packages/openai/_base_client.py:937, in SyncAPIClient.request(self, cast_to, options, remaining_retries, stream, stream_cls)
    928 def request(
    929     self,
    930     cast_to: Type[ResponseT],
   (...)
    935     stream_cls: type[_StreamT] | None = None,
    936 ) -> ResponseT | _StreamT:
--> 937     return self._request(
    938         cast_to=cast_to,
    939         options=options,
    940         stream=stream,
    941         stream_cls=stream_cls,
    942         remaining_retries=remaining_retries,
    943     )

File /opt/conda/envs/py11/lib/python3.11/site-packages/openai/_base_client.py:1041, in SyncAPIClient._request(self, cast_to, options, remaining_retries, stream, stream_cls)
   1038         err.response.read()
   1040     log.debug("Re-raising status error")
-> 1041     raise self._make_status_error_from_response(err.response) from None
   1043 return self._process_response(
   1044     cast_to=cast_to,
   1045     options=options,
   (...)
   1049     retries_taken=options.get_max_retries(self.max_retries) - retries,
   1050 )

NotFoundError: Error code: 404 - {'error': {'code': '404', 'message': 'Resource not found'}}```

[ankrgyl]

We recently had someone contribute the updated version in JS: #91. We're very open to contributions if you want to look into the same in python!