Improve error message when LLM tool meets gpt-4-vision-preview model (#…

…2211)

# Description

**Extension tests:** If don't config the workspace triad, throw internal
error

![image](https://github.com/microsoft/promptflow/assets/75061414/fc5d2cf5-875a-45a8-8edc-61fe9c55e1c0)



**Portal tests:** tool internal package pip install --upgrade
promptflow_tools[azure]==0.0.313 --extra-index-url
https://azuremlsdktestpypi.azureedge.net/test-promptflow/
Runtime:
https://ml.azure.com/prompts/runtime/llm-use-vision-model-test/details?wsid=/subscriptions/96aede12-2f73-41cb-b983-6d11a904839b/resourceGroups/chesi-eastus/providers/Microsoft.MachineLearningServices/workspaces/chesi-promptflow&tid=72f988bf-86f1-41af-91ab-2d7cd011db47
Experiment link:
https://ml.azure.com/prompts/flow/cfa1c054-13cb-4cfc-b3a6-359bf0493da9/a65f5e8f-342c-43b0-8eb0-c391d59666da/details?wsid=/subscriptions/96aede12-2f73-41cb-b983-6d11a904839b/resourceGroups/chesi-eastus/providers/Microsoft.MachineLearningServices/workspaces/chesi-promptflow&tid=72f988bf-86f1-41af-91ab-2d7cd011db47

1. stop/response_format/logit_bias/max_tokens are none:

![image](https://github.com/microsoft/promptflow/assets/75061414/4aa1edc4-0224-457b-9b99-56d892f7d1eb)

2. llm use vision with extra parameter:

![image](https://github.com/microsoft/promptflow/assets/75061414/143bd8d1-99e8-4c03-b311-cee4c4ed1065)

3. only vision deployment name shows for vision tool

![image](https://github.com/microsoft/promptflow/assets/75061414/1c795e7c-895f-4717-bc66-1b2d4036a405)



# All Promptflow Contribution checklist:
- [ ] **The pull request does not introduce [breaking changes].**
- [ ] **CHANGELOG is updated for new features, bug fixes or other
significant changes.**
- [ ] **I have read the [contribution guidelines](../CONTRIBUTING.md).**
- [ ] **Create an issue and link to the pull request to get dedicated
review from promptflow team. Learn more: [suggested
workflow](../CONTRIBUTING.md#suggested-workflow).**

## General Guidelines and Best Practices
- [ ] Title of the pull request is clear and informative.
- [ ] There are a small number of commits, each of which have an
informative message. This means that previously merged commits do not
appear in the history of the PR. For more information on cleaning up the
commits in your PR, [see this
page](https://github.com/Azure/azure-powershell/blob/master/documentation/development-docs/cleaning-up-commits.md).

### Testing Guidelines
- [ ] Pull request includes test coverage for the included changes.

---------

Co-authored-by: cs_lucky <[email protected]>

src/promptflow-tools/promptflow/tools/aoai.py

@@ -129,21 +129,28 @@ def chat(
             "top_p": float(top_p),
             "n": int(n),
             "stream": stream,
-            "stop": stop if stop else None,
-            "max_tokens": int(max_tokens) if max_tokens is not None and str(max_tokens).lower() != "inf" else None,
             "presence_penalty": float(presence_penalty),
             "frequency_penalty": float(frequency_penalty),
-            "logit_bias": logit_bias,
             "user": user,
-            "response_format": response_format,
-            "seed": seed,
             "extra_headers": {"ms-azure-ai-promptflow-called-from": "aoai-tool"}
         }
         if functions is not None:
             validate_functions(functions)
             params["functions"] = functions
             params["function_call"] = process_function_call(function_call)
 
+        # to avoid vision model validation error for empty param values.
+        if stop:
+            params["stop"] = stop
+        if max_tokens is not None and str(max_tokens).lower() != "inf":
+            params["max_tokens"] = int(max_tokens)
+        if logit_bias:
+            params["logit_bias"] = logit_bias
+        if response_format:
+            params["response_format"] = response_format
+        if seed is not None:
+            params["seed"] = seed
+
         completion = self._client.chat.completions.create(**params)
         return post_process_chat_api_response(completion, stream, functions)
 

src/promptflow-tools/promptflow/tools/aoai_gpt4v.py

@@ -1,70 +1,12 @@
-from promptflow._internal import ToolProvider, tool
-from promptflow.connections import AzureOpenAIConnection
-from promptflow.contracts.types import PromptTemplate
-from promptflow.exceptions import ErrorTarget, UserErrorException
 from typing import List, Dict
 
 from promptflow.tools.common import render_jinja_template, handle_openai_error, parse_chat, \
     preprocess_template_string, find_referenced_image_set, convert_to_chat_list, init_azure_openai_client, \
-    post_process_chat_api_response
-
-
-GPT4V_VERSION = "vision-preview"
-
-
-def _get_credential():
-    from azure.identity import DefaultAzureCredential
-    from azure.ai.ml._azure_environments import _get_default_cloud_name, EndpointURLS, _get_cloud, AzureEnvironments
-    # Support sovereign cloud cases, like mooncake, fairfax.
-    cloud_name = _get_default_cloud_name()
-    if cloud_name != AzureEnvironments.ENV_DEFAULT:
-        cloud = _get_cloud(cloud=cloud_name)
-        authority = cloud.get(EndpointURLS.ACTIVE_DIRECTORY_ENDPOINT)
-        credential = DefaultAzureCredential(authority=authority, exclude_shared_token_cache_credential=True)
-    else:
-        credential = DefaultAzureCredential()
-
-    return credential
-
-
-def _parse_resource_id(resource_id):
-    # Resource id is connection's id in following format:
-    # "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.CognitiveServices/accounts/{account}"
-    split_parts = resource_id.split("/")
-    if len(split_parts) != 9:
-        raise ParseConnectionError(
-            f"Connection resourceId format invalid, cur resourceId is {resource_id}."
-        )
-    sub, rg, account = split_parts[2], split_parts[4], split_parts[-1]
-
-    return sub, rg, account
-
-
-class Deployment:
-    def __init__(
-        self,
-        name: str,
-        model_name: str,
-        version: str
-    ):
-        self.name = name
-        self.model_name = model_name
-        self.version = version
-
-
-class ListDeploymentsError(UserErrorException):
-    def __init__(self, msg, **kwargs):
-        super().__init__(msg, target=ErrorTarget.TOOL, **kwargs)
-
+    post_process_chat_api_response, list_deployment_connections, build_deployment_dict, GPT4V_VERSION
 
-class ParseConnectionError(ListDeploymentsError):
-    def __init__(self, msg, **kwargs):
-        super().__init__(msg, **kwargs)
-
-
-def _build_deployment_dict(item) -> Deployment:
-    model = item.properties.model
-    return Deployment(item.name, model.name, model.version)
+from promptflow._internal import ToolProvider, tool
+from promptflow.connections import AzureOpenAIConnection
+from promptflow.contracts.types import PromptTemplate
 
 
 def list_deployment_names(
@@ -74,65 +16,19 @@ def list_deployment_names(
     connection=""
 ) -> List[Dict[str, str]]:
     res = []
-    try:
-        # Does not support dynamic list for local.
-        from azure.mgmt.cognitiveservices import CognitiveServicesManagementClient
-        from promptflow.azure.operations._arm_connection_operations import \
-            ArmConnectionOperations, OpenURLFailedUserError
-    except ImportError:
-        return res
-    # For local, subscription_id is None. Does not suppot dynamic list for local.
-    if not subscription_id:
+    deployment_collection = list_deployment_connections(subscription_id, resource_group_name, workspace_name,
+                                                        connection)
+    if not deployment_collection:
         return res
 
-    try:
-        credential = _get_credential()
-        try:
-            # Currently, the param 'connection' is str, not AzureOpenAIConnection type.
-            conn = ArmConnectionOperations._build_connection_dict(
-                name=connection,
-                subscription_id=subscription_id,
-                resource_group_name=resource_group_name,
-                workspace_name=workspace_name,
-                credential=credential
-            )
-            resource_id = conn.get("value").get('resource_id', "")
-            if not resource_id:
-                return res
-            conn_sub, conn_rg, conn_account = _parse_resource_id(resource_id)
-        except OpenURLFailedUserError:
-            return res
-        except ListDeploymentsError as e:
-            raise e
-        except Exception as e:
-            msg = f"Parsing connection with exception: {e}"
-            raise ListDeploymentsError(msg=msg) from e
-
-        client = CognitiveServicesManagementClient(
-            credential=credential,
-            subscription_id=conn_sub,
-        )
-        deployment_collection = client.deployments.list(
-            resource_group_name=conn_rg,
-            account_name=conn_account,
-        )
-
-        for item in deployment_collection:
-            deployment = _build_deployment_dict(item)
-            if deployment.version == GPT4V_VERSION:
-                cur_item = {
-                    "value": deployment.name,
-                    "display_value": deployment.name,
-                }
-                res.append(cur_item)
-
-    except Exception as e:
-        if hasattr(e, 'status_code') and e.status_code == 403:
-            msg = f"Failed to list deployments due to permission issue: {e}"
-            raise ListDeploymentsError(msg=msg) from e
-        else:
-            msg = f"Failed to list deployments with exception: {e}"
-            raise ListDeploymentsError(msg=msg) from e
+    for item in deployment_collection:
+        deployment = build_deployment_dict(item)
+        if deployment.version == GPT4V_VERSION:
+            cur_item = {
+                "value": deployment.name,
+                "display_value": deployment.name,
+            }
+            res.append(cur_item)
 
     return res
 

src/promptflow-tools/promptflow/tools/common.py

@@ -1,19 +1,37 @@
 import functools
 import json
+import os
 import re
 import sys
 import time
 from typing import List, Mapping
 
 from jinja2 import Template
-from openai import APIConnectionError, APIStatusError, OpenAIError, RateLimitError, APITimeoutError
+from openai import APIConnectionError, APIStatusError, OpenAIError, RateLimitError, APITimeoutError, BadRequestError
 from promptflow.tools.exception import ChatAPIInvalidRole, WrappedOpenAIError, LLMError, JinjaTemplateError, \
     ExceedMaxRetryTimes, ChatAPIInvalidFunctions, FunctionCallNotSupportedInStreamMode, \
-    ChatAPIFunctionRoleInvalidFormat, InvalidConnectionType
+    ChatAPIFunctionRoleInvalidFormat, InvalidConnectionType, ListDeploymentsError, ParseConnectionError
+
+from promptflow._cli._utils import get_workspace_triad_from_local
 from promptflow.connections import AzureOpenAIConnection, OpenAIConnection
 from promptflow.exceptions import SystemErrorException, UserErrorException
 
 
+GPT4V_VERSION = "vision-preview"
+
+
+class Deployment:
+    def __init__(
+            self,
+            name: str,
+            model_name: str,
+            version: str
+    ):
+        self.name = name
+        self.model_name = model_name
+        self.version = version
+
+
 class ChatInputList(list):
     """
     ChatInputList is a list of ChatInput objects. It is used to override the __str__ method of list to return a string
@@ -188,6 +206,134 @@ def generate_retry_interval(retry_count: int) -> float:
     return retry_interval
 
 
+def build_deployment_dict(item) -> Deployment:
+    model = item.properties.model
+    return Deployment(item.name, model.name, model.version)
+
+
+def _parse_resource_id(resource_id):
+    # Resource id is connection's id in following format:
+    # "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.CognitiveServices/accounts/{account}"
+    split_parts = resource_id.split("/")
+    if len(split_parts) != 9:
+        raise ParseConnectionError(
+            f"Connection resourceId format invalid, cur resourceId is {resource_id}."
+        )
+    sub, rg, account = split_parts[2], split_parts[4], split_parts[-1]
+
+    return sub, rg, account
+
+
+def _get_credential():
+    from azure.identity import DefaultAzureCredential
+    from azure.ai.ml._azure_environments import _get_default_cloud_name, EndpointURLS, _get_cloud, AzureEnvironments
+    # Support sovereign cloud cases, like mooncake, fairfax.
+    cloud_name = _get_default_cloud_name()
+    if cloud_name != AzureEnvironments.ENV_DEFAULT:
+        cloud = _get_cloud(cloud=cloud_name)
+        authority = cloud.get(EndpointURLS.ACTIVE_DIRECTORY_ENDPOINT)
+        credential = DefaultAzureCredential(authority=authority, exclude_shared_token_cache_credential=True)
+    else:
+        credential = DefaultAzureCredential()
+
+    return credential
+
+
+def get_workspace_triad():
+    # If flow is submitted from cloud, runtime will save the workspace triad to environment
+    if 'AZUREML_ARM_SUBSCRIPTION' in os.environ and 'AZUREML_ARM_RESOURCEGROUP' in os.environ \
+            and 'AZUREML_ARM_WORKSPACE_NAME' in os.environ:
+        return os.environ["AZUREML_ARM_SUBSCRIPTION"], os.environ["AZUREML_ARM_RESOURCEGROUP"], \
+               os.environ["AZUREML_ARM_WORKSPACE_NAME"]
+    else:
+        # If flow is submitted from local, it will get workspace triad from your azure cloud config file
+        # If this config file isn't set up, it will return None.
+        workspace_triad = get_workspace_triad_from_local()
+        return workspace_triad.subscription_id, workspace_triad.resource_group_name, workspace_triad.workspace_name
+
+
+def list_deployment_connections(
+    subscription_id,
+    resource_group_name,
+    workspace_name,
+    connection="",
+):
+    try:
+        # Do not support dynamic list for local.
+        from azure.mgmt.cognitiveservices import CognitiveServicesManagementClient
+        from promptflow.azure.operations._arm_connection_operations import \
+            ArmConnectionOperations, OpenURLFailedUserError
+    except ImportError:
+        return None
+
+    # For local, subscription_id is None. Does not support dynamic list for local.
+    if not subscription_id:
+        return None
+
+    try:
+        credential = _get_credential()
+        try:
+            # Currently, the param 'connection' is str, not AzureOpenAIConnection type.
+            conn = ArmConnectionOperations._build_connection_dict(
+                name=connection,
+                subscription_id=subscription_id,
+                resource_group_name=resource_group_name,
+                workspace_name=workspace_name,
+                credential=credential
+            )
+            resource_id = conn.get("value").get('resource_id', "")
+            if not resource_id:
+                return None
+            conn_sub, conn_rg, conn_account = _parse_resource_id(resource_id)
+        except OpenURLFailedUserError:
+            return None
+        except ListDeploymentsError as e:
+            raise e
+        except Exception as e:
+            msg = f"Parsing connection with exception: {e}"
+            raise ListDeploymentsError(msg=msg) from e
+
+        client = CognitiveServicesManagementClient(
+            credential=credential,
+            subscription_id=conn_sub,
+        )
+        return client.deployments.list(
+            resource_group_name=conn_rg,
+            account_name=conn_account,
+        )
+    except Exception as e:
+        if hasattr(e, 'status_code') and e.status_code == 403:
+            msg = f"Failed to list deployments due to permission issue: {e}"
+            raise ListDeploymentsError(msg=msg) from e
+        else:
+            msg = f"Failed to list deployments with exception: {e}"
+            raise ListDeploymentsError(msg=msg) from e
+
+
+def refine_extra_fields_not_permitted_error(connection, deployment_name, model):
+    tsg = "Please kindly avoid using vision model in LLM tool, " \
+          "because vision model cannot work with some chat api parameters. " \
+          "You can change to use tool 'Azure OpenAI GPT-4 Turbo with Vision' " \
+          "or 'OpenAI GPT-4V' for vision model."
+    try:
+        if isinstance(connection, AzureOpenAIConnection):
+            subscription_id, resource_group, workspace_name = get_workspace_triad()
+            if subscription_id and resource_group and workspace_name:
+                deployment_collection = list_deployment_connections(subscription_id, resource_group, workspace_name,
+                                                                    connection.name)
+                for item in deployment_collection:
+                    if deployment_name == item.name:
+                        if item.properties.model.version in [GPT4V_VERSION]:
+                            return tsg
+        elif isinstance(connection, OpenAIConnection) and model in ["gpt-4-vision-preview"]:
+            return tsg
+    except Exception as e:
+        print(f"Exception occurs when refine extra fields not permitted error for llm: "
+              f"{type(e).__name__}: {str(e)}", file=sys.stderr)
+
+    return None
+
+
 # TODO(2971352): revisit this tries=100 when there is any change to the 10min timeout logic
 def handle_openai_error(tries: int = 100):
     """
@@ -212,6 +358,19 @@ def wrapper(*args, **kwargs):
                     #  Handle retriable exception, please refer to
                     #  https://platform.openai.com/docs/guides/error-codes/api-errors
                     print(f"Exception occurs: {type(e).__name__}: {str(e)}", file=sys.stderr)
+                    # Vision model does not support all chat api parameters, e.g. response_format and function_call.
+                    # Recommend user to use vision model in vision tools, rather than LLM tool.
+                    # Related issue https://github.com/microsoft/promptflow/issues/1683
+                    if isinstance(e, BadRequestError) and "extra fields not permitted" in str(e).lower():
+                        refined_error_message = \
+                            refine_extra_fields_not_permitted_error(args[0].connection,
+                                                                    kwargs.get("deployment_name", ""),
+                                                                    kwargs.get("model", ""))
+                        if refined_error_message:
+                            raise LLMError(message=f"{str(e)} {refined_error_message}")
+                        else:
+                            raise WrappedOpenAIError(e)
+
                     if isinstance(e, APIConnectionError) and not isinstance(e, APITimeoutError) \
                             and "connection aborted" not in str(e).lower():
                         raise WrappedOpenAIError(e)

src/promptflow-tools/promptflow/tools/exception.py

@@ -198,3 +198,13 @@ class AzureContentSafetySystemError(SystemErrorException):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs, target=ErrorTarget.TOOL)
+
+
+class ListDeploymentsError(UserErrorException):
+    def __init__(self, msg, **kwargs):
+        super().__init__(msg, target=ErrorTarget.TOOL, **kwargs)
+
+
+class ParseConnectionError(ListDeploymentsError):
+    def __init__(self, msg, **kwargs):
+        super().__init__(msg, **kwargs)