Merge branch 'main' of https://github.com/microsoft/promptflow into l…

…iucheng/20240430_generator_span

.github/workflows/promptflow-evals-e2e-test.yml

@@ -60,6 +60,7 @@ jobs:
           poetry run pip install -e ../promptflow-devkit
           poetry run pip install -e ../promptflow-tracing
           poetry run pip install -e ../promptflow-tools
+          poetry run pip install -e ../promptflow-azure
         working-directory: ${{ env.WORKING_DIRECTORY }}
       - name: install promptflow-evals from wheel
         # wildcard expansion (*) does not work in Windows, so leverage python to find and install

.github/workflows/promptflow-evals-unit-test.yml

@@ -56,6 +56,7 @@ jobs:
           poetry run pip install -e ../promptflow-devkit
           poetry run pip install -e ../promptflow-tracing
           poetry run pip install -e ../promptflow-tools
+          poetry run pip install -e ../promptflow-azure
         working-directory: ${{ env.WORKING_DIRECTORY }}
       - name: install promptflow-evals from wheel
         # wildcard expansion (*) does not work in Windows, so leverage python to find and install

.github/workflows/tools_tests.yml

@@ -1,10 +1,16 @@
 name: tools_tests
+permissions:
+  # This is required for requesting the JWT
+  id-token: write
+  # This is required for actions/checkout
+  contents: read
 on:
+  workflow_dispatch:
   pull_request_target:
     paths:
       - src/promptflow-tools/**
-      - '**tools_tests.yml'
-  workflow_dispatch:
+      - scripts/tool/**
+      - .github/workflows/tools_tests.yml
 jobs:
   authorize:
     environment:
@@ -53,9 +59,17 @@ jobs:
             pip install azure-mgmt-cognitiveservices==13.5.0
           fi
           pip list
+
+      - name: Azure login
+        uses: azure/login@v1
+        with:
+          client-id: ${{ secrets.CLIENT_ID }}
+          tenant-id: ${{ secrets.TENANT_ID }}
+          subscription-id: ${{ secrets.TEST_WORKSPACE_SUB_ID }}
+
       - name: Generate configs
         run: |
-          python ./scripts/tool/generate_connection_config.py --tenant_id ${{ secrets.TENANT_ID }} --client_id ${{ secrets.CLIENT_ID }} --client_secret ${{ secrets.CLIENT_SECRET }}
+          python ./scripts/tool/generate_connection_config.py
 
       - name: Run tests
         run: |

examples/flows/evaluation/eval-chat-math/flow.dag.yaml

@@ -1,3 +1,4 @@
+$schema: https://azuremlschemas.azureedge.net/promptflow/latest/Flow.schema.json
 inputs:
   groundtruth:
     type: string
@@ -31,4 +32,5 @@ nodes:
   aggregation: true
   use_variants: false
 node_variants: {}
-$schema: https://azuremlschemas.azureedge.net/promptflow/latest/Flow.schema.json
+environment:
+  python_requirements_txt: requirements.txt

scripts/docs/conf.py

@@ -59,6 +59,8 @@
 
 # Options for the linkcheck builder
 linkcheck_ignore = [
+    # openai related sites blocks the IP of the CI server.
+    r"https://openai\.com/",
     r"https://platform\.openai\.com/",
     r"https://help\.openai\.com/",
     # These are used in card links, for example 'xx.html', .md can't be resolved.

scripts/installer/windows/scripts/promptflow.spec.jinja2

@@ -17,17 +17,19 @@ for package in meta_packages:
     datas += copy_metadata(package)
 
 opentelemetry_datas, opentelemetry_binaries, opentelemetry_hiddenimports = collect_all('opentelemetry')
+promptflow_datas, promptflow_binaries, promptflow_hiddenimports = collect_all('promptflow')
 datas += opentelemetry_datas
+datas += promptflow_datas
 datas += collect_data_files('streamlit_quill')
-datas += collect_data_files('promptflow')
-datas += copy_metadata('promptflow')
-datas += collect_data_files('promptflow-evals')
-datas += copy_metadata('promptflow-evals')
+
 hidden_imports = ['win32timezone', 'promptflow', 'opentelemetry.context.contextvars_context', 'streamlit.runtime.scriptrunner.magic_funcs'] + {{hidden_imports}}
 
 hidden_imports += opentelemetry_hiddenimports
+hidden_imports += promptflow_hiddenimports
+
 binaries = []
 binaries += opentelemetry_binaries
+binaries += promptflow_binaries
 
 block_cipher = None
 

scripts/json_schema/gen_json_schema.py

@@ -142,7 +142,7 @@ def get_required(self, obj):
 from promptflow._sdk.schemas._connection import AzureOpenAIConnectionSchema, OpenAIConnectionSchema, \
 QdrantConnectionSchema, CognitiveSearchConnectionSchema, SerpConnectionSchema, AzureContentSafetyConnectionSchema, \
 FormRecognizerConnectionSchema, CustomConnectionSchema, WeaviateConnectionSchema, ServerlessConnectionSchema, \
-CustomStrongTypeConnectionSchema
+CustomStrongTypeConnectionSchema, AzureAIServicesConnectionSchema
 from promptflow._sdk.schemas._run import RunSchema
 from promptflow._sdk.schemas._flow import FlowSchema, FlexFlowSchema
 
@@ -163,7 +163,7 @@ def get_required(self, obj):
         args.output_file = ["Run", "Flow", "AzureOpenAIConnection", "OpenAIConnection", "QdrantConnection",
                             "CognitiveSearchConnection", "SerpConnection", "AzureContentSafetyConnection",
                             "FormRecognizerConnection", "CustomConnection", "WeaviateConnection", "ServerlessConnection",
-                            "CustomStrongTypeConnection"]
+                            "CustomStrongTypeConnection", "AzureAIServicesConnection"]
 
     # Special case for Flow and EagerFlow
     if "Flow" in args.output_file:

scripts/tool/generate_connection_config.py

@@ -21,9 +21,6 @@ def fill_key_to_dict(template_dict, keys_dict):
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
-    parser.add_argument("--tenant_id", type=str, help="The tenant id of the service principal")
-    parser.add_argument("--client_id", type=str, help="The client id of the service principal")
-    parser.add_argument("--client_secret", type=str, help="The client secret of the service principal")
     parser.add_argument("--local", action='store_true', help="local debug mode")
     args = parser.parse_args()
 
@@ -32,7 +29,7 @@ def fill_key_to_dict(template_dict, keys_dict):
     print(f"file_path: {file_path}")
 
     if not args.local:
-        client = get_secret_client(tenant_id=args.tenant_id, client_id=args.client_id, client_secret=args.client_secret)
+        client = get_secret_client()
         all_secret_names = list_secret_names(client)
         data = {secret_name: get_secret(secret_name, client) for secret_name in all_secret_names}
 

scripts/tool/utils/secret_manager.py

@@ -1,36 +1,17 @@
-import re
-
-from azure.core.exceptions import HttpResponseError, ResourceExistsError
-from azure.identity import ClientSecretCredential
+from azure.identity import AzureCliCredential
 from azure.keyvault.secrets import SecretClient
-from exceptions import (
-    SecretNameAlreadyExistsException,
-    SecretNameInvalidException,
-    SecretNoSetPermissionException,
-)
 
 key_vault_name = "github-promptflow"
-container_name = "tools"
 KVUri = f"https://{key_vault_name}.vault.azure.net"
 
 
-def init_used_secret_names(client: SecretClient):
-    global reserved_secret_names
-    reserved_secret_names = list_secret_names(client)
-
-
-def get_secret_client(
-    tenant_id: str, client_id: str, client_secret: str
-) -> SecretClient:
-    credential = ClientSecretCredential(tenant_id, client_id, client_secret)
+def get_secret_client() -> SecretClient:
+    credential = AzureCliCredential()
     client = SecretClient(vault_url=KVUri, credential=credential)
 
     return client
 
 
-reserved_secret_names = []
-
-
 def get_secret(secret_name: str, client: SecretClient):
     secret = client.get_secret(secret_name)
 
@@ -41,37 +22,3 @@ def list_secret_names(client: SecretClient) -> list:
     secret_properties = client.list_properties_of_secrets()
 
     return [secret.name for secret in secret_properties]
-
-
-def validate_secret_name(secret_name: str):
-    # Check if secret name is valid. Secret name can only contain alphanumeric characters and dashes.
-    pattern = "^[a-zA-Z0-9-]+$"
-    if not re.match(pattern, secret_name):
-        raise SecretNameInvalidException(
-            "Secret name can only contain alphanumeric characters and dashes"
-        )
-    # Check if secret name is one of the reserved names
-    if secret_name in reserved_secret_names:
-        raise SecretNameAlreadyExistsException(
-            f"Secret name {secret_name} already exists"
-        )
-
-
-def upload_secret(client: SecretClient, secret_name: str, secret_value: str):
-    try:
-        client.set_secret(secret_name, secret_value)
-    except ResourceExistsError as ex:
-        if "in a deleted but recoverable state" in str(ex):
-            raise SecretNameAlreadyExistsException(
-                f"Secret name {secret_name} is deleted but recoverable, and its name cannot be reused"
-            )
-    except HttpResponseError as ex:
-        if (
-            ex.status_code == 403
-            and "does not have secrets set permission on key vault" in str(ex)
-        ):
-            raise SecretNoSetPermissionException(
-                f"No set permission on key vault {key_vault_name}"
-            )
-
-    print("Done.")

src/promptflow-azure/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "promptflow-azure"
-version = "1.10.0.dev0"
+version = "1.11.0.dev0"
 description = "Prompt flow azure"
 include = [
   "promptflow/azure/resources/*"

src/promptflow-core/promptflow/_constants.py

@@ -171,6 +171,9 @@ class SpanAttributeFieldName:
     COMPLETION_TOKEN_COUNT = "__computed__.cumulative_token_count.completion"
     PROMPT_TOKEN_COUNT = "__computed__.cumulative_token_count.prompt"
     TOTAL_TOKEN_COUNT = "__computed__.cumulative_token_count.total"
+    # Execution target, e.g. prompty, flex, dag, code.
+    # We may need another field to indicate the language, e.g. python, csharp.
+    EXECUTION_TARGET = "execution_target"
 
     SESSION_ID = "session_id"
 
@@ -230,6 +233,7 @@ class ConnectionType(str, Enum):
     COGNITIVE_SEARCH = "CognitiveSearch"
     SERP = "Serp"
     AZURE_CONTENT_SAFETY = "AzureContentSafety"
+    AZURE_AI_SERVICES = "AzureAIServices"
     FORM_RECOGNIZER = "FormRecognizer"
     WEAVIATE = "Weaviate"
     SERVERLESS = "Serverless"