Merge remote-tracking branch 'origin/main' into users/ninhu/async_eva…

…luators

.github/workflows/promptflow-evals-e2e-test.yml

@@ -83,7 +83,10 @@ jobs:
           creds: ${{ secrets.PF_EVALS_SP_CREDENTIALS }}
           enable-AzPSSession: true
       - name: run e2e tests
-        run: poetry run pytest -m e2etest --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml
+        id: run_all_e2e_tests
+        run: | 
+          poetry run pytest -m e2etest --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml
+          poetry run python ../../scripts/code_qa/report_to_app_insights.py --activity all_e2e_tests_run_times --junit-xml test-results.xml --git-hub-action-run-id ${{ github.run_id }} --git-hub-workflow ${{ github.workflow }} --git-hub-action ${{ github.action }} --git-branch ${{ github.ref }}
         working-directory: ${{ env.WORKING_DIRECTORY }}
       - name: upload coverage report
         uses: actions/upload-artifact@v4

.github/workflows/promptflow-evals-regression-test.yml

@@ -0,0 +1,91 @@
+name: promptflow-evals-regression-test
+
+on:
+  schedule:
+    - cron: "40 10 * * *" # 2:40 PST every day
+  pull_request:
+    paths:
+      - src/promptflow-evals/**
+      - .github/workflows/promptflow-evals-regression-test.yml
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+  WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-evals
+  PROMPT_FLOW_TEST_MODE: "live"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: snok/install-poetry@v1
+    - name: build
+      run: poetry build
+      working-directory: ${{ env.WORKING_DIRECTORY }}
+    - uses: actions/upload-artifact@v4
+      with:
+        name: promptflow-evals
+        path: ${{ env.WORKING_DIRECTORY }}/dist/promptflow_evals-*.whl
+
+  test:
+    needs: build
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-13]
+        # TODO: Encounter hash mismatch for ubuntu-latest and 3.9 combination during installing promptflow-evals package
+        # https://github.com/microsoft/promptflow/actions/runs/9009397933/job/24753518853?pr=3158
+        # Add 3.9 back after we figure out the issue
+        python-version: ['3.8', '3.10', '3.11']
+      fail-fast: false
+    # snok/install-poetry need this to support Windows
+    defaults:
+      run:
+        shell: bash
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: snok/install-poetry@v1
+      - uses: actions/download-artifact@v4
+        with:
+          name: promptflow-evals
+          path: ${{ env.WORKING_DIRECTORY }}
+      - name: install test dependency group
+        run: poetry install --only test
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: install promptflow-evals from wheel
+        id: install_promptflow
+        run: |
+          # Estimate the installation time.
+          export start_tm=`date +%s`
+          poetry run pip install -e ../promptflow
+          poetry run pip install -e ../promptflow-core
+          poetry run pip install -e ../promptflow-devkit
+          poetry run pip install -e ../promptflow-tracing
+          poetry run pip install -e ../promptflow-tools
+          poetry run pip install -e ../promptflow-azure
+          poetry run pip install --pre $(python -c "import glob; print(glob.glob('promptflow_evals-*.whl')[0])")
+          export install_time=$((`date +%s` - ${start_tm}))
+          poetry run python ../../scripts/code_qa/report_to_app_insights.py --activity install_time_s --value $install_time --git-hub-action-run-id ${{ github.run_id }} --git-hub-workflow ${{ github.workflow }} --git-hub-action ${{ github.action }} --git-branch ${{ github.ref }}
+          test ${install_time} -le $TIME_LIMIT || echo "::warning file=pyproject.toml,line=40,col=0::The installation took ${install_time} minutes, the limit is ${TIME_LIMIT}."
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: install recording
+        run: poetry run pip install -e ../promptflow-recording
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: generate end-to-end test config from secret
+        run: echo '${{ secrets.PF_EVALS_E2E_TEST_CONFIG }}' >> connections.json
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - uses: azure/login@v2
+        with:
+          creds: ${{ secrets.PF_EVALS_SP_CREDENTIALS }}
+          enable-AzPSSession: true
+      - name: run performance tests
+        id: performance_tests
+        run: |
+          # Estimate the run time for evaluator.
+          poetry run pytest -m performance_test --junit-xml=test-results.xml
+          poetry run python ../../scripts/code_qa/report_to_app_insights.py --activity evaluator_live_tests_run_time_s --junit-xml test-results.xml --git-hub-action-run-id ${{ github.run_id }} --git-hub-workflow ${{ github.workflow }} --git-hub-action ${{ github.action }} --git-branch ${{ github.ref }}
+        working-directory: ${{ env.WORKING_DIRECTORY }}

.github/workflows/promptflow-evals-unit-test.yml

@@ -72,7 +72,9 @@ jobs:
         run: poetry run pip install -e ../promptflow-recording
         working-directory: ${{ env.WORKING_DIRECTORY }}
       - name: run unit tests
-        run: poetry run pytest -m unittest --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml --cov-fail-under=63
+        id: run_unit_tests
+        run: |
+            poetry run pytest -m unittest --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml --cov-fail-under=63
         working-directory: ${{ env.WORKING_DIRECTORY }}
       - name: upload coverage report
         uses: actions/upload-artifact@v4

README.md

@@ -23,7 +23,7 @@ With prompt flow, you will be able to:
 
 - **Create and iteratively develop flow**
     - Create executable [flows](https://microsoft.github.io/promptflow/concepts/concept-flows.html) that link LLMs, prompts, Python code and other [tools](https://microsoft.github.io/promptflow/concepts/concept-tools.html) together.
-    - Debug and iterate your flows, especially the [interaction with LLMs](https://microsoft.github.io/promptflow/concepts/concept-connections.html) with ease.
+    - Debug and iterate your flows, especially [tracing interaction with LLMs](https://microsoft.github.io/promptflow/how-to-guides/tracing/index.html) with ease.
 - **Evaluate flow quality and performance**
     - Evaluate your flow's quality and performance with larger datasets.
     - Integrate the testing and evaluation into your CI/CD system to ensure quality of your flow.

benchmark/promptflow-serve/pf_flows/static_async/node1.py

@@ -5,6 +5,6 @@
 @tool
 async def my_python_tool(chat_history: list, question: str) -> str:
 
-    # sleep for 250ms to simulate open ai call async
+    # sleep for 250ms to simulate OpenAI call async
     await asyncio.sleep(0.25)
     return "completed"

benchmark/promptflow-serve/pf_flows/static_async/node2.py

@@ -5,6 +5,6 @@
 @tool
 async def my_python_tool(chat_history: list, question: str) -> str:
 
-    # sleep for 250ms to simulate open ai call async
+    # sleep for 250ms to simulate OpenAI call async
     await asyncio.sleep(0.25)
     return "completed"

benchmark/promptflow-serve/pf_flows/static_async/node3.py

@@ -5,6 +5,6 @@
 @tool
 async def my_python_tool(chat_history: list, question: str) -> str:
 
-    # sleep for 250ms to simulate open ai call async
+    # sleep for 250ms to simulate OpenAI call async
     await asyncio.sleep(0.25)
     return "completed"

benchmark/promptflow-serve/pf_flows/static_sync/node1.py

@@ -5,6 +5,6 @@
 @tool
 def my_python_tool(chat_history: list, question: str) -> str:
 
-    # sleep for 250ms to simulate open ai call
+    # sleep for 250ms to simulate OpenAI call
     time.sleep(0.25)
     return "completed"

benchmark/promptflow-serve/pf_flows/static_sync/node2.py

@@ -5,6 +5,6 @@
 @tool
 def my_python_tool(chat_history: list, question: str) -> str:
 
-    # sleep for 250ms to simulate open ai call
+    # sleep for 250ms to simulate OpenAI call
     time.sleep(0.25)
     return "completed"

benchmark/promptflow-serve/pf_flows/static_sync/node3.py

@@ -5,6 +5,6 @@
 @tool
 def my_python_tool(chat_history: list, question: str) -> str:
 
-    # sleep for 250ms to simulate open ai call
+    # sleep for 250ms to simulate OpenAI call
     time.sleep(0.25)
     return "completed"

docs/concepts/concept-connections.md

@@ -7,12 +7,12 @@ Connections are for storing information about how to access external services li
 - In your local development environment, the connections are persisted in your local machine with keys encrypted.
 - In Azure AI, connections can be configured to be shared across the entire workspace. Secrets associated with connections are securely persisted in the corresponding Azure Key Vault, adhering to robust security and compliance standards.
 
-Prompt flow provides a variety of pre-built connections, including Azure Open AI, Open AI, etc. These pre-built connections enable seamless integration with these resources within the built-in tools. Additionally, you have the flexibility to create custom connection types using key-value pairs, empowering them to tailor the connections to their specific requirements, particularly in Python tools.
+Prompt flow provides a variety of pre-built connections, including Azure OpenAI, OpenAI, etc. These pre-built connections enable seamless integration with these resources within the built-in tools. Additionally, you have the flexibility to create custom connection types using key-value pairs, empowering them to tailor the connections to their specific requirements, particularly in Python tools.
 
 | Connection type                                              | Built-in tools                  |
 | ------------------------------------------------------------ | ------------------------------- |
-| [Azure Open AI](https://azure.microsoft.com/en-us/products/cognitive-services/openai-service) | LLM or Python                   |
-| [Open AI](https://openai.com/)                               | LLM or Python                   |
+| [Azure OpenAI](https://azure.microsoft.com/en-us/products/cognitive-services/openai-service) | LLM or Python                   |
+| [OpenAI](https://openai.com/)                               | LLM or Python                   |
 | [Cognitive Search](https://azure.microsoft.com/en-us/products/search) | Vector DB Lookup or Python      |
 | [Serp](https://serpapi.com/)                                 | Serp API or Python              |
 | [Serverless](https://learn.microsoft.com/en-us/azure/ai-studio/concepts/deployments-overview#deploy-models-with-model-as-a-service-maas)                                               | LLM or Python                   |

docs/how-to-guides/develop-a-dag-flow/quick-start.md

@@ -130,7 +130,7 @@ We need to set up the connection if we haven't added it before. Once created, th
 
 Firstly we need a connection yaml file `connection.yaml`:
 
-If you are using Azure Open AI, prepare your resource follow with this [instruction](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal) and get your `api_key` if you don't have one.
+If you are using Azure OpenAI, prepare your resource follow with this [instruction](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal) and get your `api_key` if you don't have one.
 ```yaml
 $schema: https://azuremlschemas.azureedge.net/promptflow/latest/AzureOpenAIConnection.schema.json
 name: open_ai_connection

docs/how-to-guides/manage-connections.md

@@ -108,7 +108,7 @@ On the VS Code primary sidebar > prompt flow pane. You can find the connections
 :sync: CLI
 The commands below show how to update existing connections with new values:
 ```bash
-# Update an azure open ai connection with a new api base
+# Update an azure OpenAI connection with a new api base
 pf connection update -n my_azure_open_ai_connection --set api_base='new_value'
 # Update a custom connection
 pf connection update -n my_custom_connection --set configs.other_config='new_value'
@@ -120,7 +120,7 @@ pf connection update -n my_custom_connection --set configs.other_config='new_val
 :sync: SDK
 The code snippet below shows how to update existing connections with new values:
 ```python
-# Update an azure open ai connection with a new api base
+# Update an azure OpenAI connection with a new api base
 connection = pf.connections.get(name="my_azure_open_ai_connection")
 connection.api_base = "new_value"
 connection.api_key = "<original-key>"  # secrets are required when updating connection using sdk

docs/index.md

@@ -13,7 +13,7 @@ html_theme.sidebar_secondary.remove: true
 With prompt flow, you will be able to:
 
 - **Create [flows](./concepts/concept-flows.md)** that link [LLMs](./reference/tools-reference/llm-tool.md), [prompts](./reference/tools-reference/prompt-tool.md), [Python](./reference/tools-reference/python-tool.md) code and other [tools](./concepts/concept-tools.md) together in a executable workflow.
-- **Debug and iterate your flows**, especially the interaction with LLMs with ease.
+- **Debug and iterate your flows**, especially [tracing interaction with LLMs](how-to-guides/tracing/index.md) with ease.
 - **Evaluate your flows**, calculate quality and performance metrics with larger datasets.
 - **Integrate the testing and evaluation into your CI/CD system** to ensure quality of your flow.
 - **Deploy your flows** to the serving platform you choose or integrate into your app's code base easily.
@@ -39,6 +39,7 @@ This documentation site contains guides for prompt flow [sdk, cli](https://pypi.
 - header: "📒 How-to Guides"
   content: "
     Articles guide user to complete a specific task in prompt flow.<br/><br/>
+    - [Tracing](how-to-guides/tracing/index.md)<br/>
     - [Develop a flow](how-to-guides/develop-a-flex-flow/index.md)<br/>
     - [Run and evaluate a flow](how-to-guides/run-and-evaluate-a-flow/index.md)<br/>
     - [Deploy a flow](how-to-guides/deploy-a-flow/index.md)<br/>

examples/connections/README.md

@@ -34,7 +34,7 @@ pf connection show --name custom_connection
 - To update a connection that in workspace, execute following command. Currently only a few fields(description, display_name) support update:
 ```bash
 # Update an existing connection with --set to override values
-# Update an azure open ai connection with a new api base
+# Update an azure OpenAI connection with a new api base
 pf connection update -n open_ai_connection --set api_base='<your_api_base>'
 # Update a custom connection
 pf connection update -n custom_connection --set configs.key1='<your_new_key>' secrets.key2='<your_another_key>'

examples/connections/connection.ipynb

@@ -7,13 +7,13 @@
    "source": [
     "# Connection Management\n",
     "\n",
-    "Prompt flow provides various prebuilt connections, including Azure Open AI, Open AI, Azure Content Safety, etc. Prebuilt connections enable seamless integration with these resources within the built-in tools. \n",
+    "Prompt flow provides various prebuilt connections, including Azure OpenAI, OpenAI, Azure Content Safety, etc. Prebuilt connections enable seamless integration with these resources within the built-in tools. \n",
     "\n",
     "Additionally, users have the flexibility to create custom connection types using key-value pairs, empowering them to tailor the connections to their specific requirements, particularly in Python tools.\n",
     "\n",
     "Reach more details about connection types [here](https://learn.microsoft.com/en-us/azure/machine-learning/prompt-flow/concept-connections?view=azureml-api-2).\n",
     "## Create different type of connections\n",
-    "We will use Azure Open AI connection and custom connection as example to show how to create connection with promptflow sdk."
+    "We will use Azure OpenAI connection and custom connection as example to show how to create connection with promptflow sdk."
    ]
   },
   {
@@ -57,9 +57,9 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### Create an Azure Open AI connection\n",
+    "### Create an Azure OpenAI connection\n",
     "\n",
-    "Prepare your Azure Open AI resource follow this [instruction](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal) and get your `api_key` if you don't have one."
+    "Prepare your Azure OpenAI resource follow this [instruction](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal) and get your `api_key` if you don't have one."
    ]
   },
   {
@@ -160,7 +160,7 @@
    "metadata": {},
    "source": [
     "## Update a connection\n",
-    "### Update an Azure Open AI connection"
+    "### Update an Azure OpenAI connection"
    ]
   },
   {

examples/flex-flows/basic/README.md

@@ -10,11 +10,11 @@ pip install -r requirements.txt
 
 ## Run flow
 
-- Prepare your Azure Open AI resource follow this [instruction](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal) and get your `api_key` if you don't have one.
+- Prepare your Azure OpenAI resource follow this [instruction](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal) and get your `api_key` if you don't have one.
 
 - Setup environment variables
 
-Ensure you have put your azure open ai endpoint key in [.env](../.env) file. You can create one refer to this [example file](../.env.example).
+Ensure you have put your azure OpenAI endpoint key in [.env](../.env) file. You can create one refer to this [example file](../.env.example).
 
 ```bash
 cat ../.env

examples/flex-flows/chat-async-stream/README.md

@@ -31,7 +31,7 @@ See <a href="https://platform.openai.com/docs/api-reference/chat/create#chat/cre
 
 ## Run flow
 
-- Prepare your Azure Open AI resource follow this [instruction](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal) and get your `api_key` if you don't have one.
+- Prepare your Azure OpenAI resource follow this [instruction](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal) and get your `api_key` if you don't have one.
 
 - Setup connection
 

examples/flex-flows/chat-async-stream/chat-stream-with-async-flex-flow.ipynb

@@ -80,7 +80,7 @@
     "\n",
     "Above prompty uses connection `open_ai_connection` inside, we need to set up the connection if we haven't added it before. After created, it's stored in local db and can be used in any flow.\n",
     "\n",
-    "Prepare your Azure Open AI resource follow this [instruction](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal) and get your `api_key` if you don't have one."
+    "Prepare your Azure OpenAI resource follow this [instruction](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal) and get your `api_key` if you don't have one."
    ]
   },
   {
@@ -99,7 +99,7 @@
     "    conn = pf.connections.get(name=conn_name)\n",
     "    print(\"using existing connection\")\n",
     "except:\n",
-    "    # Follow https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/create-resource?pivots=web-portal to create an Azure Open AI resource.\n",
+    "    # Follow https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/create-resource?pivots=web-portal to create an Azure OpenAI resource.\n",
     "    connection = AzureOpenAIConnection(\n",
     "        name=conn_name,\n",
     "        api_key=\"<your_AOAI_key>\",\n",

examples/flex-flows/chat-basic/README.md

@@ -31,7 +31,7 @@ See <a href="https://platform.openai.com/docs/api-reference/chat/create#chat/cre
 
 ## Run flow
 
-- Prepare your Azure Open AI resource follow this [instruction](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal) and get your `api_key` if you don't have one.
+- Prepare your Azure OpenAI resource follow this [instruction](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal) and get your `api_key` if you don't have one.
 
 - Setup connection