Merge branch 'main' into chjinche/llm_use_custom

.cspell.json

@@ -41,7 +41,8 @@
     "scripts/installer/windows/**",
     ".github/workflows/**",
     ".github/actions/**",
-    ".github/pipelines/**"
+    ".github/pipelines/**",
+    ".github/CODEOWNERS"
   ],
   "words": [
     "aoai",
@@ -196,7 +197,9 @@
     "nologo",
     "wscript",
     "raisvc",
-    "evals"
+    "evals",
+    "wargs",
+    "dcid"
   ],
   "flagWords": [
     "Prompt Flow"

.github/CODEOWNERS

@@ -29,6 +29,7 @@
 /src/promptflow-core/promptflow/executor/ @microsoft/promptflow-execution
 /src/promptflow-core/promptflow/integrations/ @microsoft/promptflow-execution
 /src/promptflow-core/promptflow/storage/ @microsoft/promptflow-execution
+/src/promptflow-devkit/promptflow/_sdk/_mlflow.py @brynn-code
 /src/promptflow-devkit/promptflow/_internal/ @microsoft/promptflow-execution
 /src/promptflow-devkit/promptflow/batch/ @microsoft/promptflow-execution
 /src/promptflow/tests/executor/ @microsoft/promptflow-execution

.github/workflows/build_msi_installer.yml

@@ -108,7 +108,8 @@ jobs:
           setupType: promptflow_with_extra
           scriptPath: ${{ env.testWorkingDirectory }}
 
-      - name: Setup and Install dev promptflow
+      - name: Setup and Install separate dev promptflow
+        if: ${{ github.event.inputs.version == null || github.event.inputs.version == '' }}
         shell: pwsh
         run: |
           Set-PSDebug -Trace 1

.github/workflows/promptflow-core-test.yml

@@ -16,10 +16,10 @@ env:
   RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
 
 jobs:
-  test:
+  core_test:
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: [ubuntu-latest]
         python-version: ['3.8', '3.9', '3.10', '3.11']
       fail-fast: false
     # snok/install-poetry need this to support Windows
@@ -46,18 +46,58 @@ jobs:
         run: poetry install
         working-directory: ${{ env.RECORD_DIRECTORY }}
       - name: run core tests
-        run: poetry run pytest --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml
+        run: poetry run pytest ./tests/core --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml
         working-directory: ${{ env.WORKING_DIRECTORY }}
       - name: upload coverage report
         uses: actions/upload-artifact@v4
         with:
           name: report-${{ matrix.os }}-py${{ matrix.python-version }}
           path: |
-            ${{ env.WORKING_DIRECTORY }}/*.xml
-            ${{ env.WORKING_DIRECTORY }}/htmlcov/
+            ${{ env.WORKING_DIRECTORY }}/core/*.xml
+            ${{ env.WORKING_DIRECTORY }}/core/htmlcov/
+
+  azureml_serving_test:
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ['3.8', '3.9', '3.10', '3.11']
+      fail-fast: false
+    # snok/install-poetry need this to support Windows
+    defaults:
+      run:
+        shell: bash
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: snok/install-poetry@v1
+      - name: install promptflow-tracing
+        run: poetry install
+        working-directory: ${{ env.TRACING_DIRECTORY }}
+      - name: install promptflow-core[azureml-serving]
+        run: poetry install -E azureml-serving
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: install test dependency group
+        run: poetry install --only test
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: install recording
+        run: poetry install
+        working-directory: ${{ env.RECORD_DIRECTORY }}
+      - name: run azureml-serving tests
+        run: poetry run pytest ./tests/azureml-serving --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: upload coverage report
+        uses: actions/upload-artifact@v4
+        with:
+          name: report-${{ matrix.os }}-py${{ matrix.python-version }}
+          path: |
+            ${{ env.WORKING_DIRECTORY }}/azureml-serving/*.xml
+            ${{ env.WORKING_DIRECTORY }}/azureml-serving/htmlcov/
 
   report:
-    needs: test
+    needs: [core_test, azureml_serving_test]
     runs-on: ubuntu-latest
     permissions:
       checks: write

.github/workflows/promptflow-evals-e2e-test.yml

@@ -0,0 +1,115 @@
+name: promptflow-evals-e2e-test
+
+on:
+  schedule:
+    - cron: "40 10 * * *" # 2:40 PST every day
+  pull_request:
+    paths:
+      - src/promptflow-evals/**
+      - .github/workflows/promptflow-evals-e2e-test.yml
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+  WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-evals
+  RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: snok/install-poetry@v1
+    - name: build
+      run: poetry build
+      working-directory: ${{ env.WORKING_DIRECTORY }}
+    - uses: actions/upload-artifact@v4
+      with:
+        name: promptflow-evals
+        path: ${{ env.WORKING_DIRECTORY }}/dist/promptflow_evals-*.whl
+
+  test:
+    needs: build
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        # TODO: Following up with PF team for the attribute error from 3.8 and 3.9.
+        python-version: ['3.10', '3.11']
+        #python-version: ['3.8', '3.9', '3.10', '3.11']
+      fail-fast: false
+    # snok/install-poetry need this to support Windows
+    defaults:
+      run:
+        shell: bash
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: set test mode
+        # Always run in replay mode for now until we figure out the test resource to run live mode
+        run: echo "PROMPT_FLOW_TEST_MODE=replay" >> $GITHUB_ENV
+        #run: echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: snok/install-poetry@v1
+      - uses: actions/download-artifact@v4
+        with:
+          name: promptflow-evals
+          path: ${{ env.WORKING_DIRECTORY }}
+      - name: install promptflow packages in editable mode
+        run: |
+          poetry run pip install -e ../promptflow-core
+          poetry run pip install -e ../promptflow-devkit
+          poetry run pip install -e ../promptflow-tools
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: install promptflow-evals from wheel
+        # wildcard expansion (*) does not work in Windows, so leverage python to find and install
+        run: poetry run pip install --pre $(python -c "import glob; print(glob.glob('promptflow_evals-*.whl')[0])")
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: install test dependency group
+        run: poetry install --only test
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: install recording
+        run: poetry install
+        working-directory: ${{ env.RECORD_DIRECTORY }}
+      - name: generate end-to-end test config from secret
+        # TODO: replace with evals secret
+        run: echo '${{ secrets.PF_TRACING_E2E_TEST_CONFIG }}' >> connections.json
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: run e2e tests
+        run: poetry run pytest -m e2etest --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: upload coverage report
+        uses: actions/upload-artifact@v4
+        with:
+          name: report-${{ matrix.os }}-py${{ matrix.python-version }}
+          path: |
+            ${{ env.WORKING_DIRECTORY }}/*.xml
+            ${{ env.WORKING_DIRECTORY }}/htmlcov/
+
+  report:
+    needs: test
+    runs-on: ubuntu-latest
+    permissions:
+      checks: write
+      pull-requests: write
+      contents: read
+      issues: read
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          path: artifacts
+      - uses: EnricoMi/publish-unit-test-result-action@v2
+        with:
+          check_name: promptflow-evals test result
+          comment_title: promptflow-evals test result
+          files: "artifacts/**/test-results.xml"  # align with `--junit-xml` in pyproject.toml
+      - uses: irongut/[email protected]
+        with:
+          filename: "artifacts/report-ubuntu-latest-py3.11/coverage.xml"
+          badge: true
+          fail_below_min: false
+          format: markdown
+          hide_complexity: true
+          output: both
+          thresholds: 40 80

.github/workflows/promptflow-executor-e2e-test.yml

@@ -19,6 +19,8 @@ on:
       - src/promptflow/tests/*
       - src/promptflow/tests/executor/**
       - src/promptflow-tracing/promptflow/**
+      - src/promptflow-core/promptflow/**
+      - src/promptflow-devkit/promptflow/**
       - scripts/building/**
       - .github/workflows/promptflow-executor-e2e-test.yml
   workflow_dispatch:

.github/workflows/promptflow-executor-unit-test.yml

@@ -18,6 +18,9 @@ on:
       - src/promptflow/promptflow/storage/**
       - src/promptflow/tests/*
       - src/promptflow/tests/executor/**
+      - src/promptflow-tracing/promptflow/**
+      - src/promptflow-core/promptflow/**
+      - src/promptflow-devkit/promptflow/**
       - scripts/building/**
       - .github/workflows/promptflow-executor-unit-test.yml
   workflow_dispatch:

.github/workflows/promptflow-release-testing-matrix.yml

@@ -313,7 +313,7 @@ jobs:
         run: poetry install
         working-directory: ${{ github.workspace }}/src/promptflow-recording
       - name: run core tests
-        run: poetry run pytest --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml
+        run: poetry run pytest ./tests/core --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml
         working-directory: ${{ github.workspace }}/src/promptflow-core
       - name: upload coverage report
         uses: actions/upload-artifact@v4
@@ -328,9 +328,58 @@ jobs:
         with:
           name: promptflow_core_tests Test Results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
           path: ${{ github.workspace }}/*.xml
+
+  promptflow_core_azureml_serving_tests:
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' || github.event_name == 'pull_request' }}
+    needs: build
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        pythonVersion: ['3.8', '3.9', '3.10', '3.11']
+    # snok/install-poetry need this to support Windows
+    defaults:
+      run:
+        shell: bash
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: snok/install-poetry@v1
+      - name: install promptflow-tracing
+        run: poetry install
+        working-directory: ${{ github.workspace }}/src/promptflow-tracing
+      - name: install promptflow-core[azureml-serving]
+        run: poetry install -E azureml-serving
+        working-directory: ${{ github.workspace }}/src/promptflow-core
+      - name: install test dependency group
+        run: poetry install --only test
+        working-directory: ${{ github.workspace }}/src/promptflow-core
+      - name: install recording
+        run: poetry install
+        working-directory: ${{ github.workspace }}/src/promptflow-recording
+      - name: run azureml-serving tests
+        run: poetry run pytest ./tests/azureml-serving --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml
+        working-directory: ${{ github.workspace }}/src/promptflow-core/
+      - name: upload coverage report
+        uses: actions/upload-artifact@v4
+        with:
+          name: report-${{ matrix.os }}-py${{ matrix.python-version }}
+          path: |
+            ${{ env.WORKING_DIRECTORY }}/*.xml
+            ${{ env.WORKING_DIRECTORY }}/htmlcov/
+      - name: Upload pytest test results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: promptflow_core_tests Test Results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
+          path: ${{ github.workspace }}/*.xml
+
   publish-test-results:
     name: "Publish Tests Results"
-    needs: [ promptflow_sdk_cli_tests, promptflow_sdk_cli_azure_tests, promptflow_executor_tests ]
+    needs: [ promptflow_sdk_cli_tests, promptflow_sdk_cli_azure_tests, promptflow_executor_tests, promptflow_core_tests, promptflow_core_azureml_serving_tests ]
     runs-on: ubuntu-latest
     permissions:
       checks: write

.github/workflows/promptflow-sdk-cli-test.yml

@@ -10,6 +10,7 @@ on:
       - src/promptflow-tracing/**
       - scripts/building/**
       - .github/workflows/promptflow-sdk-cli-test.yml
+      - src/promptflow-recording/**
   workflow_dispatch:
 env:
   packageSetupType: promptflow_with_extra

.github/workflows/samples_flows_chat_chatwithpdf_chatwithpdf.yml

@@ -31,16 +31,18 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: "3.9"
-      - name: Prepare sample requirements
-        working-directory: examples/flows/chat/chat-with-pdf
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
       - name: Prepare requirements
         run: |
           python -m pip install --upgrade pip
           pip install -r ${{ github.workspace }}/examples/requirements.txt
           pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
+          pip freeze
+          pf --version
+        shell: bash -el {0}
+      - name: Prepare sample requirements
+        working-directory: examples/flows/chat/chat-with-pdf
+        run: |
+          pip install -r requirements.txt
       - name: Create Chat With PDF Custom Connection
         working-directory: examples/flows/chat/chat-with-pdf
         run: |

.github/workflows/samples_flows_chat_chatwithpdf_chatwithpdfazure.yml

@@ -31,16 +31,18 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: "3.9"
-      - name: Prepare sample requirements
-        working-directory: examples/flows/chat/chat-with-pdf
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
       - name: Prepare requirements
         run: |
           python -m pip install --upgrade pip
           pip install -r ${{ github.workspace }}/examples/requirements.txt
           pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
+          pip freeze
+          pf --version
+        shell: bash -el {0}
+      - name: Prepare sample requirements
+        working-directory: examples/flows/chat/chat-with-pdf
+        run: |
+          pip install -r requirements.txt
       - name: Create Chat With PDF Custom Connection
         working-directory: examples/flows/chat/chat-with-pdf
         run: |

docs/how-to-guides/manage-connections.md

@@ -227,6 +227,15 @@ Supported types are as follows:
 
 For example, with `OPENAI_API_KEY` set to environment, an `OpenAIConnection` object can be loaded with `OpenAIConnection.from_env()`.
 
+## Authenticate with Microsoft Entra ID
+[Microsoft Entra ID](https://learn.microsoft.com/entra/fundamentals/whatis) is a cloud-based identity and access management service that enables your employees access external resources.
+
+Some promptflow connection types supports connection authentication with Microsoft Entra ID.
+
+| Connection Type       | Yaml Field | Value     | Package Requirements                                          | VS Code Extension |
+| --------------------- |------------|-----------|---------------------------------------------------------------|------------------|
+| AzureOpenAIConnection | auth_mode  | meid_token | `promptflow[azureml-serving]>=1.7.0, promptflow-tools>=1.4.0` | 1.20.0           |
+
 
 ## Next steps
 - Reach more detail about [connection concepts](../../concepts/concept-connections.md).

docs/how-to-guides/quick-start.md

@@ -36,7 +36,7 @@ pf -v
 
 ## Understand what's a flow
 
-A flow, represented as a YAML file, is a DAG of functions, which is connected via input/output dependencies, and executed based on the topology by prompt flow executor. See [Flows](../../concepts/concept-flows.md) for more details.
+A flow, represented as a YAML file, is a DAG of functions, which is connected via input/output dependencies, and executed based on the topology by prompt flow executor. See [Flows](../concepts/concept-flows.md) for more details.
 
 ### Get the flow sample