Merge branch 'main' into yigao/pipeline_az_login

.cspell.json

@@ -106,7 +106,9 @@
     "Bhavik",
     "meid",
     "Entra",
-    "uvicorn"
+    "uvicorn",
+    "attribited",
+    "MistralAI"
   ],
   "ignoreWords": [
     "openmpi",
@@ -208,6 +210,8 @@
     "wscript",
     "raisvc",
     "evals",
+    "setenv",
+    "pypdf",
     "redoc",
     "starlette",
     "mlindex",
@@ -220,7 +224,9 @@
     "piezo",
     "Piezo",
     "cmpop",
-    "omap"
+    "omap",
+    "azureopenaimodelconfiguration",
+    "openaimodelconfiguration"
   ],
   "flagWords": [
     "Prompt Flow"

.github/workflows/build_doc_ci.yml

@@ -37,11 +37,12 @@ jobs:
         shell: pwsh
         # Note: Use -e to avoid duplicate object warning when build apidoc.
         run: |
-          pip uninstall -y promptflow-tracing promptflow-core promptflow-devkit promptflow-azure
+          pip uninstall -y promptflow-tracing promptflow-core promptflow-devkit promptflow-azure promptflow-rag
           pip install -e ${{ github.workspace }}/src/promptflow-tracing
           pip install -e ${{ github.workspace }}/src/promptflow-core
           pip install -e ${{ github.workspace }}/src/promptflow-devkit
           pip install -e ${{ github.workspace }}/src/promptflow-azure
+          pip install -e ${{ github.workspace }}/src/promptflow-rag
           pip freeze
 
       - name: Build doc with reference doc
@@ -68,11 +69,12 @@ jobs:
         shell: pwsh
         # Note: Use -e to avoid duplicate object warning when build apidoc.
         run: |
-          pip uninstall -y promptflow-tracing promptflow-core promptflow-devkit promptflow-azure
+          pip uninstall -y promptflow-tracing promptflow-core promptflow-devkit promptflow-azure promptflow-rag
           pip install -e ${{ github.workspace }}/src/promptflow-tracing
           pip install -e ${{ github.workspace }}/src/promptflow-core
           pip install -e ${{ github.workspace }}/src/promptflow-devkit
           pip install -e ${{ github.workspace }}/src/promptflow-azure
+          pip install -e ${{ github.workspace }}/src/promptflow-rag
           pip freeze
 
       - name: Build LinkCheck

.github/workflows/promptflow-executor-e2e-test.yml

@@ -22,6 +22,7 @@ on:
       - src/promptflow-core/promptflow/**
       - src/promptflow-devkit/promptflow/**
       - scripts/building/**
+      - src/promptflow-recording/recordings/local/executor_node_cache.*
       - .github/workflows/promptflow-executor-e2e-test.yml
   workflow_dispatch:
 env:

.github/workflows/promptflow-executor-unit-test.yml

@@ -22,6 +22,7 @@ on:
       - src/promptflow-core/promptflow/**
       - src/promptflow-devkit/promptflow/**
       - scripts/building/**
+      - src/promptflow-recording/recordings/local/executor_node_cache.*
       - .github/workflows/promptflow-executor-unit-test.yml
   workflow_dispatch:
 permissions:

.github/workflows/publish_doc.yml

@@ -47,11 +47,12 @@ jobs:
         shell: pwsh
         # Note: Use -e to avoid duplicate object warning when build apidoc.
         run: |
-          pip uninstall -y promptflow-tracing promptflow-core promptflow-devkit promptflow-azure
+          pip uninstall -y promptflow-tracing promptflow-core promptflow-devkit promptflow-azure promptflow-rag
           pip install -e ${{ github.workspace }}/src/promptflow-tracing
           pip install -e ${{ github.workspace }}/src/promptflow-core
           pip install -e ${{ github.workspace }}/src/promptflow-devkit
           pip install -e ${{ github.workspace }}/src/promptflow-azure
+          pip install -e ${{ github.workspace }}/src/promptflow-rag
           pip freeze
 
       - name: Build Doc

.github/workflows/samples_flex_flows_eval_criteria_with_langchain.yml

@@ -0,0 +1,110 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flex_flows_eval_criteria_with_langchain
+on:
+  schedule:
+    - cron: "21 20 * * *" # Every day starting at 4:21 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flex-flows/eval-criteria-with-langchain/**, examples/*requirements.txt, .github/workflows/samples_flex_flows_eval_criteria_with_langchain.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+jobs:
+  samples_flex_flows_eval_criteria_with_langchain:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flex-flows/eval-criteria-with-langchain
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flex-flows/eval-criteria-with-langchain
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS }}
+      - name: Extract Steps examples/flex-flows/eval-criteria-with-langchain/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flex-flows/eval-criteria-with-langchain/README.md -o examples/flex-flows/eval-criteria-with-langchain
+      - name: Cat script
+        working-directory: examples/flex-flows/eval-criteria-with-langchain
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flex-flows/eval-criteria-with-langchain
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flex-flows/eval-criteria-with-langchain
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flex-flows/eval-criteria-with-langchain
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: artifact
+          path: examples/flex-flows/eval-criteria-with-langchain/bash_script.sh

.github/workflows/samples_flexflows_evalcriteriawithlangchain_langchaineval.yml

@@ -0,0 +1,64 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flexflows_evalcriteriawithlangchain_langchaineval
+on:
+  schedule:
+    - cron: "42 20 * * *" # Every day starting at 4:42 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flex-flows/eval-criteria-with-langchain/**, examples/*requirements.txt, .github/workflows/samples_flexflows_evalcriteriawithlangchain_langchaineval.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+jobs:
+  samples_flexflows_evalcriteriawithlangchain_langchaineval:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS }}
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r ${{ github.workspace }}/examples/requirements.txt
+          pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
+      - name: setup .env file
+        working-directory: examples/flex-flows/eval-criteria-with-langchain
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create Aoai Connection
+        run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY_TEST }}" api_base="${{ secrets.AOAI_API_ENDPOINT_TEST }}"
+      - name: Test Notebook
+        working-directory: examples/flex-flows/eval-criteria-with-langchain
+        run: |
+          papermill -k python langchain-eval.ipynb langchain-eval.output.ipynb
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: artifact
+          path: examples/flex-flows/eval-criteria-with-langchain

.github/workflows/samples_flows_evaluation_eval_multi_turn_metrics.yml

@@ -0,0 +1,110 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_flows_evaluation_eval_multi_turn_metrics
+on:
+  schedule:
+    - cron: "7 21 * * *" # Every day starting at 5:7 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/flows/evaluation/eval-multi-turn-metrics/**, examples/*requirements.txt, .github/workflows/samples_flows_evaluation_eval_multi_turn_metrics.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+jobs:
+  samples_flows_evaluation_eval_multi_turn_metrics:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        working-directory: examples
+        run: |
+          if [[ -e requirements.txt ]]; then
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+          fi
+      - name: Prepare dev requirements
+        working-directory: examples
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements.txt
+      - name: Refine .env file
+        working-directory: examples/flows/evaluation/eval-multi-turn-metrics
+        run: |
+          AOAI_API_KEY=${{ secrets.AOAI_API_KEY_TEST }}
+          AOAI_API_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          AOAI_API_ENDPOINT=$(echo ${AOAI_API_ENDPOINT//\//\\/})
+          if [[ -e .env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" .env.example
+            mv .env.example .env
+          fi
+          if [[ -e ../.env.example ]]; then
+            echo "env replacement"
+            sed -i -e "s/<your_AOAI_key>/$AOAI_API_KEY/g" -e "s/<your_AOAI_endpoint>/$AOAI_API_ENDPOINT/g" ../.env.example
+            mv ../.env.example ../.env
+          fi
+      - name: Create run.yml
+        working-directory: examples/flows/evaluation/eval-multi-turn-metrics
+        run: |
+          gpt_base=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          gpt_base=$(echo ${gpt_base//\//\\/})
+          if [[ -e run.yml ]]; then
+            sed -i -e "s/\${azure_open_ai_connection.api_key}/${{ secrets.AOAI_API_KEY_TEST }}/g" -e "s/\${azure_open_ai_connection.api_base}/$gpt_base/g" run.yml
+          fi
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS }}
+      - name: Extract Steps examples/flows/evaluation/eval-multi-turn-metrics/README.md
+        working-directory: ${{ github.workspace }}
+        run: |
+          python scripts/readme/extract_steps_from_readme.py -f examples/flows/evaluation/eval-multi-turn-metrics/README.md -o examples/flows/evaluation/eval-multi-turn-metrics
+      - name: Cat script
+        working-directory: examples/flows/evaluation/eval-multi-turn-metrics
+        run: |
+          cat bash_script.sh
+      - name: Run scripts against canary workspace (scheduled runs only)
+        if: github.event_name == 'schedule'
+        working-directory: examples/flows/evaluation/eval-multi-turn-metrics
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_CANARY }}
+          bash bash_script.sh
+      - name: Run scripts against production workspace
+        if: github.event_name != 'schedule'
+        working-directory: examples/flows/evaluation/eval-multi-turn-metrics
+        run: |
+          export aoai_api_key=${{secrets.AOAI_API_KEY_TEST }}
+          export aoai_api_endpoint=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export AZURE_OPENAI_API_KEY=${{secrets.AOAI_API_KEY_TEST }}
+          export AZURE_OPENAI_ENDPOINT=${{ secrets.AOAI_API_ENDPOINT_TEST }}
+          export test_workspace_sub_id=${{ secrets.TEST_WORKSPACE_SUB_ID }}
+          export test_workspace_rg=${{ secrets.TEST_WORKSPACE_RG }}
+          export test_workspace_name=${{ secrets.TEST_WORKSPACE_NAME_PROD }}
+          bash bash_script.sh
+      - name: Pip List for Debug
+        if : ${{ always() }}
+        working-directory: examples/flows/evaluation/eval-multi-turn-metrics
+        run: |
+          pip list
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: artifact
+          path: examples/flows/evaluation/eval-multi-turn-metrics/bash_script.sh