Merge branch 'main' into dev/chenyin/experiment_test

microsoft · Mar 27, 2024 · 905af3d · 905af3d
2 parents dd276c4 + 36c5c91
commit 905af3d
Show file tree

Hide file tree

Showing 419 changed files with 14,357 additions and 12,648 deletions.
diff --git a/.cspell.json b/.cspell.json
@@ -28,13 +28,18 @@
     "src/promptflow/promptflow/azure/_restclient/flow/**",
     "src/promptflow/promptflow/azure/_restclient/swagger.json",
     "src/promptflow/promptflow/azure/_models/**",
+    "src/promptflow-core/promptflow/core/_connection_provider/_models/**",
     "src/promptflow/tests/**",
+    "src/promptflow-recording/**",
     "src/promptflow-tools/tests/**",
     "**/flow.dag.yaml",
     "**/setup.py",
     "scripts/installer/curl_install_pypi/**",
     "scripts/installer/windows/**",
-    "src/promptflow/promptflow/_sdk/_service/pfsvc.py"
+    "src/promptflow/promptflow/_sdk/_service/pfsvc.py",
+    ".github/workflows/**",
+    ".github/actions/**",
+    ".github/pipelines/**"
   ],
   "words": [
     "aoai",

diff --git a/.github/actions/step_create_python_environment/action.yml b/.github/actions/step_create_python_environment/action.yml
@@ -12,9 +12,10 @@ runs:
   using: composite
   steps:
   - name: Set up Python
-    uses: actions/setup-python@v4
+    uses: actions/setup-python@v5
     with:
       python-version: ${{ inputs.pythonVersion }}
+  - uses: snok/install-poetry@v1
   - run: |
       python -m pip install --upgrade pip
       pip install -r ${{ inputs.pipFilePath }}

diff --git a/.github/workflows/build_doc_ci.yml b/.github/workflows/build_doc_ci.yml
@@ -2,16 +2,16 @@ name: Build Doc CI
 
 on:
   workflow_dispatch:
-  pull_request:
-    branches:
-      - main
-      - preview/docs
-    paths:
-      - 'README.md'
-      - 'docs/**'
-      - 'scripts/docs/**'
-      - '.github/workflows/build_doc_ci.yml'
-      - 'src/promptflow/promptflow/**'
+# TODO: Enable this after package separation
+#  pull_request:
+#    branches:
+#      - main
+#    paths:
+#      - 'README.md'
+#      - 'docs/**'
+#      - 'scripts/docs/**'
+#      - '.github/workflows/build_doc_ci.yml'
+#      - 'src/promptflow/promptflow/**'
 
 env:
   packageSetupType: promptflow_with_extra
@@ -36,6 +36,13 @@ jobs:
           setupType: ${{ env.packageSetupType }}
           scriptPath: ${{ env.testWorkingDirectory }}
 
+      - name: Install tracing
+        shell: pwsh
+        run: |
+          pip uninstall -y promptflow-tracing
+          pip install -e ${{ github.workspace }}/src/promptflow-tracing
+          pip freeze
+
       - name: Build doc with reference doc
         shell: powershell
         working-directory: scripts/docs/

diff --git a/.github/workflows/promptflow-executor-e2e-test.yml b/.github/workflows/promptflow-executor-e2e-test.yml
@@ -4,6 +4,7 @@ on:
     - cron: "40 20 * * *" # Every day starting at 4:40 BJT
   pull_request_target:
     paths:
+      - src/promptflow-core/*
       - src/promptflow/*
       - src/promptflow/promptflow/*
       - src/promptflow/promptflow/_core/**
@@ -17,6 +18,7 @@ on:
       - src/promptflow/promptflow/storage/**
       - src/promptflow/tests/*
       - src/promptflow/tests/executor/**
+      - src/promptflow-tracing/promptflow/**
       - scripts/building/**
       - .github/workflows/promptflow-executor-e2e-test.yml
   workflow_dispatch:
@@ -25,6 +27,7 @@ env:
   testWorkingDirectory: ${{ github.workspace }}/src/promptflow
   PYTHONPATH: ${{ github.workspace }}/src/promptflow
   IS_IN_CI_PIPELINE: "true"
+  RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
 jobs:
   authorize:
     environment:
@@ -102,9 +105,16 @@ jobs:
       run: |
         Set-PSDebug -Trace 1
         pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
+        pip install ${{ github.workspace }}/src/promptflow-tracing
+        pip install ${{ github.workspace }}/src/promptflow-core
         gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[executor-service]"}}
         gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
         pip freeze
+    - name: install recording
+      run: |
+        pip install vcrpy
+        pip install -e .
+      working-directory: ${{ env.RECORD_DIRECTORY }}
     - name: Azure Login
       uses: azure/login@v1
       with:

diff --git a/.github/workflows/promptflow-executor-unit-test.yml b/.github/workflows/promptflow-executor-unit-test.yml
@@ -4,6 +4,7 @@ on:
     - cron: "40 19 * * *" # Every day starting at 3:40 BJT
   pull_request_target:
     paths:
+      - src/promptflow-core/*
       - src/promptflow/*
       - src/promptflow/promptflow/*
       - src/promptflow/promptflow/_core/**
@@ -25,6 +26,7 @@ env:
   testWorkingDirectory: ${{ github.workspace }}/src/promptflow
   PYTHONPATH: ${{ github.workspace }}/src/promptflow
   IS_IN_CI_PIPELINE: "true"
+  RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
 jobs:
   authorize:
     environment:
@@ -107,9 +109,16 @@ jobs:
       run: |
         Set-PSDebug -Trace 1
         pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
+        pip install ${{ github.workspace }}/src/promptflow-tracing
+        pip install ${{ github.workspace }}/src/promptflow-core
         gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[executor-service]"}}
         gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
         pip freeze
+    - name: install recording
+      run: |
+        pip install vcrpy
+        pip install -e .
+      working-directory: ${{ env.RECORD_DIRECTORY }}
     - name: Azure Login
       uses: azure/login@v1
       with:

diff --git a/.github/workflows/promptflow-global-config-test.yml b/.github/workflows/promptflow-global-config-test.yml
@@ -4,6 +4,7 @@ on:
     - cron: "40 18 * * *" # Every day starting at 2:40 BJT
   pull_request_target:
     paths:
+      - src/promptflow-core/*
       - src/promptflow/**
       - scripts/building/**
       - .github/workflows/promptflow-global-config-test.yml
@@ -60,6 +61,15 @@ jobs:
       with:
         setupType: ${{ env.packageSetupType }}
         scriptPath: ${{ env.testWorkingDirectory }}
+    - name: Install dependency
+      shell: pwsh
+      run: |
+        pip uninstall -y promptflow-tracing
+        pip install ${{ github.workspace }}/src/promptflow-tracing
+        echo "Installed promptflow-tracing"
+        pip uninstall -y promptflow-core
+        pip install ${{ github.workspace }}/src/promptflow-core
+        pip freeze
     - name: Azure Login
       uses: azure/login@v1
       with:

diff --git a/.github/workflows/promptflow-release-testing-matrix.yml b/.github/workflows/promptflow-release-testing-matrix.yml
@@ -16,6 +16,8 @@ env:
   testWorkingDirectory: src/promptflow
   PYTHONPATH: ${{ github.workspace }}/src/promptflow
   IS_IN_CI_PIPELINE: "true"
+  RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
+  TRACING_PATH: ${{ github.workspace }}/src/promptflow-tracing
 jobs:
   id:
     runs-on: ubuntu-latest
@@ -44,6 +46,7 @@ jobs:
         path: |
           ${{ github.workspace }}/src/promptflow/dist/*.whl
           ${{ github.workspace }}/src/promptflow-tools/dist/*.whl
+
   promptflow_sdk_cli_tests:
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' || github.event_name == 'pull_request' }}
     needs: build
@@ -71,6 +74,11 @@ jobs:
       with:
         name: wheel
         path: artifacts
+    - name: install recording
+      run:
+        pip install vcrpy
+        pip install -e .
+      working-directory: ${{ env.RECORD_DIRECTORY }}
     - name: Azure Login
       uses: azure/login@v1
       if: env.PROMPT_FLOW_TEST_MODE == 'live'
@@ -155,6 +163,11 @@ jobs:
       with:
         name: wheel
         path: artifacts
+    - name: install recording
+      run: |
+        pip install vcrpy
+        pip install -e .
+      working-directory: ${{ env.RECORD_DIRECTORY }}
     - name: Azure Login
       uses: azure/login@v1
       if: env.PROMPT_FLOW_TEST_MODE == 'live'
@@ -214,6 +227,11 @@ jobs:
       with:
         name: wheel
         path: artifacts
+    - name: install recording
+      run: |
+        pip install vcrpy
+        pip install -e .
+      working-directory: ${{ env.RECORD_DIRECTORY }}
     - name: Azure Login
       uses: azure/login@v1
       with:

diff --git a/.github/workflows/promptflow-sdk-cli-test.yml b/.github/workflows/promptflow-sdk-cli-test.yml
@@ -4,7 +4,9 @@ on:
     - cron: "40 18 * * *" # Every day starting at 2:40 BJT
   pull_request:
     paths:
+      - src/promptflow-core/**
       - src/promptflow/**
+      - src/promptflow-tracing/**
       - scripts/building/**
       - .github/workflows/promptflow-sdk-cli-test.yml
   workflow_dispatch:
@@ -13,6 +15,7 @@ env:
   testWorkingDirectory: ${{ github.workspace }}/src/promptflow
   PYTHONPATH: ${{ github.workspace }}/src/promptflow
   IS_IN_CI_PIPELINE: "true"
+  RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
 jobs:
   build:
     strategy:
@@ -76,9 +79,16 @@ jobs:
       run: |
         Set-PSDebug -Trace 1
         pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
+        pip install ${{ github.workspace }}/src/promptflow-tracing
+        pip install ${{ github.workspace }}/src/promptflow-core
         gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)"}}
         gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
         pip freeze
+    - name: install recording
+      run: |
+        pip install vcrpy
+        pip install -e .
+      working-directory: ${{ env.RECORD_DIRECTORY }}
     - name: Azure login (non pull_request workflow)
       if: github.event_name != 'pull_request'
       uses: azure/login@v1

diff --git a/.github/workflows/promptflow-tracing-e2e-test.yml b/.github/workflows/promptflow-tracing-e2e-test.yml
@@ -0,0 +1,104 @@
+name: promptflow-tracing-e2e-test
+
+on:
+  schedule:
+    - cron: "40 18 * * *" # 2:40 Beijing Time (GMT+8) every day
+  pull_request:
+    paths:
+      - src/promptflow-tracing/**
+      - .github/workflows/promptflow-tracing-e2e-test.yml
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+  WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing
+  RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: snok/install-poetry@v1
+    - name: build
+      run: poetry build
+      working-directory: ${{ env.WORKING_DIRECTORY }}
+    - uses: actions/upload-artifact@v4
+      with:
+        name: promptflow-tracing
+        path: ${{ env.WORKING_DIRECTORY }}/dist/promptflow_tracing-*.whl
+
+  test:
+    needs: build
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: ['3.8', '3.9', '3.10', '3.11']
+      fail-fast: false
+    # snok/install-poetry need this to support Windows
+    defaults:
+      run:
+        shell: bash
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: set test mode
+        run: echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: snok/install-poetry@v1
+      - uses: actions/download-artifact@v4
+        with:
+          name: promptflow-tracing
+          path: ${{ env.WORKING_DIRECTORY }}
+      - name: install promptflow-tracing from wheel
+        # wildcard expansion (*) does not work in Windows, so leverage python to find and install
+        run: poetry run pip install $(python -c "import glob; print(glob.glob('promptflow_tracing-*.whl')[0])")
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: install test dependency group
+        run: poetry install --only test
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: install recording
+        run: poetry install
+        working-directory: ${{ env.RECORD_DIRECTORY }}
+      - name: generate end-to-end test config from secret
+        run: echo '${{ secrets.PF_TRACING_E2E_TEST_CONFIG }}' >> connections.json
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: run e2e tests
+        run: poetry run pytest -m e2etest --cov=promptflow --cov-config=pyproject.toml --cov-report=term --cov-report=html --cov-report=xml
+        working-directory: ${{ env.WORKING_DIRECTORY }}
+      - name: upload coverage report
+        uses: actions/upload-artifact@v4
+        with:
+          name: report-${{ matrix.os }}-py${{ matrix.python-version }}
+          path: |
+            ${{ env.WORKING_DIRECTORY }}/*.xml
+            ${{ env.WORKING_DIRECTORY }}/htmlcov/
+
+  report:
+    needs: test
+    runs-on: ubuntu-latest
+    permissions:
+      checks: write
+      pull-requests: write
+      contents: read
+      issues: read
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          path: artifacts
+      - uses: EnricoMi/publish-unit-test-result-action@v2
+        with:
+          check_name: promptflow-tracing test result
+          comment_title: promptflow-tracing test result
+          files: "artifacts/**/test-results.xml"  # align with `--junit-xml` in pyproject.toml
+      - uses: irongut/[email protected]
+        with:
+          filename: "artifacts/report-ubuntu-latest-py3.9/coverage.xml"
+          badge: true
+          fail_below_min: true
+          format: markdown
+          hide_complexity: true
+          output: both
+          thresholds: 40 80