promptflow-tracing-e2e-test.yml

name: promptflow-tracing-e2e-test

on:
  schedule:
    - cron: "40 17 * * *" # Every day starting at 1:40 BJT

  pull_request:
    paths:
      - src/promptflow/**
      - scripts/building/**
      - .github/workflows/promptflow-tracing-e2e-test.yml

  workflow_dispatch:


env:
  packageSetupType: promptflow_with_extra
  testWorkingDirectory: ${{ github.workspace }}/src/promptflow
  PYTHONPATH: ${{ github.workspace }}/src/promptflow
  IS_IN_CI_PIPELINE: "true"


jobs:
  build:
    strategy:
      fail-fast: false
    runs-on: ubuntu-latest
    steps:
    - name: checkout
      uses: actions/checkout@v4
    - name: Display and Set Environment Variables
      run: |
        env | sort >> $GITHUB_OUTPUT
      id: display_env
      shell: bash -el {0}
    - name: Python Setup - ubuntu-latest - Python Version 3.9
      uses: "./.github/actions/step_create_python_environment"
      with:
        pythonVersion: 3.9
    - name: Build wheel
      uses: "./.github/actions/step_sdk_setup"
      with:
        setupType: promptflow_with_extra
        scriptPath: ${{ env.testWorkingDirectory }}
    - name: Upload Wheel
      if: always()
      uses: actions/upload-artifact@v3
      with:
        name: wheel
        path: |
          ${{ github.workspace }}/src/promptflow/dist/*.whl
          ${{ github.workspace }}/src/promptflow-tools/dist/*.whl

  tracing_tests:
    needs: build
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest]
        pythonVersion: ['3.8', '3.9', '3.10', '3.11']
    runs-on: ${{ matrix.os }}
    steps:
    - name: checkout
      uses: actions/checkout@v4

    - name: Display and Set Environment Variables
      run: |
        env | sort >> $GITHUB_OUTPUT
      id: display_env
      shell: bash -el {0}

    - name: Python Setup - ${{ matrix.os }} - Python Version ${{ matrix.pythonVersion }}
      uses: "./.github/actions/step_create_python_environment"
      with:
        pythonVersion: ${{ matrix.pythonVersion }}

    - name: Download Artifacts
      uses: actions/download-artifact@v3
      with:
        name: wheel
        path: artifacts

    - name: Install wheel
      shell: pwsh
      working-directory: artifacts
      run: |
        Set-PSDebug -Trace 1
        pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
        gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)"}}
        gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
        pip freeze

    - name: Azure Login
      uses: azure/login@v1
      with:
        creds: ${{ secrets.AZURE_CREDENTIALS }}

    - name: Generate Configs
      uses: "./.github/actions/step_generate_configs"
      with:
        targetFolder: ${{ env.testWorkingDirectory }}

    - name: Get number of CPU cores
      uses: SimenB/github-actions-cpu-cores@v1
      id: cpu-cores

    - name: run promptflow-tracing test
      shell: pwsh
      working-directory: ${{ env.testWorkingDirectory }}
      run: |
        python "../../scripts/building/run_coverage_tests.py" `
          -p promptflow `
          -t ${{ github.workspace }}/src/promptflow/tests/tracing_test/e2etests `
          -l eastus `
          -m "e2etest" `
          -n ${{ steps.cpu-cores.outputs.count }} `
          --coverage-config ${{ github.workspace }}/src/promptflow/tests/tracing_test/.coveragerc `
          -o "${{ env.testWorkingDirectory }}/test-results-tracing.xml"

    - name: Upload Test Results
      if: always()
      uses: actions/upload-artifact@v3
      with:
        name: Test Results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }})
        path: |
          ${{ env.testWorkingDirectory }}/*.xml
          ${{ env.testWorkingDirectory }}/htmlcov/


  publish-test-results-tracing-test:
    needs: tracing_tests
    if: always()

    runs-on: ubuntu-latest
    permissions:
      checks: write
      pull-requests: write
      contents: read
      issues: read

    steps:
    - name: checkout
      uses: actions/checkout@v4
    - name: Publish Test Results
      uses: "./.github/actions/step_publish_test_results"
      with:
        testActionFileName: promptflow-tracing-e2e-test.yml
        testResultTitle: promptflow-tracing e2e test result
        osVersion: ubuntu-latest
        pythonVersion: 3.9
        coverageThreshold: 40
        context: test/tracing