e2e-test.yml

name: E2E test

on:
  workflow_dispatch:
    inputs:
      cases:
        description: 'specific cases to be excuted. Sample: ["./aad/a.tests.ts", "./bot/b.tests.ts"]. Set empty to run all cases'
        required: false

      sample-branch:
        description: "default is 'dev', options: 'main'"
        required: true
        type: string
        default: 'dev'
      
      target-testplan-id:
        description: "target testplan id: 24569079. If not set, will not archive ado testplan"
        required: false
        type: string
        default: '24569079'

      target-testplan-name:
        description: "For example: CY230919. Sync test result to this test plan."
        required: false
        type: string

      get-coverage:
        description: "check e2e coverage for fx-core"
        required: false
        type: boolean
        default: false

      cli-version:
        description: "cli version"
        required: false
        type: string
        default: ""

  schedule:
    - cron: "0 22 * * *"

  pull_request:

permissions:
  actions: read

jobs:
  setup:
    if: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.get-coverage == 'false' ) || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'teamsfx-bot/TeamsFx') }}
    runs-on: ubuntu-latest
    environment: engineering
    permissions:
      id-token: write
      contents: read
    outputs:
      cases: ${{ steps.schedule-cases.outputs.cases || steps.dispatch-cases.outputs.cases || steps.pr-cases.outputs.cases }}
    env:
      AZURE_ACCOUNT_NAME: ${{ secrets.TEST_USER_NAME }}
      AZURE_ACCOUNT_OBJECT_ID: ${{ secrets.TEST_USER_OBJECT_ID }}
      AZURE_ACCOUNT_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.TEST_SUBSCRIPTION_ID }}
      AZURE_TENANT_ID: ${{ secrets.TEST_TENANT_ID }}
      M365_ACCOUNT_NAME: ${{ secrets.TEST_M365_NAME }}
      M365_ACCOUNT_PASSWORD: ${{ secrets.TEST_M365_PASSWORD }}
      M365_TENANT_ID: ${{ secrets.TEST_M365_TENANT_ID }}
      CI_ENABLED: "true"
      M365_ACCOUNT_COLLABORATOR: ${{ secrets.TEST_COLLABORATOR_USER_NAME_2 }}
      AUTO_TEST_PLAN_ID: ${{ github.event.inputs.target-testplan-id }}
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - uses: azure/login@v1
        with:
          client-id: ${{secrets.DEVOPS_CLIENT_ID}}
          tenant-id: ${{secrets.DEVOPS_TENANT_ID}}
          subscription-id: ${{secrets.DEVOPS_SUB_ID}}

      - name: setup project
        uses: ./.github/actions/setup-project

      - name: List cases for schedule
        id: schedule-cases
        if: ${{  github.event_name == 'schedule' }}
        working-directory: packages/tests/src/e2e
        run: |
          cases=`find . -wholename "*.tests.ts" | jq -Rsc '[split("\n") | .[]| select(.!="")]'`
          echo "cases=$cases" >> $GITHUB_OUTPUT

      - name: List cases for dispatch
        id: dispatch-cases
        if: ${{  github.event_name == 'workflow_dispatch' }}
        working-directory: packages/tests/src/e2e
        run: |
          inputCases='${{ github.event.inputs.cases }}'
          if [ -z "$inputCases" ]; then
            allCases=`find . -wholename "*.tests.ts" | jq -Rsc '[split("\n") | .[]| select(.!="")]'`
            echo "cases=$allCases" >> $GITHUB_OUTPUT
          else
            echo "cases=$inputCases" >> $GITHUB_OUTPUT
          fi

      - name: List cases for pull request
        id: pr-cases
        if: ${{  github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'teamsfx-bot/TeamsFx'  }}
        working-directory: packages/tests/src/e2e
        run: |
          cases=`find ./bot ./frontend -path "*.tests.ts" -and \
            -not '(' -path '*/*.dotnet.tests.ts' -or -path '*/Blazor*' ')' \
            | jq -Rsc '[split("\n") | .[]| select(.!="")]'`
          echo "cases=$cases" >> $GITHUB_OUTPUT

      - name: E2E Test clean
        working-directory: packages/tests
        run: |
          npm run test:e2e:clean

      - name: Archive Test Plan
        if: ${{ github.event.inputs.target-testplan-name != '' }}
        working-directory: ./packages/tests
        run: |
          pnpm install
          testplanid=`npx ts-node src/scripts/testPlan.ts obtain vscode ${{ github.event.inputs.target-testplan-name }}`
          npx ts-node src/scripts/testPlan.ts archive $testplanid
  
      - name: Upload testplan to artifact
        if: ${{ github.event.inputs.target-testplan-name != '' }}
        uses: actions/upload-artifact@v4
        with:
          name: testplan
          path: |
            ./packages/tests/testplan.json


  execute-case:
    if: ${{ needs.setup.outputs.cases }}
    env:
      AZURE_ACCOUNT_NAME: ${{ secrets.TEST_USER_NAME }}
      AZURE_ACCOUNT_OBJECT_ID: ${{ secrets.TEST_USER_OBJECT_ID }}
      AZURE_ACCOUNT_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.TEST_SUBSCRIPTION_ID }}
      AZURE_TENANT_ID: ${{ secrets.TEST_TENANT_ID }}
      M365_ACCOUNT_NAME: ${{ secrets.TEST_M365_NAME }}
      M365_ACCOUNT_PASSWORD: ${{ secrets.TEST_M365_PASSWORD }}
      M365_TENANT_ID: ${{ secrets.TEST_M365_TENANT_ID }}
      CI_ENABLED: "true"
      M365_ACCOUNT_COLLABORATOR: ${{ secrets.TEST_COLLABORATOR_USER_NAME_2 }}
      TEAMSFX_DEBUG_TEMPLATE: "true"
      NODE_ENV: "development"
      TEAMSFX_AAD_DEPLOY_ONLY: "true"
      SIDELOADING_SERVICE_ENDPOINT: ${{ secrets.SIDELOADING_SERVICE_ENDPOINT }}
      SIDELOADING_SERVICE_SCOPE: ${{ secrets.SIDELOADING_SERVICE_SCOPE }}
    needs: setup
    runs-on: ubuntu-latest
    environment: engineering
    permissions:
      id-token: write
      contents: read
    strategy:
      fail-fast: false
      matrix:
        cases: ${{ fromJson(needs.setup.outputs.cases) }}
    name: ${{ matrix.cases }}
    outputs:
      cases: ${{ matrix.cases }}

    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - uses: azure/login@v1
        with:
          client-id: ${{secrets.DEVOPS_CLIENT_ID}}
          tenant-id: ${{secrets.DEVOPS_TENANT_ID}}
          subscription-id: ${{secrets.DEVOPS_SUB_ID}}

      - name: Setup node
        uses: actions/setup-node@v3
        with:
          node-version: 18

      - name: Setup node for SPFx project
        if: contains(matrix.cases, 'SPFx')
        uses: actions/setup-node@v3
        with:
          node-version: 16

      - name: Setup Azure Functions Core Tools For Linux
        run: |
          curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg
          sudo mv microsoft.gpg /etc/apt/trusted.gpg.d/microsoft.gpg
          sudo sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-$(lsb_release -cs)-prod $(lsb_release -cs) main" > /etc/apt/sources.list.d/dotnetdev.list'
          sudo apt-get update
          sudo apt-get install azure-functions-core-tools-4
          which func
          func --version

      - name: Setup .net
        uses: actions/setup-dotnet@v3
        with:
          dotnet-version: |
            6.0.x
            8.0.x

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - uses: pnpm/action-setup@v4

      - name: Setup project
        run: |
          npm run setup

      - name: install cli with specific version
        if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.cli-version != '' }}
        run: |
          npm install -g @microsoft/teamsapp-cli@${{ github.event.inputs.cli-version }}

      - name: install cli for schedule
        if: ${{  github.event_name == 'schedule' || ( github.event.ref == 'refs/heads/dev' && github.event_name == 'workflow_dispatch' && github.event.inputs.cli-version == '' ) }}
        run: |
          npm install -g @microsoft/teamsapp-cli@alpha
      
      - name: link cli for workflow_dispatch
        if: ${{  github.event_name == 'workflow_dispatch' && github.event.ref != 'refs/heads/dev' && github.event.inputs.cli-version == '' }}
        run: |
          pnpm link --global

      - name: Update CLI and legacy-peer-deps for PR cases
        if: ${{  github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'teamsfx-bot/TeamsFx'  }}
        run: |
          npm install -g @microsoft/teamsapp-cli@alpha
          npm config set legacy-peer-deps false

      - name: print system info
        run: |
          lscpu

      - name: Download samples(daily)
        if: github.event_name == 'schedule' && startsWith(matrix.cases, './samples/') && contains(matrix.cases, 'ProactiveMessage') == false
        uses: actions/checkout@v3
        with:
          repository: OfficeDev/TeamsFx-Samples
          ref: dev
          path: packages/tests/src/e2e/resource

      - name: Download samples(rc)
        if: github.event_name == 'workflow_dispatch' && startsWith(matrix.cases, './samples/') && contains(matrix.cases, 'ProactiveMessage') == false && contains(matrix.cases, 'SignatureOutlook') == false && contains(matrix.cases, 'ChefBot') == false
        uses: actions/checkout@v3
        with:
          repository: OfficeDev/TeamsFx-Samples
          ref: ${{ github.event.inputs.sample-branch }}
          path: packages/tests/src/e2e/resource

      - name: Download samples from another repo
        if: startsWith(matrix.cases, './samples/') && contains(matrix.cases, 'ProactiveMessage')
        uses: actions/checkout@v3
        with:
          repository: OfficeDev/Microsoft-Teams-Samples
          ref: main
          path: packages/tests/src/e2e/resource
      
      - name: Download samples from office repo
        if: startsWith(matrix.cases, './samples/') && contains(matrix.cases, 'SignatureOutlook')
        uses: actions/checkout@v3
        with:
          repository: OfficeDev/Office-Add-in-samples
          ref: main
          path: packages/tests/src/e2e/resource

      - name: Download samples from ai repo
        if: startsWith(matrix.cases, './samples/') && contains(matrix.cases, 'ChefBot')
        uses: actions/checkout@v3
        with:
          repository: microsoft/teams-ai
          ref: main
          path: packages/tests/src/e2e/resource

      - name: run test
        working-directory: packages/tests/src/e2e
        run: |
          file=`find . -wholename "${{ matrix.cases }}"`
          if [ -z "$file" ]; then
            echo "can't find target case in $file"
            exit 1
          else
            npx mocha --reporter mochawesome --timeout 1200000 $file
          fi

      - name: get report name
        id: get-report-name
        if: ${{ always() }}
        run: |
          name="${{ matrix.cases }}"
          name="${name//'.tests.ts'/}"
          name="${name//.\//}"
          name="${name//\//_}"
          echo "name=$name" >> $GITHUB_OUTPUT

      - name: Upload test report
        uses: actions/upload-artifact@v4
        if: ${{ github.event_name != 'schedule' || success() || (failure() && github.run_attempt >= 5) }}
        with:
          name: test-result-${{ steps.get-report-name.outputs.name }}
          path: |
            ./packages/tests/src/e2e/mochawesome-report/mochawesome.json
      
      - name: Download TestPlan
        if: ${{ always() && github.event.inputs.target-testplan-name != '' }}
        uses: actions/download-artifact@v4
        with:
          name: testplan
          path: ./packages/tests

      - name: Sync to Azure DevOps Test Plan
        if: ${{ always() && github.event.inputs.target-testplan-name != '' }}
        working-directory: packages/tests
        run: |
          npx ts-node src/scripts/testPlan.ts report ./testplan.json src/e2e/mochawesome-report

  tear-down:
    needs: [execute-case, e2e-coverage]
    if: ${{always() && (toJSON(needs.execute-case.outputs.cases) != 'null' || needs.e2e-coverage.result != 'skipped' )}}
    runs-on: ubuntu-latest
    env:
      AZURE_ACCOUNT_NAME: ${{ secrets.TEST_USER_NAME }}
      AZURE_ACCOUNT_OBJECT_ID: ${{ secrets.TEST_USER_OBJECT_ID }}
      AZURE_ACCOUNT_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.TEST_SUBSCRIPTION_ID }}
      AZURE_TENANT_ID: ${{ secrets.TEST_TENANT_ID }}
      M365_ACCOUNT_NAME: ${{ secrets.TEST_M365_NAME }}
      M365_ACCOUNT_PASSWORD: ${{ secrets.TEST_M365_PASSWORD }}
      M365_TENANT_ID: ${{ secrets.TEST_M365_TENANT_ID }}
      CI_ENABLED: "true"
      M365_ACCOUNT_COLLABORATOR: ${{ secrets.TEST_COLLABORATOR_USER_NAME_2 }}
    steps:
      - name: Checkout (dev)
        uses: actions/checkout@v3

      - name: setup project
        uses: ./.github/actions/setup-project

      - name: E2E Test clean
        working-directory: packages/tests
        run: |
          npm run test:e2e:clean

  rerun:
    permissions:
      actions: write
    needs: tear-down
    if: ${{ (github.event_name == 'schedule' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'teamsfx-bot/TeamsFx')) && failure() && github.run_attempt < 5 }}
    runs-on: ubuntu-latest
    steps:
      - name: rerun
        run: |
          curl \
           -X POST \
           -H "Accept: application/vnd.github+json" \
           -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}"\
           -H "X-GitHub-Api-Version: 2022-11-28" \
           https://api.github.com/repos/${{ github.repository }}/actions/workflows/rerun.yml/dispatches \
           -d '{"ref":"${{ github.ref_name }}","inputs":{"run_id":"${{ github.run_id }}", "max_attempts":"5"}}'

  report:
    needs: execute-case
    if: ${{ github.event_name == 'schedule' && (success() || (failure() && github.run_attempt >= 5)) }}
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Install Dateutils
        run: |
          sudo apt install dateutils

      - uses: actions/download-artifact@v4
        with:
          path: ~/artifacts

      - name: List jobs
        id: list-jobs
        working-directory: packages/tests
        run: |
          page=1
          jobs="[]"

          while :
          do

            url=https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}/jobs\?per_page\=100\&page\=$page
            resp=`curl -H "Accept: application/vnd.github.v3+json" -u:${{ secrets.GITHUB_TOKEN }} $url`

            new_jobs=`echo $resp | jq -cr '.jobs'`

            jobs=`jq -cr --slurp 'add' <(echo "$jobs") <(echo "$new_jobs")`

            has_next=`curl -I -H "Accept: application/vnd.github.v3+json" -u:${{ secrets.GITHUB_TOKEN }} $url | grep -Fi "link:" | grep "rel=\"last\"" || true`
            if [ -z "$has_next" ]; then
              break
            fi

            page=$((page+1))
          done

          cases=`echo $jobs| jq -r '.[] | select(.name | contains("tests.ts")) | .name'`

          passed=0
          failed=0
          skipped=0
          passed_lists=""
          failed_lists=""
          skipped_lists=""

          emails="teamsfxqa@microsoft.com;"
          while IFS= read -r case;
          do
            if [ -z "$case" ]; then
              continue
            fi

            file="src/e2e/$case"
            elegant_path="${file//.\//}"

            started_at=`echo $jobs | jq --arg case $case -r '.[] | select(.name == $case ) | .steps[] | select(.name == "run test") | .started_at'`
            completed_at=`echo $jobs | jq --arg case $case -r '.[] | select(.name == $case ) | .steps[] | select(.name == "run test") | .completed_at'`
            duration=`dateutils.ddiff $started_at $completed_at -f "%Mm %Ss"`

            email=""
            if grep -q "@author" $file; then
              email=`grep '@author' $file | grep -i -o '[A-Z0-9._%+-]\+@[A-Z0-9.-]\+\.[A-Z]\{2,4\}'`
            fi

            author=""
            if [ -z "$email" ]; then
              author="N/A"
            else
              author="<a href=\\\"mailto:$email\\\"><span>$email</span></a>"
            fi

            url=`echo $jobs | jq --arg case $case -r '.[] | select(.name == $case ) | .html_url'`
            url="<a href=\\\"$url\\\">$elegant_path</a>"

            target_type="TS/JS"
            if [[ $case == *".dotnet."* ]]; then
              target_type=".NET"
            fi

            name="${case//'.tests.ts'/}"
            name="${name//.\//}"
            name="${name//\//_}"
            job_id=`echo $jobs | jq --arg case $case -r '.[] | select(.name == $case ) | .id'`
            report_file=`find ~/artifacts -wholename "*${name}*/mochawesome.json"`
            if [ ! -z "$report_file" ]; then
              echo "Found the $report_file with $job_id and $name"
              tests=`cat $report_file | jq -cr '[ .. | objects | with_entries(select(.key=="tests")) | select(. != {}) | select(.tests | type=="array") ] | map(.tests | .[]) | .[]'`
              lable=""
              while IFS= read -r test;
              do
                name=`echo $test | jq -cr .fullTitle`
                duration=`echo $test | jq -cr .duration`
                if [[ ! -z `echo $test | jq 'select(.pass==true)'` ]]; then
                  passed=$((passed+1))
                  label="<span style=\\\"background-color:#2aa198;color:white;font-weight:bold;\\\">PASSED</span>"
                elif [[ ! -z `echo $test | jq 'select(.fail==true)'` ]]; then
                  failed=$((failed+1))
                  label="<span style=\\\"background-color: #dc322f;color:white;font-weight:bold;\\\">FAILED</span>"
                  if [[ ! -z "$email" && ! "$emails" == *"$email"* ]]; then
                    emails="$emails;$email;zhendr@microsoft.com;ccdevexperiencefc@microsoft.com"
                  fi
                elif [[ ! -z `echo $test | jq 'select(.skipped==true)'` || ! -z `echo $test | jq 'select(.pending==true)'` ]]; then
                  skipped=$((skipped+1))
                  label="<span style=\\\"background-color: #b58900;color:white;font-weight:bold;\\\">SKIPPED</span>"
                fi
                
                row="<tr> <td style=\\\"text-align: left;\\\">$url</td> <td style=\\\"text-align: left;\\\">$name</td> <td style=\\\"text-align: center;\\\">$target_type</td> <td style=\\\"text-align: center;\\\">$label</td> <td style=\\\"text-align: center;\\\">$author</td> <td>$((duration/1000)) sec</td> </tr>"
                if [[ ! -z `echo $test | jq 'select(.pass==true)'` ]]; then
                  passed_lists="$passed_lists $row"
                elif [[ ! -z `echo $test | jq 'select(.fail==true)'` ]]; then
                  failed_lists="$failed_lists $row"
                elif [[ ! -z `echo $test | jq 'select(.skipped==true)'` || ! -z `echo $test | jq 'select(.pending==true)'` ]]; then
                  skipped_lists="$skipped_lists $row"
                fi
              done <<< $tests
            else
              echo "Failed to find the $report_file with $job_id and $name"
            fi

          done <<< $cases

          body="Dashboard App: <a href=\\\"https:\/\/teams.microsoft.com\/l\/entity\/c439ae8d-3ab3-4efd-9223-87366d8c170c\/_djb2_msteams_prefix_1252604900?context=%7B%22channelId%22%3A%2219%3A79488ced607f4fbf8d8433e931cad176%40thread.tacv2%22%7D&tenantId=72f988bf-86f1-41af-91ab-2d7cd011db47\\\">Click Here to Open Dashboard App</a>  <table class=\\\"w3-table w3-striped w3-bordered\\\"> <tr> <th>PATH</th> <th>CASE</th> <th>TARGET TYPE</th> <th>STATUS</th> <th>AUTHOR</th> <th>DURATION</th> </tr> $failed_lists $skipped_lists $passed_lists </table> <br />"

          total=$((passed+failed+skipped))

          subject="TeamsFx E2E V3 Test Report ($passed/$total Passed, $failed/$total Failed, $skipped/$total Skipped)"
          if [ $failed -gt 0 ]; then
            subject="[FAILED] $subject"
          else
            subject="[PASSED] $subject"
          fi

          echo "body=$body" >> $GITHUB_OUTPUT
          echo "to=$emails" >> $GITHUB_OUTPUT
          echo "subject=$subject" >> $GITHUB_OUTPUT
          
      - name: Send E-mail to the whole team
        uses: ./.github/actions/send-email-report
        env:
          TO: ${{ steps.list-jobs.outputs.to }}
          BODY: '"${{ steps.list-jobs.outputs.body }}"'
          SUBJECT: ${{ steps.list-jobs.outputs.subject }}
          MAIL_CLIENT_ID: ${{ secrets.TEST_CLEAN_CLIENT_ID }}
          MAIL_CLIENT_SECRET: ${{ secrets.TEST_CLEAN_CLIENT_SECRET }}
          MAIL_TENANT_ID: ${{ secrets.TEST_CLEAN_TENANT_ID }}

  e2e-coverage:
    if: ${{github.event_name == 'workflow_dispatch' && github.event.inputs.get-coverage == 'true' }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-node@v3
        with:
          node-version: 18
      - uses: pnpm/action-setup@v4

      - name: mv files
        working-directory: packages/tests
        run: |
          rm src/e2e/commonUtils.ts src/utils/commonUtils.ts
          mv src/e2e/commonUtils.ts.back src/e2e/commonUtils.ts
          mv src/utils/commonUtils.ts.back src/utils/commonUtils.ts

      - uses: jossef/action-set-json-field@v1
        with:
            file: ./packages/fx-core/.nycrc
            field: excludeAfterRemap
            value: false

      - name: Update config.json
        working-directory: packages/fx-core
        run: echo "`jq '.include=["build"]' .nycrc`" > .nycrc

      - name: less nycrc
        working-directory: packages/fx-core
        run: |
          less .nycrc
      
      - name: setup
        run: |
          npm run setup

      - name: Download samples(daily)
        working-directory: packages/tests/src/e2e
        run: |
          mkdir resource && cd resource
          wget https://github.com/OfficeDev/TeamsFx-Samples/archive/refs/heads/dev.zip
          unzip dev.zip -d .
          mv TeamsFx-Samples-dev/* .
          rm -rf dev.zip TeamsFx-Samples-dev

      - name: run test for coverage
        working-directory: packages/fx-core
        env:
          AZURE_ACCOUNT_NAME: ${{ secrets.TEST_USER_NAME }}
          AZURE_ACCOUNT_OBJECT_ID: ${{ secrets.TEST_USER_OBJECT_ID }}
          AZURE_ACCOUNT_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
          AZURE_SUBSCRIPTION_ID: ${{ secrets.TEST_SUBSCRIPTION_ID }}
          AZURE_TENANT_ID: ${{ secrets.TEST_TENANT_ID }}
          M365_ACCOUNT_NAME: ${{ secrets.TEST_USER_NAME }}
          M365_ACCOUNT_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
          M365_TENANT_ID: ${{ secrets.TEST_TENANT_ID_2 }}
          CI_ENABLED: "true"
          M365_ACCOUNT_COLLABORATOR: ${{ secrets.TEST_COLLABORATOR_USER_NAME }}
          TEAMSFX_DEBUG_TEMPLATE: "true"
          NODE_ENV: "development"
          TEAMSFX_AAD_DEPLOY_ONLY: "true"
          SIDELOADING_SERVICE_ENDPOINT: ${{ secrets.SIDELOADING_SERVICE_ENDPOINT }}
          SIDELOADING_SERVICE_SCOPE: ${{ secrets.SIDELOADING_SERVICE_SCOPE }}
        run: |
          npx nyc mocha --parallel ../tests/src/e2e/*/*.tests.ts
      
      - name: pack zip
        if: ${{always()}}
        working-directory: packages/fx-core
        run: |
          zip -r out1.zip .nyc_output/
          zip -r out2.zip coverage/

      - uses: actions/upload-artifact@v4
        if: ${{ always() }}
        with:
          name: coverage
          path: |
            packages/fx-core/out1.zip
            packages/fx-core/out2.zip

      - name: CodeCov report attempt
        if: ${{always()}}
        continue-on-error: true
        uses: codecov/codecov-action@v3
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          fail_ci_if_error: true
          verbose: true