use pytest-xdist to utilize multiple workers for Python tests (#6696)

danmar · Dec 15, 2024 · 9b45922 · 9b45922
1 parent 3a0f039
commit 9b45922
Show file tree

Hide file tree

Showing 8 changed files with 88 additions and 74 deletions.
diff --git a/.github/workflows/CI-unixish.yml b/.github/workflows/CI-unixish.yml
@@ -268,6 +268,11 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-20.04, ubuntu-22.04, macos-13]
+        include:
+          - xdist_n: auto
+          # FIXME: test_color_tty fails with xdist
+          - os: macos-13
+            xdist_n: '1'
       fail-fast: false # Prefer quick result
 
     runs-on: ${{ matrix.os }}
@@ -306,6 +311,7 @@ jobs:
           python3 -m pip install pip --upgrade
           python3 -m pip install pytest
           python3 -m pip install pytest-timeout
+          python3 -m pip install pytest-xdist
           python3 -m pip install psutil
 
       - name: Build cppcheck
@@ -329,7 +335,7 @@ jobs:
 
       - name: Run test/cli
         run: |
-          python3 -m pytest -Werror --strict-markers -vv test/cli
+          python3 -m pytest -Werror --strict-markers -vv -n ${{ matrix.xdist_n }} test/cli
 
       # TODO: use the step below instead
       # do not use pushd in this step since we go below the working directory
@@ -338,26 +344,26 @@ jobs:
           cd ..
           ln -s cppcheck 'cpp check'
           cd 'cpp check/test/cli'
-          python3 -m pytest -Werror --strict-markers -vv
+          python3 -m pytest -Werror --strict-markers -vv -n ${{ matrix.xdist_n }}
 
       # FIXME: proj2_test.py fails because of the relative path cleanups in ImportProject::setRelativePaths()
       # It fails because the application path used as base path has its symlink resolved by getcwd().
       - name: Run test/cli (symlink)
         if: false
         run: |
           ln -s . 'cpp check'
-          python3 -m pytest -Werror --strict-markers -vv 'cpp check/test/cli'
+          python3 -m pytest -Werror --strict-markers -vv -n ${{ matrix.xdist_n }} 'cpp check/test/cli'
 
       - name: Run test/cli (-j2)
         run: |
-          python3 -m pytest -Werror --strict-markers -vv test/cli
+          python3 -m pytest -Werror --strict-markers -vv -n ${{ matrix.xdist_n }} test/cli
         env:
           TEST_CPPCHECK_INJECT_J: 2
 
       - name: Run test/cli (--clang)
         if: false
         run: |
-          python3 -m pytest -Werror --strict-markers -vv test/cli
+          python3 -m pytest -Werror --strict-markers -vv -n ${{ matrix.xdist_n }} test/cli
         env:
           TEST_CPPCHECK_INJECT_CLANG: clang
 

diff --git a/.github/workflows/CI-windows.yml b/.github/workflows/CI-windows.yml
@@ -143,6 +143,7 @@ jobs:
           python -m pip install pytest || exit /b !errorlevel!
           python -m pip install pytest-custom_exit_code || exit /b !errorlevel!
           python -m pip install pytest-timeout || exit /b !errorlevel!
+          python -m pip install pytest-xdist || exit /b !errorlevel!
           python -m pip install psutil || exit /b !errorlevel!
 
       # TODO: build with CMake
@@ -182,20 +183,20 @@ jobs:
       - name: Run test/cli
         if: matrix.config == 'release'
         run: |
-          python -m pytest -Werror --strict-markers -vv test/cli || exit /b !errorlevel!
+          python -m pytest -Werror --strict-markers -vv -n auto test/cli || exit /b !errorlevel!
 
       - name: Run test/cli (-j2)
         if: matrix.config == 'release'
         run: |
-          python -m pytest -Werror --strict-markers -vv test/cli || exit /b !errorlevel!
+          python -m pytest -Werror --strict-markers -vv -n auto test/cli || exit /b !errorlevel!
         env:
           TEST_CPPCHECK_INJECT_J: 2
 
       # TODO: install clang
       - name: Run test/cli (--clang)
         if: false # matrix.config == 'release'
         run: |
-          python -m pytest -Werror --strict-markers -vv test/cli || exit /b !errorlevel!
+          python -m pytest -Werror --strict-markers -vv -n auto test/cli || exit /b !errorlevel!
         env:
           TEST_CPPCHECK_INJECT_CLANG: clang
 

diff --git a/.github/workflows/asan.yml b/.github/workflows/asan.yml
@@ -67,6 +67,7 @@ jobs:
           python3 -m pip install pip --upgrade
           python3 -m pip install pytest
           python3 -m pip install pytest-timeout
+          python3 -m pip install pytest-xdist
           python3 -m pip install psutil
 
       # TODO: disable all warnings
@@ -103,20 +104,20 @@ jobs:
       - name: Run test/cli
         run: |
           pwd=$(pwd)
-          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv test/cli
+          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv -n auto test/cli
 
       - name: Run test/cli (-j2)
         run: |
           pwd=$(pwd)
-          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv test/cli
+          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv -n auto test/cli
         env:
           TEST_CPPCHECK_INJECT_J: 2
 
       - name: Run test/cli (--clang)
         if: false
         run: |
           pwd=$(pwd)
-          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv test/cli
+          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv -n auto test/cli
         env:
           TEST_CPPCHECK_INJECT_CLANG: clang
 

diff --git a/.github/workflows/scriptcheck.yml b/.github/workflows/scriptcheck.yml
@@ -92,6 +92,7 @@ jobs:
           python -m pip install 'pylint<=3.3.0'
           python -m pip install unittest2
           python -m pip install pytest
+          python -m pip install pytest-xdist
           python -m pip install pygments
           python -m pip install requests
           python -m pip install psutil
@@ -135,18 +136,20 @@ jobs:
         run: |
           python tools/test_matchcompiler.py
 
+      # TODO: run with "-n auto" when misra_test.py can be run in parallel
       # we cannot specify -Werror since xml/etree/ElementTree.py in Python 3.9/3.10 contains an unclosed file
       - name: test addons
         if: matrix.python-version == '3.9' || matrix.python-version == '3.10'
         run: |
-          python -m pytest --strict-markers -vv addons/test
+          python -m pytest --strict-markers -vv -n 1 addons/test
         env:
           PYTHONPATH: ./addons
 
+      # TODO: run with "-n auto" when misra_test.py can be run in parallel
       - name: test addons
         if: matrix.python-version != '3.9' && matrix.python-version != '3.10'
         run: |
-          python -m pytest -Werror --strict-markers -vv addons/test
+          python -m pytest -Werror --strict-markers -vv -n 1 addons/test
         env:
           PYTHONPATH: ./addons
 

diff --git a/.github/workflows/tsan.yml b/.github/workflows/tsan.yml
@@ -67,6 +67,7 @@ jobs:
           python3 -m pip install pip --upgrade
           python3 -m pip install pytest
           python3 -m pip install pytest-timeout
+          python3 -m pip install pytest-xdist
           python3 -m pip install psutil
 
       - name: CMake
@@ -103,22 +104,22 @@ jobs:
       - name: Run test/cli
         run: |
           pwd=$(pwd)
-          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv test/cli
+          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv -n auto test/cli
         env:
           TEST_CPPCHECK_INJECT_EXECUTOR: thread
 
       - name: Run test/cli (-j2)
         run: |
           pwd=$(pwd)
-          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv test/cli
+          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv -n auto test/cli
         env:
           TEST_CPPCHECK_INJECT_J: 2
 
       - name: Run test/cli (--clang)
         if: false
         run: |
           pwd=$(pwd)
-          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv test/cli
+          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv -n auto test/cli
         env:
           TEST_CPPCHECK_INJECT_CLANG: clang
 

diff --git a/.github/workflows/ubsan.yml b/.github/workflows/ubsan.yml
@@ -67,6 +67,7 @@ jobs:
           python3 -m pip install pip --upgrade
           python3 -m pip install pytest
           python3 -m pip install pytest-timeout
+          python3 -m pip install pytest-xdist
           python3 -m pip install psutil
 
       # TODO: disable warnings
@@ -103,20 +104,20 @@ jobs:
       - name: Run test/cli
         run: |
           pwd=$(pwd)
-          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv test/cli
+          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv -n auto test/cli
 
       - name: Run test/cli (-j2)
         run: |
           pwd=$(pwd)
-          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv test/cli
+          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv -n auto test/cli
         env:
           TEST_CPPCHECK_INJECT_J: 2
 
       - name: Run test/cli (--clang)
         if: false
         run: |
           pwd=$(pwd)
-          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv test/cli
+          TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv -n auto test/cli
         env:
           TEST_CPPCHECK_INJECT_CLANG: clang
 

diff --git a/addons/test/misra_test.py b/addons/test/misra_test.py
@@ -30,6 +30,7 @@ def checker():
     return MisraChecker(settings)
 
 
+# FIXME: files are generates in the source tree so it will cause issues if tests are run with xdist.
 @pytest.fixture
 def test_files():
     for f in TEST_SOURCE_FILES: