ci: use pytest-retry for flaky test instead of retrying the entire su…

…ite in ci

.github/workflows/build.yml

@@ -56,15 +56,15 @@ jobs:
       if: contains(matrix.extras, 'browser') || contains(matrix.extras, 'all')
       run: poetry run playwright install chromium
 
-    # OpenAI sometimes randomly aborts connections, so we retry once
+    # old comment: OpenAI sometimes randomly aborts connections
     - name: Run tests
       uses: nick-fields/retry@v2
       env:
         TERM: xterm
         MODEL: ${{ matrix.provider }}
       with:
         timeout_minutes: 5
-        max_attempts: 2
+        max_attempts: 1  # favor pytest retries (mark with flaky)
         retry_wait_seconds: 10
         command: make test SLOW=true
 

Makefile

@@ -20,7 +20,7 @@ test:
 		--cov=gptme --cov-report=xml --cov-report=term-missing --cov-report=html --junitxml=junit.xml \
 		-n 8 \
 		$(if $(EVAL), , -m "not eval") \
-		$(if $(SLOW), --timeout 60, --timeout 5 -m "not slow and not eval") \
+		$(if $(SLOW), --timeout 60 --retries 2 --retry-delay 5, --timeout 5 -m "not slow and not eval") \
 		$(if $(PROFILE), --profile-svg)
 
 eval:

pyproject.toml

@@ -71,6 +71,7 @@ myst-parser = "^2.0.0"
 pyupgrade = "^3.15.0"
 greenlet = "*"  # dependency of playwright, but needed for coverage
 types-tabulate = "^0.9.0.20240106"
+pytest-retry = "^1.6.3"
 
 [tool.poetry.extras]
 server = ["llama-cpp-python", "flask"]

tests/test_cli.py

@@ -276,6 +276,7 @@ def test_terminal(args: list[str], runner: CliRunner):
 
 # TODO: move elsewhere
 @pytest.mark.slow
+@pytest.mark.flaky(retries=2, delay=5)
 def test_subagent(args: list[str], runner: CliRunner):
     # f14: 377
     # f15: 610