From 580cc3645ffaebfd11433a43bba78a9a23f0682c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Erik=20Bj=C3=A4reholt?= <erik@bjareho.lt>
Date: Wed, 14 Aug 2024 21:31:08 +0200
Subject: [PATCH] ci: use pytest-retry for flaky test instead of retrying the
 entire suite in ci

---
 .github/workflows/build.yml |  4 ++--
 Makefile                    |  2 +-
 poetry.lock                 | 19 ++++++++++++++++++-
 pyproject.toml              |  1 +
 tests/test_cli.py           |  1 +
 5 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 49f4484f..2d3b2062 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -56,7 +56,7 @@ jobs:
       if: contains(matrix.extras, 'browser') || contains(matrix.extras, 'all')
       run: poetry run playwright install chromium
 
-    # OpenAI sometimes randomly aborts connections, so we retry once
+    # old comment: OpenAI sometimes randomly aborts connections
     - name: Run tests
       uses: nick-fields/retry@v2
       env:
@@ -64,7 +64,7 @@ jobs:
         MODEL: ${{ matrix.provider }}
       with:
         timeout_minutes: 5
-        max_attempts: 2
+        max_attempts: 1  # favor pytest retries (mark with flaky)
         retry_wait_seconds: 10
         command: make test SLOW=true
 
diff --git a/Makefile b/Makefile
index c2751605..7e124966 100644
--- a/Makefile
+++ b/Makefile
@@ -20,7 +20,7 @@ test:
 		--cov=gptme --cov-report=xml --cov-report=term-missing --cov-report=html --junitxml=junit.xml \
 		-n 8 \
 		$(if $(EVAL), , -m "not eval") \
-		$(if $(SLOW), --timeout 60, --timeout 5 -m "not slow and not eval") \
+		$(if $(SLOW), --timeout 60 --retries 2 --retry-delay 5, --timeout 5 -m "not slow and not eval") \
 		$(if $(PROFILE), --profile-svg)
 
 eval:
diff --git a/poetry.lock b/poetry.lock
index 082dda2b..b91bcf23 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -2303,6 +2303,23 @@ six = "*"
 [package.extras]
 tests = ["pytest-virtualenv"]
 
+[[package]]
+name = "pytest-retry"
+version = "1.6.3"
+description = "Adds the ability to retry flaky tests in CI environments"
+optional = false
+python-versions = ">=3.9"
+files = [
+    {file = "pytest_retry-1.6.3-py3-none-any.whl", hash = "sha256:e96f7df77ee70b0838d1085f9c3b8b5b7d74bf8947a0baf32e2b8c71b27683c8"},
+    {file = "pytest_retry-1.6.3.tar.gz", hash = "sha256:36ccfa11c8c8f9ddad5e20375182146d040c20c4a791745139c5a99ddf1b557d"},
+]
+
+[package.dependencies]
+pytest = ">=7.0.0"
+
+[package.extras]
+dev = ["black", "flake8", "isort", "mypy"]
+
 [[package]]
 name = "pytest-timeout"
 version = "2.2.0"
@@ -3482,4 +3499,4 @@ training = ["torch", "transformers"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.10"
-content-hash = "9bffd44303bc535789e8438a0a8bbaa44821fd87e65406009e385dde6ea887c7"
+content-hash = "3a70a3cbba9b00ce58e598953c71d5af5db72087311c2750d46bba4605695742"
diff --git a/pyproject.toml b/pyproject.toml
index a7dc72ff..663613f1 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -71,6 +71,7 @@ myst-parser = "^2.0.0"
 pyupgrade = "^3.15.0"
 greenlet = "*"  # dependency of playwright, but needed for coverage
 types-tabulate = "^0.9.0.20240106"
+pytest-retry = "^1.6.3"
 
 [tool.poetry.extras]
 server = ["llama-cpp-python", "flask"]
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 9ccc97a8..40dfe47e 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -276,6 +276,7 @@ def test_terminal(args: list[str], runner: CliRunner):
 
 # TODO: move elsewhere
 @pytest.mark.slow
+@pytest.mark.flaky(retries=2, delay=5)
 def test_subagent(args: list[str], runner: CliRunner):
     # f14: 377
     # f15: 610