feat(plugins): Create concurrency pytest plugin (#824)

* refactor(plugins): Create `concurrency` plugin

* docs: changelog

* fix(plugins): Use `session_temp_folder_name` fixture

* fix(plugins): nit

* Apply suggestions from code review

Co-authored-by: danceratopz <[email protected]>

* fix(fw): tox

---------

Co-authored-by: danceratopz <[email protected]>

docs/CHANGELOG.md

@@ -50,6 +50,7 @@ Test fixtures for use by clients are available for each release on the [Github r
 - ✨ Releases for feature eip7692 now include both Cancun and Prague based tests in the same release, in files `fixtures_eip7692.tar.gz` and `fixtures_eip7692-prague.tar.gz` respectively ([#743](https://github.com/ethereum/execution-spec-tests/pull/743)).
 ✨ Re-write the test case reference doc flow as a pytest plugin and add pages for test functions with a table providing an overview of their parametrized test cases ([#801](https://github.com/ethereum/execution-spec-tests/pull/801)).
 - 🔀 Simplify Python project configuration and consolidate it into `pyproject.toml` ([#764](https://github.com/ethereum/execution-spec-tests/pull/764)).
+- 🔀 Created `pytest_plugins.concurrency` plugin to sync multiple `xdist` processes without using a command flag to specify the temporary working folder ([#824](https://github.com/ethereum/execution-spec-tests/pull/824))
 - 🔀 Move pytest plugin `pytest_plugins.filler.solc` to `pytest_plugins.solc.solc` ([#823](https://github.com/ethereum/execution-spec-tests/pull/823)).
 
 ### 💥 Breaking Change

pytest-consume.ini

@@ -5,5 +5,6 @@ python_files = test_*
 addopts = 
     -rxXs
     --tb short
+    -p pytest_plugins.concurrency
     -p pytest_plugins.consume.consume
     -p pytest_plugins.help.help

pytest.ini

@@ -7,6 +7,7 @@ markers =
     slow
     pre_alloc_modify
 addopts = 
+    -p pytest_plugins.concurrency
     -p pytest_plugins.filler.pre_alloc
     -p pytest_plugins.solc.solc
     -p pytest_plugins.filler.filler

src/cli/pytest_commands/consume.py

@@ -6,7 +6,6 @@
 import sys
 import warnings
 from pathlib import Path
-from tempfile import TemporaryDirectory
 from typing import Any, Callable, List
 
 import click
@@ -92,13 +91,7 @@ def create_command(
         def command(pytest_args: List[str], **kwargs) -> None:
             args = handle_consume_command_flags(pytest_args, is_hive)
             args += [str(p) for p in command_paths]
-            if is_hive and not any(arg.startswith("--hive-session-temp-folder") for arg in args):
-                with TemporaryDirectory() as temp_dir:
-                    args.extend(["--hive-session-temp-folder", temp_dir])
-                    result = pytest.main(args)
-            else:
-                result = pytest.main(args)
-            sys.exit(result)
+            sys.exit(pytest.main(args))
 
         return command
 

src/cli/pytest_commands/fill.py

@@ -3,7 +3,6 @@
 """
 
 import sys
-from tempfile import TemporaryDirectory
 from typing import List
 
 import click
@@ -60,10 +59,9 @@ def fill(pytest_args: List[str], **kwargs) -> None:
     """
     Entry point for the fill command.
     """
-    with TemporaryDirectory() as temp_dir:
-        result = pytest.main(
-            handle_fill_command_flags(
-                [f"--session-temp-folder={temp_dir}", "--index", *pytest_args],
-            ),
-        )
+    result = pytest.main(
+        handle_fill_command_flags(
+            ["--index", *pytest_args],
+        ),
+    )
     sys.exit(result)

src/pytest_plugins/concurrency.py

@@ -0,0 +1,80 @@
+"""
+Pytest plugin to create a temporary folder for the session where
+multi-process tests can store data that is shared between processes.
+
+The provided `session_temp_folder` fixture is used, for example, by `consume`
+when running hive simulators to ensure that only one `test_suite` is created
+(used to create tests on the hive simulator) when they are being created using
+multiple workers with pytest-xdist.
+"""
+
+import os
+import shutil
+from pathlib import Path
+from tempfile import gettempdir as get_temp_dir  # noqa: SC200
+from typing import Generator
+
+import pytest
+from filelock import FileLock
+
+
+@pytest.fixture(scope="session")
+def session_temp_folder_name(testrun_uid: str) -> str:  # noqa: SC200
+    """
+    Define the name of the temporary folder that will be shared among all the
+    xdist workers to coordinate the tests.
+
+    "testrun_uid" is a fixture provided by the xdist plugin, and is unique for each test run,
+    so it is used to create the unique folder name.
+    """
+    return f"pytest-{testrun_uid}"  # noqa: SC200
+
+
+@pytest.fixture(scope="session")
+def session_temp_folder(
+    session_temp_folder_name: str,
+) -> Generator[Path, None, None]:
+    """
+    Create a global temporary folder that will be shared among all the
+    xdist workers to coordinate the tests.
+
+    We also create a file to keep track of how many workers are still using the folder, so we can
+    delete it when the last worker is done.
+    """
+    session_temp_folder = Path(get_temp_dir()) / session_temp_folder_name
+    session_temp_folder.mkdir(exist_ok=True)
+
+    folder_users_file_name = "folder_users"
+    folder_users_file = session_temp_folder / folder_users_file_name
+    folder_users_lock_file = session_temp_folder / f"{folder_users_file_name}.lock"
+
+    with FileLock(folder_users_lock_file):
+        if folder_users_file.exists():
+            with folder_users_file.open("r") as f:
+                folder_users = int(f.read())
+        else:
+            folder_users = 0
+        folder_users += 1
+        with folder_users_file.open("w") as f:
+            f.write(str(folder_users))
+
+    yield session_temp_folder
+
+    with FileLock(folder_users_lock_file):
+        with folder_users_file.open("r") as f:
+            folder_users = int(f.read())
+        folder_users -= 1
+        if folder_users == 0:
+            shutil.rmtree(session_temp_folder)
+        else:
+            with folder_users_file.open("w") as f:
+                f.write(str(folder_users))
+
+
+@pytest.fixture(scope="session")
+def worker_count() -> int:
+    """
+    Get the number of workers for the test.
+    """
+    worker_count_env = os.environ.get("PYTEST_XDIST_WORKER_COUNT", "1")
+    return max(int(worker_count_env), 1)

src/pytest_plugins/filler/filler.py

@@ -5,8 +5,6 @@
 and that modifies pytest hooks in order to fill test specs for all tests and
 writes the generated fixtures to file.
 """
-
-import argparse
 import configparser
 import datetime
 import os
@@ -191,16 +189,6 @@ def pytest_addoption(parser: pytest.Parser):
         help="Path to dump the transition tool debug output.",
     )
 
-    internal_group = parser.getgroup("internal", "Internal arguments")
-    internal_group.addoption(
-        "--session-temp-folder",
-        action="store",
-        dest="session_temp_folder",
-        type=Path,
-        default=None,
-        help=argparse.SUPPRESS,
-    )
-
 
 @pytest.hookimpl(tryfirst=True)
 def pytest_configure(config):
@@ -603,11 +591,6 @@ def get_fixture_collection_scope(fixture_name, config):
     return "module"
 
 
-@pytest.fixture(scope="session")
-def session_temp_folder(request) -> Path | None:  # noqa: D103
-    return request.config.option.session_temp_folder
-
-
 @pytest.fixture(scope="session")
 def generate_index(request) -> bool:  # noqa: D103
     return request.config.option.generate_index

src/pytest_plugins/pytest_hive/pytest_hive.py

@@ -8,12 +8,10 @@
 
 These fixtures are used when creating the hive test suite.
 """
-import argparse
 import json
 import os
 from dataclasses import asdict
 from pathlib import Path
-from tempfile import TemporaryDirectory
 
 import pytest
 from filelock import FileLock
@@ -23,7 +21,7 @@
 
 
 def pytest_configure(config):  # noqa: D103
-    hive_simulator_url = os.environ.get("HIVE_SIMULATOR")
+    hive_simulator_url = config.getoption("hive_simulator")
     if hive_simulator_url is None:
         pytest.exit(
             "The HIVE_SIMULATOR environment variable is not set.\n\n"
@@ -58,12 +56,14 @@ def pytest_configure(config):  # noqa: D103
 def pytest_addoption(parser: pytest.Parser):  # noqa: D103
     pytest_hive_group = parser.getgroup("pytest_hive", "Arguments related to pytest hive")
     pytest_hive_group.addoption(
-        "--hive-session-temp-folder",
+        "--hive-simulator",
         action="store",
-        dest="hive_session_temp_folder",
-        type=Path,
-        default=TemporaryDirectory(),
-        help=argparse.SUPPRESS,
+        dest="hive_simulator",
+        default=os.environ.get("HIVE_SIMULATOR"),
+        help=(
+            "The Hive simulator endpoint, e.g. http://127.0.0.1:3000. By default, the value is "
+            "taken from the HIVE_SIMULATOR environment variable."
+        ),
     )