From 33dbbfdc53ef150b7bd24675a544154a21191d91 Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Wed, 16 Oct 2024 15:52:22 +0100 Subject: [PATCH] Fix pytest from working outside breeze This was missed in https://github.com/apache/airflow/pull/42985 . Without this `airflow.providers.__path__` had 2 registered paths: ``` ['/Users/kaxilnaik/Documents/GitHub/astronomer/airflow/providers/src/airflow/providers', '/Users/kaxilnaik/Documents/GitHub/astronomer/airflow/providers'] ``` This prevents the tests from running outside of breeze and we get the following error: ``` ERROR tests/core/test_settings.py::test_usage_data_collection_disabled[true-True-True] - airflow.exceptions.AirflowConfigException: ("The provider apache-airflow-providers-src-airflow-providers-amazon is attempting to contribute configuration section aws that has already been added before. The source of it: apache-airflow-providers-amazon. This is forbidden. A provider can only add new sections. It cannot contribute options to existing sections or override other provider's configuration.", ) ``` We get this error because the *Providers Manager* uses `airflow.providers.__path__` to register providers. Because we have 2 paths, it registers the same provider twice leading two the above error. https://github.com/apache/airflow/blob/75b22940ac4d36c31380669da2aa32fe46d70d32/airflow/providers_manager.py#L662 Example registration: ``` ('apache-airflow-providers-src-airflow-providers-yandex', {'yandex': {'description': 'This section contains settings for Yandex Cloud ', ... ('apache-airflow-providers-yandex', {'yandex': {'description': 'This section contains settings for Yandex Cloud ' ``` This wasn't a problem in breeze as it sets `AIRFLOW_SOURCES` env var in Dockerfile https://github.com/apache/airflow/blob/75b22940ac4d36c31380669da2aa32fe46d70d32/scripts/docker/entrypoint_ci.sh#L24 --- tests_common/pytest_plugin.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests_common/pytest_plugin.py b/tests_common/pytest_plugin.py index 60638d8a5a756..bf34ae5be5f4b 100644 --- a/tests_common/pytest_plugin.py +++ b/tests_common/pytest_plugin.py @@ -114,7 +114,7 @@ _airflow_sources = os.getenv("AIRFLOW_SOURCES", None) AIRFLOW_SOURCES_ROOT_DIR = ( - Path(_airflow_sources) if _airflow_sources else Path(__file__).parents[2] + Path(_airflow_sources) if _airflow_sources else Path(__file__).parents[1] ).resolve() AIRFLOW_TESTS_DIR = AIRFLOW_SOURCES_ROOT_DIR / "tests" @@ -376,6 +376,8 @@ def pytest_configure(config: pytest.Config) -> None: if path == desired: break else: + # This "desired" path should be the Airflow source directory (repo root) + assert (AIRFLOW_SOURCES_ROOT_DIR / ".asf.yaml").exists(), f"Path {desired} is not Airflow root" sys.path.append(desired) if (backend := config.getoption("backend", default=None)) and backend not in SUPPORTED_DB_BACKENDS: