From 254eac1a69684c0b036dc5e797f3dec4f29610ac Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Thu, 5 Dec 2024 12:49:27 +0000 Subject: [PATCH] Fix compatibility with apache-airflow-providers-cncf-kubernetes>=10.0.0 (#311) Users were not able to use DAG Factory with apache-airflow-providers-cncf-kubernetes>=10.0.0, which was [released](https://pypi.org/project/apache-airflow-providers-cncf-kubernetes/#history) on 18 November 2024. When attempting to install a local dev environment using: ``` python3.10 -m venv venv source venv/bin/activate pip install '.[tests]' ``` If we attempted to run: ``` ln -s dev/dags dags AIRFLOW_HOME=`pwd` airflow dags list-import-errors ``` All DAG factory example DAGs would error with messages like: ``` filepath | error ===============================================================+============================================================== /Users/tati/Code/dag-factory/dags/example_da | Traceback (most recent call last): g_factory.py | File | "/Users/tati/Code/dag-factory/dagfactory/da | gbuilder.py", line 69, in | from | airflow.providers.cncf.kubernetes.operators.kubernetes_pod | import KubernetesPodOperator | ModuleNotFoundError: No module named | 'airflow.providers.cncf.kubernetes.operators.kubernetes_pod' | | During handling of the above exception, another exception | occurred: | | Traceback (most recent call last): | File | "/Users/tati/Code/dag-factory/dagfactory/da | gfactory.py", line 13, in | from dagfactory.dagbuilder import DagBuilder | File | "/Users/tati/Code/dag-factory/dagfactory/da | gbuilder.py", line 71, in | from airflow.contrib.kubernetes.pod import Port | ModuleNotFoundError: No module named | 'airflow.contrib.kubernetes' | /Users/tati/Code/dag-factory/dags/example_ta | Traceback (most recent call last): sk_group.py | File | "/Users/tati/Code/dag-factory/dagfactory/da | gbuilder.py", line 69, in | from | airflow.providers.cncf.kubernetes.operators.kubernetes_pod | import KubernetesPodOperator | ModuleNotFoundError: No module named | 'airflow.providers.cncf.kubernetes.operators.kubernetes_pod' | | During handling of the above exception, another exception | occurred: | | Traceback (most recent call last): | File | "/Users/tati/Code/dag-factory/dagfactory/da | gfactory.py", line 13, in | from dagfactory.dagbuilder import DagBuilder | File | "/Users/tati/Code/dag-factory/dagfactory/da | gbuilder.py", line 71, in | from airflow.contrib.kubernetes.pod import Port | ModuleNotFoundError: No module named | 'airflow.contrib.kubernetes' ``` This was due to a breaking change introduced in `apache-airflow-providers-cncf-kubernetes` 10: https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/stable/changelog.html > Remove airflow.providers.cncf.kubernetes.operators.kubernetes_pod. Use airflow.providers.cncf.kubernetes.operators.pod instead. --- dagfactory/dagbuilder.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dagfactory/dagbuilder.py b/dagfactory/dagbuilder.py index 3eabf900..3ea191f7 100644 --- a/dagfactory/dagbuilder.py +++ b/dagfactory/dagbuilder.py @@ -66,7 +66,11 @@ V1VolumeMount as VolumeMount, ) from airflow.kubernetes.secret import Secret - from airflow.providers.cncf.kubernetes.operators.kubernetes_pod import KubernetesPodOperator + + if version.parse(K8S_PROVIDER_VERSION) < version.parse("10"): + from airflow.providers.cncf.kubernetes.operators.kubernetes_pod import KubernetesPodOperator + else: + from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator except ImportError: # pragma: no cover from airflow.contrib.kubernetes.pod import Port from airflow.contrib.kubernetes.pod_runtime_info_env import PodRuntimeInfoEnv