Fix edge-case when conflicting constraints prevent k8s env creation (… (

#43298)

* Fix edge-case when conflicting constraints prevent k8s env creation (#43276)

The #36930 added constraints to creation of k8s environment, but it
had a side effect - the constraints could not be created if source
of airflow had dependencies conflicting with constraints (which
might happen for example when we upgrade FAB - because locally
pinned FAB might be different than the one in constraints).

Also the constraints were "hard-coded" - the python version,
branch and github repository were hard-coded.

This PR fixes both problems:

* constraints URL is dynamically generated based on current
  branch, repo and python version
* in case attempts to create the venv with constraints fails,
  we attempt to install it again without constraints

(cherry picked from commit 274b6e1)

* Update k8s_requirements.txt

dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py

@@ -33,9 +33,11 @@
 from typing import Any, NamedTuple
 from urllib import request
 
+from airflow_breeze.branch_defaults import AIRFLOW_BRANCH
 from airflow_breeze.global_constants import (
     ALLOWED_ARCHITECTURES,
     ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS,
+    APACHE_AIRFLOW_GITHUB_REPOSITORY,
     HELM_VERSION,
     KIND_VERSION,
     PIP_VERSION,
@@ -299,7 +301,7 @@ def _requirements_changed() -> bool:
 
 
 def _install_packages_in_k8s_virtualenv():
-    install_command = [
+    install_command_no_constraints = [
         str(PYTHON_BIN_PATH),
         "-m",
         "pip",
@@ -311,16 +313,42 @@ def _install_packages_in_k8s_virtualenv():
     capture_output = True
     if get_verbose():
         capture_output = False
+    python_major_minor_version = run_command(
+        [
+            str(PYTHON_BIN_PATH),
+            "-c",
+            "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')",
+        ],
+        capture_output=True,
+        check=True,
+        text=True,
+    ).stdout.strip()
+    install_command_with_constraints = install_command_no_constraints.copy()
+    install_command_with_constraints.extend(
+        [
+            "--constraint",
+            "https://raw.githubusercontent.com/"
+            f"{APACHE_AIRFLOW_GITHUB_REPOSITORY}/"
+            f"constraints-{AIRFLOW_BRANCH}/constraints-{python_major_minor_version}.txt",
+        ],
+    )
     install_packages_result = run_command(
-        install_command, check=False, capture_output=capture_output, text=True, env=env
+        install_command_with_constraints, check=False, capture_output=capture_output, text=True, env=env
     )
     if install_packages_result.returncode != 0:
-        get_console().print(
-            f"[error]Error when installing packages from : {K8S_REQUIREMENTS_PATH.resolve()}[/]\n"
-        )
         if not get_verbose():
             get_console().print(install_packages_result.stdout)
             get_console().print(install_packages_result.stderr)
+        install_packages_result = run_command(
+            install_command_no_constraints, check=False, capture_output=capture_output, text=True, env=env
+        )
+        if install_packages_result.returncode != 0:
+            get_console().print(
+                f"[error]Error when installing packages from : {K8S_REQUIREMENTS_PATH.resolve()}[/]\n"
+            )
+            if not get_verbose():
+                get_console().print(install_packages_result.stdout)
+                get_console().print(install_packages_result.stderr)
     return install_packages_result
 
 

scripts/ci/kubernetes/k8s_requirements.txt

@@ -1 +1 @@
--e .[devel-devscripts,devel-tests,cncf.kubernetes] --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-3.8.txt"
+-e .[devel-devscripts,devel-tests,cncf.kubernetes]