From a1257337c88e9b698e152692d25c260af4a4c230 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Mon, 21 Aug 2023 07:53:08 +0200 Subject: [PATCH] Bump min-version for docker and docker-compose It's been quite some time since we bumped min versions for docker and docker-compose for Breeze and there are already a few cases where it held us back from more efficiently using some newer docker or docker-compose features (for example in #33547). Also docker-compose v1 is sufficiently old an unmaintained, that it's about time to get rid of it and ask the users to migrate to docker-compose v2. The new min versions propose: * Docker: 23.0.0 released 2023-02-01 (6 months ago) * Docker Compose 2.14.0 release 2022-12-15 (8 months ago) Docker v1 is not supported any more and the users are directed to migrate to v2 to continue using Breeze. --- .../commands/developer_commands.py | 7 +-- .../commands/testing_commands.py | 12 ++-- .../src/airflow_breeze/global_constants.py | 4 +- .../utils/docker_command_utils.py | 51 ++++++++-------- dev/breeze/tests/test_docker_command_utils.py | 58 ++++++++++--------- 5 files changed, 67 insertions(+), 65 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py index 59671a273f367..40fc0954780e9 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py @@ -77,7 +77,6 @@ from airflow_breeze.utils.console import get_console from airflow_breeze.utils.custom_param_types import BetterChoice, NotVerifiedBetterChoice from airflow_breeze.utils.docker_command_utils import ( - DOCKER_COMPOSE_COMMAND, check_docker_resources, get_env_variables_for_docker_commands, get_extra_docker_flags, @@ -652,7 +651,7 @@ def compile_www_assets(dev: bool): @option_dry_run def down(preserve_volumes: bool, cleanup_mypy_cache: bool): perform_environment_checks() - command_to_execute = [*DOCKER_COMPOSE_COMMAND, "down", "--remove-orphans"] + command_to_execute = ["docker", "compose", "down", "--remove-orphans"] if not preserve_volumes: command_to_execute.append("--volumes") shell_params = ShellParams(backend="all", include_mypy_volume=True) @@ -723,7 +722,7 @@ def enter_shell(**kwargs) -> RunCommandResult: if shell_params.include_mypy_volume: create_mypy_volume_if_needed() shell_params.print_badge_info() - cmd = [*DOCKER_COMPOSE_COMMAND, "run", "--service-ports", "-e", "BREEZE", "--rm", "airflow"] + cmd = ["docker", "compose", "run", "--service-ports", "-e", "BREEZE", "--rm", "airflow"] cmd_added = shell_params.command_passed env_variables = get_env_variables_for_docker_commands(shell_params) if cmd_added is not None: @@ -763,7 +762,7 @@ def find_airflow_container() -> str | None: check_docker_resources(exec_shell_params.airflow_image_name) exec_shell_params.print_badge_info() env_variables = get_env_variables_for_docker_commands(exec_shell_params) - cmd = [*DOCKER_COMPOSE_COMMAND, "ps", "--all", "--filter", "status=running", "airflow"] + cmd = ["docker", "compose", "ps", "--all", "--filter", "status=running", "airflow"] docker_compose_ps_command = run_command( cmd, text=True, capture_output=True, env=env_variables, check=False ) diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py b/dev/breeze/src/airflow_breeze/commands/testing_commands.py index 68e3c688c63a3..c7c018a0da6f6 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py @@ -58,7 +58,6 @@ from airflow_breeze.utils.console import Output, get_console from airflow_breeze.utils.custom_param_types import BetterChoice, NotVerifiedBetterChoice from airflow_breeze.utils.docker_command_utils import ( - DOCKER_COMPOSE_COMMAND, get_env_variables_for_docker_commands, perform_environment_checks, remove_docker_networks, @@ -170,7 +169,8 @@ def _run_test( # This is needed for Docker-compose 1 compatibility env_variables["COMPOSE_PROJECT_NAME"] = compose_project_name down_cmd = [ - *DOCKER_COMPOSE_COMMAND, + "docker", + "compose", "--project-name", compose_project_name, "down", @@ -178,7 +178,8 @@ def _run_test( ] run_command(down_cmd, env=env_variables, output=output, check=False) run_cmd = [ - *DOCKER_COMPOSE_COMMAND, + "docker", + "compose", "--project-name", compose_project_name, "run", @@ -226,7 +227,8 @@ def _run_test( if not skip_docker_compose_down: run_command( [ - *DOCKER_COMPOSE_COMMAND, + "docker", + "compose", "--project-name", compose_project_name, "rm", @@ -587,6 +589,6 @@ def helm_tests( env_variables["HELM_TEST_PACKAGE"] = helm_test_package perform_environment_checks() cleanup_python_generated_files() - cmd = [*DOCKER_COMPOSE_COMMAND, "run", "--service-ports", "--rm", "airflow", *extra_pytest_args] + cmd = ["docker", "compose", "run", "--service-ports", "--rm", "airflow", *extra_pytest_args] result = run_command(cmd, env=env_variables, check=False, output_outside_the_group=True) sys.exit(result.returncode) diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index ceafd6c43f5cc..c0740801d59c0 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -350,8 +350,8 @@ def get_airflow_extras(): ISSUE_ID = "" NUM_RUNS = "" -MIN_DOCKER_VERSION = "20.10.0" -MIN_DOCKER_COMPOSE_VERSION = "1.29.0" +MIN_DOCKER_VERSION = "23.0.0" +MIN_DOCKER_COMPOSE_VERSION = "2.14.0" AIRFLOW_SOURCES_FROM = "." AIRFLOW_SOURCES_TO = "/opt/airflow" diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index 5332b149edd0c..d0a12e965abcf 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -250,6 +250,7 @@ def check_docker_version(): [warning]install docker at least: {MIN_DOCKER_VERSION} version.[/] """ ) + sys.exit(1) else: good_version = compare_version(docker_version, MIN_DOCKER_VERSION) if good_version: @@ -257,10 +258,12 @@ def check_docker_version(): else: get_console().print( f""" -[warning]Your version of docker is too old:{docker_version}. -Please upgrade to at least {MIN_DOCKER_VERSION}[/] +[error]Your version of docker is too old: {docker_version}.\n[/] +[warning]Please upgrade to at least {MIN_DOCKER_VERSION}.\n[/] +You can find installation instructions here: https://docs.docker.com/engine/install/ """ ) + sys.exit(1) def check_remote_ghcr_io_commands(): @@ -306,9 +309,6 @@ def check_remote_ghcr_io_commands(): sys.exit(1) -DOCKER_COMPOSE_COMMAND = ["docker", "compose"] - - def check_docker_compose_version(): """Checks if the docker compose version is as expected. @@ -328,43 +328,40 @@ def check_docker_compose_version(): dry_run_override=False, ) except Exception: - docker_compose_version_command = ["docker-compose", "--version"] - docker_compose_version_result = run_command( - docker_compose_version_command, - no_output_dump_on_exception=True, - capture_output=True, - text=True, - dry_run_override=False, + get_console().print( + "[error]You either do not have docker-composer or have docker-compose v1 installed.[/]\n" + "[warning]Breeze does not support docker-compose v1 any more as it has been replaced by v2.[/]\n" + "Follow https://docs.docker.com/compose/migrate/ to migrate to v2" ) - DOCKER_COMPOSE_COMMAND.clear() - DOCKER_COMPOSE_COMMAND.append("docker-compose") + sys.exit(1) if docker_compose_version_result.returncode == 0: docker_compose_version = docker_compose_version_result.stdout version_extracted = version_pattern.search(docker_compose_version) if version_extracted is not None: - docker_version = ".".join(version_extracted.groups()) - good_version = compare_version(docker_version, MIN_DOCKER_COMPOSE_VERSION) + docker_compose_version = ".".join(version_extracted.groups()) + good_version = compare_version(docker_compose_version, MIN_DOCKER_COMPOSE_VERSION) if good_version: - get_console().print(f"[success]Good version of docker-compose: {docker_version}[/]") + get_console().print(f"[success]Good version of docker-compose: {docker_compose_version}[/]") else: get_console().print( f""" -[warning]You have too old version of docker-compose: {docker_version}! At least 1.29 needed! Please upgrade! -""" - ) - get_console().print( - """ -See https://docs.docker.com/compose/install/ for instructions. -Make sure docker-compose you install is first on the PATH variable of yours. +[error]You have too old version of docker-compose: {docker_compose_version}!\n[/] +[warning]At least {MIN_DOCKER_COMPOSE_VERSION} needed! Please upgrade!\n[/] +See https://docs.docker.com/compose/install/ for installation instructions.\n +Make sure docker-compose you install is first on the PATH variable of yours.\n """ ) + sys.exit(1) else: get_console().print( - """ -[warning]Unknown docker-compose version. At least 1.29 is needed![/] -[warning]If Breeze fails upgrade to latest available docker-compose version.[/] + f""" +[error]Unknown docker-compose version.[/] +[warning]At least {MIN_DOCKER_COMPOSE_VERSION} needed! Please upgrade!\n[/] +See https://docs.docker.com/compose/install/ for installation instructions.\n +Make sure docker-compose you install is first on the PATH variable of yours.\n """ ) + sys.exit(1) def get_env_variable_value(arg_name: str, params: CommonBuildParams | ShellParams): diff --git a/dev/breeze/tests/test_docker_command_utils.py b/dev/breeze/tests/test_docker_command_utils.py index 83081ad770668..4b1513e30fc71 100644 --- a/dev/breeze/tests/test_docker_command_utils.py +++ b/dev/breeze/tests/test_docker_command_utils.py @@ -36,7 +36,9 @@ def test_check_docker_version_unknown( mock_get_console, mock_run_command, mock_check_docker_permission_denied ): mock_check_docker_permission_denied.return_value = False - check_docker_version() + with pytest.raises(SystemExit) as e: + check_docker_version() + assert e.value.code == 1 expected_run_command_calls = [ call( ["docker", "version", "--format", "{{.Client.Version}}"], @@ -51,7 +53,7 @@ def test_check_docker_version_unknown( mock_get_console.return_value.print.assert_called_with( """ [warning]Your version of docker is unknown. If the scripts fail, please make sure to[/] -[warning]install docker at least: 20.10.0 version.[/] +[warning]install docker at least: 23.0.0 version.[/] """ ) @@ -65,7 +67,9 @@ def test_check_docker_version_too_low( mock_check_docker_permission_denied.return_value = False mock_run_command.return_value.returncode = 0 mock_run_command.return_value.stdout = "0.9" - check_docker_version() + with pytest.raises(SystemExit) as e: + check_docker_version() + assert e.value.code == 1 mock_check_docker_permission_denied.assert_called() mock_run_command.assert_called_with( ["docker", "version", "--format", "{{.Client.Version}}"], @@ -77,7 +81,8 @@ def test_check_docker_version_too_low( ) mock_get_console.return_value.print.assert_called_with( """ -[warning]Your version of docker is too old:0.9.\nPlease upgrade to at least 20.10.0[/] +[error]Your version of docker is too old: 0.9.\n[/]\n[warning]Please upgrade to at least 23.0.0.\n[/]\n\ +You can find installation instructions here: https://docs.docker.com/engine/install/ """ ) @@ -88,7 +93,7 @@ def test_check_docker_version_too_low( def test_check_docker_version_ok(mock_get_console, mock_run_command, mock_check_docker_permission_denied): mock_check_docker_permission_denied.return_value = False mock_run_command.return_value.returncode = 0 - mock_run_command.return_value.stdout = "20.10.0" + mock_run_command.return_value.stdout = "23.0.0" check_docker_version() mock_check_docker_permission_denied.assert_called() mock_run_command.assert_called_with( @@ -99,7 +104,7 @@ def test_check_docker_version_ok(mock_get_console, mock_run_command, mock_check_ check=False, dry_run_override=False, ) - mock_get_console.return_value.print.assert_called_with("[success]Good version of Docker: 20.10.0.[/]") + mock_get_console.return_value.print.assert_called_with("[success]Good version of Docker: 23.0.0.[/]") @mock.patch("airflow_breeze.utils.docker_command_utils.check_docker_permission_denied") @@ -108,7 +113,7 @@ def test_check_docker_version_ok(mock_get_console, mock_run_command, mock_check_ def test_check_docker_version_higher(mock_get_console, mock_run_command, mock_check_docker_permission_denied): mock_check_docker_permission_denied.return_value = False mock_run_command.return_value.returncode = 0 - mock_run_command.return_value.stdout = "21.10.0" + mock_run_command.return_value.stdout = "24.0.0" check_docker_version() mock_check_docker_permission_denied.assert_called() mock_run_command.assert_called_with( @@ -119,13 +124,15 @@ def test_check_docker_version_higher(mock_get_console, mock_run_command, mock_ch check=False, dry_run_override=False, ) - mock_get_console.return_value.print.assert_called_with("[success]Good version of Docker: 21.10.0.[/]") + mock_get_console.return_value.print.assert_called_with("[success]Good version of Docker: 24.0.0.[/]") @mock.patch("airflow_breeze.utils.docker_command_utils.run_command") @mock.patch("airflow_breeze.utils.docker_command_utils.get_console") def test_check_docker_compose_version_unknown(mock_get_console, mock_run_command): - check_docker_compose_version() + with pytest.raises(SystemExit) as e: + check_docker_compose_version() + assert e.value.code == 1 expected_run_command_calls = [ call( ["docker", "compose", "version"], @@ -138,8 +145,9 @@ def test_check_docker_compose_version_unknown(mock_get_console, mock_run_command mock_run_command.assert_has_calls(expected_run_command_calls) mock_get_console.return_value.print.assert_called_with( """ -[warning]Unknown docker-compose version. At least 1.29 is needed![/] -[warning]If Breeze fails upgrade to latest available docker-compose version.[/] +[error]Unknown docker-compose version.[/]\n[warning]At least 2.14.0 needed! Please upgrade!\n[/] +See https://docs.docker.com/compose/install/ for installation instructions.\n +Make sure docker-compose you install is first on the PATH variable of yours.\n """ ) @@ -149,7 +157,9 @@ def test_check_docker_compose_version_unknown(mock_get_console, mock_run_command def test_check_docker_compose_version_low(mock_get_console, mock_run_command): mock_run_command.return_value.returncode = 0 mock_run_command.return_value.stdout = "1.28.5" - check_docker_compose_version() + with pytest.raises(SystemExit) as e: + check_docker_compose_version() + assert e.value.code == 1 mock_run_command.assert_called_with( ["docker", "compose", "version"], no_output_dump_on_exception=True, @@ -157,27 +167,21 @@ def test_check_docker_compose_version_low(mock_get_console, mock_run_command): text=True, dry_run_override=False, ) - expected_print_calls = [ - call( - """ -[warning]You have too old version of docker-compose: 1.28.5! At least 1.29 needed! Please upgrade! -""" - ), - call( - """ -See https://docs.docker.com/compose/install/ for instructions. -Make sure docker-compose you install is first on the PATH variable of yours. + mock_get_console.return_value.print.assert_called_with( + """ +[error]You have too old version of docker-compose: 1.28.5!\n[/] +[warning]At least 2.14.0 needed! Please upgrade!\n[/] +See https://docs.docker.com/compose/install/ for installation instructions.\n +Make sure docker-compose you install is first on the PATH variable of yours.\n """ - ), - ] - mock_get_console.return_value.print.assert_has_calls(expected_print_calls) + ) @mock.patch("airflow_breeze.utils.docker_command_utils.run_command") @mock.patch("airflow_breeze.utils.docker_command_utils.get_console") def test_check_docker_compose_version_ok(mock_get_console, mock_run_command): mock_run_command.return_value.returncode = 0 - mock_run_command.return_value.stdout = "1.29.0" + mock_run_command.return_value.stdout = "2.14.0" check_docker_compose_version() mock_run_command.assert_called_with( ["docker", "compose", "version"], @@ -187,7 +191,7 @@ def test_check_docker_compose_version_ok(mock_get_console, mock_run_command): dry_run_override=False, ) mock_get_console.return_value.print.assert_called_with( - "[success]Good version of docker-compose: 1.29.0[/]" + "[success]Good version of docker-compose: 2.14.0[/]" )