From 69b2ac30146d56598bc42b22a8d015e327c2f234 Mon Sep 17 00:00:00 2001 From: Einar Wigum Arbo Date: Thu, 15 Aug 2024 11:25:09 +0200 Subject: [PATCH] Allow for multiple env-files (#616) --- python_on_whales/client_config.py | 14 +- python_on_whales/docker_client.py | 2 + .../components/dummy_compose.yml | 16 +- tests/python_on_whales/test_client_config.py | 143 +++++++++++++++++- 4 files changed, 167 insertions(+), 8 deletions(-) diff --git a/python_on_whales/client_config.py b/python_on_whales/client_config.py index 6709f4ab..d6663e8a 100644 --- a/python_on_whales/client_config.py +++ b/python_on_whales/client_config.py @@ -74,6 +74,7 @@ class ClientConfig: compose_files: List[ValidPath] = field(default_factory=list) compose_profiles: List[str] = field(default_factory=list) compose_env_file: Optional[ValidPath] = None + compose_env_files: Iterable[ValidPath] = field(default_factory=list) compose_project_name: Optional[str] = None compose_project_directory: Optional[ValidPath] = None compose_compatibility: Optional[bool] = None @@ -159,7 +160,18 @@ def docker_compose_cmd(self) -> Command: base_cmd = self.docker_cmd + ["compose"] base_cmd.add_args_iterable_or_single("--file", self.compose_files) base_cmd.add_args_iterable_or_single("--profile", self.compose_profiles) - base_cmd.add_simple_arg("--env-file", self.compose_env_file) + if self.compose_env_files: + if self.compose_env_file: + warnings.warn( + "You can't set both `compose_env_file` and `compose_env_files`. Files used in `compose_env_files` will be used." + ) + base_cmd.add_args_iterable("--env-file", self.compose_env_files) + elif self.compose_env_file: + warnings.warn( + "`compose_env_file` is deprecated. Use `compose_env_files` instead." + ) + base_cmd.add_simple_arg("--env-file", self.compose_env_file) + base_cmd.add_simple_arg("--project-name", self.compose_project_name) base_cmd.add_simple_arg("--project-directory", self.compose_project_directory) base_cmd.add_flag("--compatibility", self.compose_compatibility) diff --git a/python_on_whales/docker_client.py b/python_on_whales/docker_client.py index 21dbe7b1..dfc2c3c9 100644 --- a/python_on_whales/docker_client.py +++ b/python_on_whales/docker_client.py @@ -180,6 +180,7 @@ def __init__( compose_files: List[ValidPath] = [], compose_profiles: List[str] = [], compose_env_file: Optional[ValidPath] = None, + compose_env_files: List[ValidPath] = [], compose_project_name: Optional[str] = None, compose_project_directory: Optional[ValidPath] = None, compose_compatibility: Optional[bool] = None, @@ -209,6 +210,7 @@ def __init__( compose_files=compose_files, compose_profiles=compose_profiles, compose_env_file=compose_env_file, + compose_env_files=compose_env_files, compose_project_name=compose_project_name, compose_project_directory=compose_project_directory, compose_compatibility=compose_compatibility, diff --git a/tests/python_on_whales/components/dummy_compose.yml b/tests/python_on_whales/components/dummy_compose.yml index 374b7e25..5ab12886 100644 --- a/tests/python_on_whales/components/dummy_compose.yml +++ b/tests/python_on_whales/components/dummy_compose.yml @@ -6,12 +6,12 @@ services: context: my_service_build image: some_random_image ports: - - "5000:5000" + - "5000:5000" volumes: - - /tmp:/tmp + - /tmp:/tmp environment: - - DATADOG_HOST=something + - DATADOG_HOST=something busybox: image: busybox:latest command: sleep infinity @@ -24,8 +24,8 @@ services: image: alpine:latest command: sleep infinity environment: - - DD_API_KEY=__your_datadog_api_key_here__ - - POSTGRES_HOST_AUTH_METHOD=trust + - DD_API_KEY=__your_datadog_api_key_here__ + - POSTGRES_HOST_AUTH_METHOD=trust dodo: image: busybox:latest entrypoint: /bin/sh @@ -39,3 +39,9 @@ services: command: sleep infinity profiles: - my_test_profile2 + service_using_env_variables: + image: busybox:latest + command: sleep infinity + environment: + - A=${A:-0} + - B=${B:-0} diff --git a/tests/python_on_whales/test_client_config.py b/tests/python_on_whales/test_client_config.py index 63128a1d..78e6b2e9 100644 --- a/tests/python_on_whales/test_client_config.py +++ b/tests/python_on_whales/test_client_config.py @@ -1,10 +1,13 @@ import json +from contextlib import contextmanager from pathlib import Path +from typing import Iterator, Sequence, Tuple import pytest -from python_on_whales import docker -from python_on_whales.client_config import ParsingError +from python_on_whales import DockerClient, docker +from python_on_whales.client_config import ClientConfig, ParsingError +from python_on_whales.utils import PROJECT_ROOT fake_json_message = { "CreatedAt": "2020-10-08T18:32:55Z", @@ -36,3 +39,139 @@ def test_pretty_exception_message_and_report(mocker): raise IndexError assert Path(word).read_text() == json.dumps(fake_json_message, indent=2) + + +def test_compose_env_file(): + """Test that the deprecated `compose_env_file` gives a warning, and adds the `--env-file` argument to the compose command""" + with pytest.warns(UserWarning): + example_env_file_name = "example.env" + client_config = ClientConfig(compose_env_file=example_env_file_name) + assert client_config.docker_compose_cmd.count("--env-file") == 1 + # Since we are operating of a list of commands, we first find the index of the `--env-file` argument, then we check that the next argument is the file name. + index = client_config.docker_compose_cmd.index("--env-file") + assert client_config.docker_compose_cmd[index + 1] == example_env_file_name + + +@pytest.mark.parametrize( + "compose_env_files", + [ + ["example1.env", "example2.env"], + ["example1.env"], + ], + ids=["multiple_files", "single_file"], +) +def test_compose_env_files(compose_env_files: Sequence[str]): + """Test that using only `compose_env_files` adds all the files to the command line arguments""" + client_config = ClientConfig(compose_env_files=compose_env_files) + # Since we are operating of a list of commands, we first find the indices of the `--env-file` argument, then we check that the next argument is the file name. + indices = [ + index + for index, item in enumerate(client_config.docker_compose_cmd) + if item == "--env-file" + ] + assert len(indices) == len(compose_env_files) + for index, env_file in zip(indices, compose_env_files): + assert client_config.docker_compose_cmd[index + 1] == env_file + + +def test_compose_env_files_and_env_file(): + """Test that using both `compose_env_files` and `compose_env_file` gives a warning, and only uses `compose_env_files`""" + env_files_input = "example1.env" + env_file_input = "example2.env" + + with pytest.warns(UserWarning): + client_config = ClientConfig( + compose_env_files=[env_files_input], compose_env_file=env_file_input + ) + assert client_config.docker_compose_cmd.count("--env-file") == len( + [env_files_input] + ) + # Since we are operating of a list of commands, we first find the index of the `--env-file` argument, then we check that the next argument is the file name. + index = client_config.docker_compose_cmd.index("--env-file") + assert client_config.docker_compose_cmd[index + 1] == env_files_input + + +@contextmanager +def _create_temp_env_files() -> Iterator[Tuple[Path, Path]]: + """Creates two temporary env files and yields their paths. The files are deleted after the context manager is exited.""" + path = PROJECT_ROOT / "tests/python_on_whales/components" + env_file_1 = path / "example1.env" + with env_file_1.open("w") as f: + f.write("A=1\n") + f.write("B=1\n") + env_file_2 = path / "example2.env" + with env_file_2.open("w") as f: + f.write("B=2\n") + yield env_file_1, env_file_2 + env_file_1.unlink() + env_file_2.unlink() + + +def test_run_compose_command_with_env_file(): + """Test that checks the configuration for the compose stack and checks that the env files are loaded correctly + + The docker compose service, `service_using_env_variables`, is defined in `dummy_compose.yml` and uses the env variables `A` and `B`. + If no env files are provided, the values of `A` and `B` are set to a default value of `0` in the service. + We check if the values of `A` and `B` are set to the values in the env files. + + The cases tested are: + * Single env file using the old `compose_env_file` argument + * Single env file using the new `compose_env_files` argument + * Multiple env files using the new `compose_env_files` argument + * Later env files should override the values of earlier env files + * The order of the env files should matter + """ + with _create_temp_env_files() as (env_file_1, env_file_2): + COMPOSE_FILE = ( + PROJECT_ROOT / "tests/python_on_whales/components/dummy_compose.yml" + ) + SERVICE = "service_using_env_variables" + + # Test with no env-file to check the default values, should be `0`, as is default in the service definition. + client = DockerClient(compose_files=[COMPOSE_FILE]) + config = client.compose.config() + environment = config.services[SERVICE].environment + assert "A" in environment and environment["A"] == "0" + assert "B" in environment and environment["B"] == "0" + + # Test with single env-file using the old `compose_env_file` argument + client = DockerClient( + compose_files=[COMPOSE_FILE], + compose_env_file=env_file_1, + ) + config = client.compose.config() + environment = config.services[SERVICE].environment + assert "A" in environment and environment["A"] == "1" + assert "B" in environment and environment["B"] == "1" + + # Test with single env-file using the new `compose_env_files` argument + client = DockerClient( + compose_files=[COMPOSE_FILE], + compose_env_files=[env_file_1], + ) + config = client.compose.config() + environment = config.services[SERVICE].environment + assert "A" in environment and environment["A"] == "1" + assert "B" in environment and environment["B"] == "1" + + # Test with multiple env-files using the new `compose_env_files` argument. + # Since `env_file2` is loaded after `env_file1`, the value of `B` should be overridden by `env_file_2`. + client = DockerClient( + compose_files=[COMPOSE_FILE], + compose_env_files=[env_file_1, env_file_2], + ) + config = client.compose.config() + environment = config.services[SERVICE].environment + assert "A" in environment and environment["A"] == "1" + assert "B" in environment and environment["B"] == "2" + + # Test with multiple env-files using the new `compose_env_files` argument. + # Since `env_file1` is loaded after `env_file2`, the value of `B` should be overridden by `env_file_1`. + client = DockerClient( + compose_files=[COMPOSE_FILE], + compose_env_files=[env_file_2, env_file_1], + ) + config = client.compose.config() + environment = config.services[SERVICE].environment + assert "A" in environment and environment["A"] == "1" + assert "B" in environment and environment["B"] == "1"