From 1a59efc33d67092e06579e1eeea84b2a116104e6 Mon Sep 17 00:00:00 2001 From: Filips Nastins Date: Sat, 29 Jul 2023 09:52:02 +0200 Subject: [PATCH 1/5] support Docker BuildKit --- .../containers/common.py | 37 ++++++++++-- .../pytest/tomodachi_fixtures.py | 4 +- tests/containers/test_common.py | 56 +++++++++++++++++++ 3 files changed, 89 insertions(+), 8 deletions(-) create mode 100644 tests/containers/test_common.py diff --git a/src/tomodachi_testcontainers/containers/common.py b/src/tomodachi_testcontainers/containers/common.py index 6cf612e6..c87cd2d5 100644 --- a/src/tomodachi_testcontainers/containers/common.py +++ b/src/tomodachi_testcontainers/containers/common.py @@ -1,5 +1,6 @@ import os -import pathlib +import subprocess # nosec: B404 +from pathlib import Path from typing import Any, Dict, Iterator, Optional, Tuple, cast import testcontainers.core.container @@ -36,7 +37,7 @@ def get_container_gateway_ip(self) -> str: class EphemeralDockerImage: - def __init__(self, dockerfile: pathlib.Path, docker_client_kw: Optional[Dict] = None) -> None: + def __init__(self, dockerfile: Path, docker_client_kw: Optional[Dict] = None) -> None: self.client = DockerClient(**(docker_client_kw or {})) self.image = self.build_image(dockerfile) @@ -46,7 +47,7 @@ def __enter__(self) -> DockerImage: def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: self.remove_image() - def build_image(self, path: pathlib.Path) -> DockerImage: + def build_image(self, path: Path) -> DockerImage: if path.is_dir(): dockerfile_dir = path dockerfile_name = "" @@ -54,6 +55,33 @@ def build_image(self, path: pathlib.Path) -> DockerImage: dockerfile_dir = path.parent dockerfile_name = path.name + if os.getenv("DOCKER_BUILDKIT"): + return self._build_with_docker_buildkit(dockerfile_dir, dockerfile_name) + + return self._build_with_docker_client(dockerfile_dir, dockerfile_name) + + def remove_image(self) -> None: + self.client.client.images.remove(image=str(self.image.id)) + + def _build_with_docker_buildkit(self, dockerfile_dir: Path, dockerfile_name: str) -> DockerImage: + filepath = dockerfile_dir / dockerfile_name + + cmd = ["docker", "build", "-q", "--rm=true"] + if filepath.is_file(): + cmd += ["-f", str(filepath)] + cmd += [str(dockerfile_dir)] + + result = subprocess.run( # nosec: B603 + cmd, + shell=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + ) + image_id = result.stdout.decode("utf-8").strip() + return cast(DockerImage, self.client.client.images.get(image_id)) + + def _build_with_docker_client(self, dockerfile_dir: Path, dockerfile_name: str) -> DockerImage: image, _ = cast( Tuple[DockerImage, Iterator], self.client.client.images.build( @@ -64,9 +92,6 @@ def build_image(self, path: pathlib.Path) -> DockerImage: ) return image - def remove_image(self) -> None: - self.client.client.images.remove(image=str(self.image.id)) - def get_docker_image(image_id: str, docker_client_kw: Optional[Dict] = None) -> Optional[DockerImage]: client = DockerClient(**(docker_client_kw or {})) diff --git a/src/tomodachi_testcontainers/pytest/tomodachi_fixtures.py b/src/tomodachi_testcontainers/pytest/tomodachi_fixtures.py index 14149b09..357883f8 100644 --- a/src/tomodachi_testcontainers/pytest/tomodachi_fixtures.py +++ b/src/tomodachi_testcontainers/pytest/tomodachi_fixtures.py @@ -1,5 +1,5 @@ import os -import pathlib +from pathlib import Path from typing import Generator import pytest @@ -13,6 +13,6 @@ def tomodachi_image() -> Generator[DockerImage, None, None]: if image := get_docker_image(os.environ.get("TOMODACHI_TESTCONTAINER_IMAGE_ID", "")): yield image else: - dockerfile = pathlib.Path(os.environ.get("TOMODACHI_TESTCONTAINER_DOCKERFILE_PATH", ".")) + dockerfile = Path(os.environ.get("TOMODACHI_TESTCONTAINER_DOCKERFILE_PATH", ".")) with EphemeralDockerImage(dockerfile) as image: yield image diff --git a/tests/containers/test_common.py b/tests/containers/test_common.py new file mode 100644 index 00000000..c586a45b --- /dev/null +++ b/tests/containers/test_common.py @@ -0,0 +1,56 @@ +import tempfile +from pathlib import Path +from typing import Generator + +import docker +import pytest +from docker.errors import BuildError, ImageNotFound + +from tomodachi_testcontainers.containers.common import EphemeralDockerImage + + +class TestEphemeralDockerImage: + @pytest.fixture() + def dockerfile_hello_world(self, tmp_path: Path) -> Generator[Path, None, None]: + with tempfile.NamedTemporaryFile(mode="wt", encoding="utf-8", dir=tmp_path) as dockerfile: + dockerfile.writelines( + [ + "FROM alpine:latest\n", + "RUN echo 'Hello, world!'\n", + ] + ) + dockerfile.flush() + yield Path(dockerfile.name) + + @pytest.fixture() + def dockerfile_buildkit(self, tmp_path: Path) -> Generator[Path, None, None]: + with tempfile.NamedTemporaryFile(mode="wt", encoding="utf-8", dir=tmp_path) as dockerfile: + dockerfile.writelines( + [ + "FROM alpine:latest\n", + # -- mount is a buildkit feature + "RUN --mount=type=secret,id=test,target=test echo 'Hello, World!'\n", + ] + ) + dockerfile.flush() + yield Path(dockerfile.name) + + def test_build_docker_image_and_remove_on_cleanup(self, dockerfile_hello_world: Path) -> None: + client = docker.from_env() + + with EphemeralDockerImage(dockerfile_hello_world) as image: + assert client.images.get(image.id) + + with pytest.raises(ImageNotFound): + client.images.get(image.id) + + def test_build_with_docker_buildkit(self, dockerfile_buildkit: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("DOCKER_BUILDKIT", "1") + client = docker.from_env() + + with EphemeralDockerImage(dockerfile_buildkit) as image: + assert client.images.get(image.id) + + def test_build_error_when_docker_buildkit_envvar_not_set(self, dockerfile_buildkit: Path) -> None: + with pytest.raises(BuildError), EphemeralDockerImage(dockerfile_buildkit): + pass From 7ca88267ab14bee670dd13b9351d0f24761ebc93 Mon Sep 17 00:00:00 2001 From: Filips Nastins Date: Sat, 29 Jul 2023 10:06:50 +0200 Subject: [PATCH 2/5] refactor get_docker_image --- src/tomodachi_testcontainers/containers/common.py | 8 +++----- .../pytest/tomodachi_fixtures.py | 4 ++-- tests/containers/test_common.py | 12 ++++-------- tests/fixtures/test_tomodachi_fixures.py | 4 ++-- 4 files changed, 11 insertions(+), 17 deletions(-) diff --git a/src/tomodachi_testcontainers/containers/common.py b/src/tomodachi_testcontainers/containers/common.py index c87cd2d5..ccb284ca 100644 --- a/src/tomodachi_testcontainers/containers/common.py +++ b/src/tomodachi_testcontainers/containers/common.py @@ -68,8 +68,8 @@ def _build_with_docker_buildkit(self, dockerfile_dir: Path, dockerfile_name: str cmd = ["docker", "build", "-q", "--rm=true"] if filepath.is_file(): - cmd += ["-f", str(filepath)] - cmd += [str(dockerfile_dir)] + cmd.extend(["-f", str(filepath)]) + cmd.append(str(dockerfile_dir)) result = subprocess.run( # nosec: B603 cmd, @@ -93,10 +93,8 @@ def _build_with_docker_client(self, dockerfile_dir: Path, dockerfile_name: str) return image -def get_docker_image(image_id: str, docker_client_kw: Optional[Dict] = None) -> Optional[DockerImage]: +def get_docker_image(image_id: str, docker_client_kw: Optional[Dict] = None) -> DockerImage: client = DockerClient(**(docker_client_kw or {})) - if not image_id: - return None try: return cast(DockerImage, client.client.images.get(image_id)) except ImageNotFound: diff --git a/src/tomodachi_testcontainers/pytest/tomodachi_fixtures.py b/src/tomodachi_testcontainers/pytest/tomodachi_fixtures.py index 357883f8..750f7462 100644 --- a/src/tomodachi_testcontainers/pytest/tomodachi_fixtures.py +++ b/src/tomodachi_testcontainers/pytest/tomodachi_fixtures.py @@ -10,8 +10,8 @@ @pytest.fixture(scope="session") def tomodachi_image() -> Generator[DockerImage, None, None]: - if image := get_docker_image(os.environ.get("TOMODACHI_TESTCONTAINER_IMAGE_ID", "")): - yield image + if image_id := os.environ.get("TOMODACHI_TESTCONTAINER_IMAGE_ID"): + yield get_docker_image(image_id) else: dockerfile = Path(os.environ.get("TOMODACHI_TESTCONTAINER_DOCKERFILE_PATH", ".")) with EphemeralDockerImage(dockerfile) as image: diff --git a/tests/containers/test_common.py b/tests/containers/test_common.py index c586a45b..17aa2ea0 100644 --- a/tests/containers/test_common.py +++ b/tests/containers/test_common.py @@ -2,11 +2,10 @@ from pathlib import Path from typing import Generator -import docker import pytest from docker.errors import BuildError, ImageNotFound -from tomodachi_testcontainers.containers.common import EphemeralDockerImage +from tomodachi_testcontainers.containers.common import EphemeralDockerImage, get_docker_image class TestEphemeralDockerImage: @@ -36,20 +35,17 @@ def dockerfile_buildkit(self, tmp_path: Path) -> Generator[Path, None, None]: yield Path(dockerfile.name) def test_build_docker_image_and_remove_on_cleanup(self, dockerfile_hello_world: Path) -> None: - client = docker.from_env() - with EphemeralDockerImage(dockerfile_hello_world) as image: - assert client.images.get(image.id) + assert get_docker_image(image_id=str(image.id)) with pytest.raises(ImageNotFound): - client.images.get(image.id) + get_docker_image(image_id=str(image.id)) def test_build_with_docker_buildkit(self, dockerfile_buildkit: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("DOCKER_BUILDKIT", "1") - client = docker.from_env() with EphemeralDockerImage(dockerfile_buildkit) as image: - assert client.images.get(image.id) + assert get_docker_image(image_id=str(image.id)) def test_build_error_when_docker_buildkit_envvar_not_set(self, dockerfile_buildkit: Path) -> None: with pytest.raises(BuildError), EphemeralDockerImage(dockerfile_buildkit): diff --git a/tests/fixtures/test_tomodachi_fixures.py b/tests/fixtures/test_tomodachi_fixures.py index ca24c4fa..e2a7314b 100644 --- a/tests/fixtures/test_tomodachi_fixures.py +++ b/tests/fixtures/test_tomodachi_fixures.py @@ -19,11 +19,11 @@ def test_tomodachi_image_id_set_from_envvar(pytester: pytest.Pytester) -> None: dedent( """\ from docker.models.images import Image as DockerImage - from testcontainers.core.docker_client import DockerClient + from tomodachi_testcontainers.containers.common import get_docker_image def test_tomodachi_image_id_set_from_envvar(tomodachi_image: DockerImage) -> None: - image = DockerClient().client.images.get("alpine:3.18.2") + image = get_docker_image("alpine:3.18.2") assert tomodachi_image.id == image.id """ From b340977d7edff79964786f22ddf6913e9f885e14 Mon Sep 17 00:00:00 2001 From: Filips Nastins Date: Sat, 29 Jul 2023 10:08:14 +0200 Subject: [PATCH 3/5] lint --- tests/fixtures/test_tomodachi_fixures.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/fixtures/test_tomodachi_fixures.py b/tests/fixtures/test_tomodachi_fixures.py index e2a7314b..0f6eb618 100644 --- a/tests/fixtures/test_tomodachi_fixures.py +++ b/tests/fixtures/test_tomodachi_fixures.py @@ -19,6 +19,7 @@ def test_tomodachi_image_id_set_from_envvar(pytester: pytest.Pytester) -> None: dedent( """\ from docker.models.images import Image as DockerImage + from tomodachi_testcontainers.containers.common import get_docker_image From db0e15e92c7740b557072ad0436c37d6d8f020e7 Mon Sep 17 00:00:00 2001 From: Filips Nastins Date: Sat, 29 Jul 2023 10:10:19 +0200 Subject: [PATCH 4/5] add documentation about DOCKER_BUILDKIT envvar --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 808b5c51..c45a09db 100644 --- a/README.md +++ b/README.md @@ -487,6 +487,7 @@ by setting it in the shell before running `pytest`. | `TESTCONTAINER_DOCKER_NETWORK` | Launch testcontainers in specified Docker network. Defaults to 'bridge'. Network must be created beforehand | | `TOMODACHI_TESTCONTAINER_DOCKERFILE_PATH` | Override path to Dockerfile for building Tomodachi service image. Defaults to '.' | | `_TESTCONTAINER_IMAGE_ID` | Override any supported Testcontainer Image ID. Defaults to `None`, `TOMODACHI_TESTCONTAINER_DOCKERFILE_PATH` takes precedence | +| `DOCKER_BUILDKIT` | Set `DOCKER_BUILDKIT=1` to use Docker BuildKit in `EphemeralDockerImage` | ## Changing default Docker network From 94c98a51440672fcae8b55acb234ed37b62d2b32 Mon Sep 17 00:00:00 2001 From: Filips Nastins Date: Sat, 29 Jul 2023 10:41:12 +0200 Subject: [PATCH 5/5] use BuildKit in tests --- pyproject.toml | 1 + tests/containers/test_common.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e4960c00..feb04ae4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -168,4 +168,5 @@ log_cli_level = "INFO" env = [ "TESTCONTAINER_DOCKER_NETWORK=tomodachi-testcontainers", "TOMODACHI_TESTCONTAINER_DOCKERFILE_PATH=examples/Dockerfile", + "DOCKER_BUILDKIT=1", ] diff --git a/tests/containers/test_common.py b/tests/containers/test_common.py index 17aa2ea0..9541c95d 100644 --- a/tests/containers/test_common.py +++ b/tests/containers/test_common.py @@ -34,7 +34,11 @@ def dockerfile_buildkit(self, tmp_path: Path) -> Generator[Path, None, None]: dockerfile.flush() yield Path(dockerfile.name) - def test_build_docker_image_and_remove_on_cleanup(self, dockerfile_hello_world: Path) -> None: + def test_build_docker_image_and_remove_on_cleanup( + self, dockerfile_hello_world: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("DOCKER_BUILDKIT", raising=False) + with EphemeralDockerImage(dockerfile_hello_world) as image: assert get_docker_image(image_id=str(image.id)) @@ -47,6 +51,10 @@ def test_build_with_docker_buildkit(self, dockerfile_buildkit: Path, monkeypatch with EphemeralDockerImage(dockerfile_buildkit) as image: assert get_docker_image(image_id=str(image.id)) - def test_build_error_when_docker_buildkit_envvar_not_set(self, dockerfile_buildkit: Path) -> None: + def test_build_error_when_docker_buildkit_envvar_not_set( + self, dockerfile_buildkit: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("DOCKER_BUILDKIT", raising=False) + with pytest.raises(BuildError), EphemeralDockerImage(dockerfile_buildkit): pass