From 8d424d3f0d7a6782d7b64f65e42930c08e764e87 Mon Sep 17 00:00:00 2001 From: Steve Murphy Date: Mon, 29 May 2023 05:31:40 +0100 Subject: [PATCH] Nox session to build and push multiplatform images (#3324) Co-authored-by: Thomas Co-authored-by: Thomas --- .github/workflows/backend_checks.yml | 14 +- .github/workflows/publish_docker.yaml | 5 +- clients/sample-app/Dockerfile | 2 +- noxfiles/ci_nox.py | 3 +- noxfiles/dev_nox.py | 7 +- noxfiles/docker_nox.py | 278 ++++++++++++------ noxfiles/docs_nox.py | 1 + noxfiles/git_nox.py | 3 +- .../{test_setup_nox.py => setup_tests_nox.py} | 7 +- noxfiles/test_docker_nox.py | 57 ++++ {tests/nox => noxfiles}/test_git_nox.py | 2 +- noxfiles/utils_nox.py | 1 + tests/nox/__init__.py | 0 13 files changed, 276 insertions(+), 104 deletions(-) rename noxfiles/{test_setup_nox.py => setup_tests_nox.py} (97%) create mode 100644 noxfiles/test_docker_nox.py rename {tests/nox => noxfiles}/test_git_nox.py (98%) delete mode 100644 tests/nox/__init__.py diff --git a/.github/workflows/backend_checks.yml b/.github/workflows/backend_checks.yml index 2f62a33bf7..82a3646924 100644 --- a/.github/workflows/backend_checks.yml +++ b/.github/workflows/backend_checks.yml @@ -75,7 +75,15 @@ jobs: strategy: matrix: session_name: - ["isort", "black", "mypy", "pylint", "xenon", "check_install"] + [ + "isort", + "black", + "mypy", + "pylint", + "xenon", + "check_install", + '"pytest(nox)"', + ] runs-on: ubuntu-latest continue-on-error: true steps: @@ -91,6 +99,9 @@ jobs: - name: Install Nox run: pip install nox>=2022 + - name: Install Dev Requirements + run: pip install -r dev-requirements.txt + - name: Run Static Check run: nox -s ${{ matrix.session_name }} @@ -148,7 +159,6 @@ jobs: - "ops-unit" - "ops-integration" - "lib" - - "nox" runs-on: ubuntu-latest timeout-minutes: 25 diff --git a/.github/workflows/publish_docker.yaml b/.github/workflows/publish_docker.yaml index 111ab08100..d2509e5c20 100644 --- a/.github/workflows/publish_docker.yaml +++ b/.github/workflows/publish_docker.yaml @@ -1,4 +1,4 @@ -name: Docker Build & Push +name: Publish Docker Images on: push: @@ -28,9 +28,6 @@ jobs: - name: Install Dev Requirements run: pip install -r dev-requirements.txt - - name: Build Fides Image - run: nox -s "build(prod)" - - name: Push Fides Dev Tag run: nox -s "push(dev)" diff --git a/clients/sample-app/Dockerfile b/clients/sample-app/Dockerfile index 4a3a13c8d7..52582e87de 100644 --- a/clients/sample-app/Dockerfile +++ b/clients/sample-app/Dockerfile @@ -1,4 +1,4 @@ -FROM node:16-alpine +FROM node:16-alpine as prod ENV DATABASE_HOST localhost ENV DATABASE_PORT 5432 diff --git a/noxfiles/ci_nox.py b/noxfiles/ci_nox.py index 27390ce7a6..2be71fe651 100644 --- a/noxfiles/ci_nox.py +++ b/noxfiles/ci_nox.py @@ -3,6 +3,7 @@ from typing import Callable, Dict import nox + from constants_nox import ( CONTAINER_NAME, IMAGE_NAME, @@ -11,7 +12,7 @@ START_APP, WITH_TEST_CONFIG, ) -from test_setup_nox import pytest_ctl, pytest_lib, pytest_nox, pytest_ops +from setup_tests_nox import pytest_ctl, pytest_lib, pytest_nox, pytest_ops from utils_nox import install_requirements diff --git a/noxfiles/dev_nox.py b/noxfiles/dev_nox.py index 25d50397db..f06efd2811 100644 --- a/noxfiles/dev_nox.py +++ b/noxfiles/dev_nox.py @@ -3,6 +3,10 @@ from pathlib import Path from typing import Literal +from nox import Session, param, parametrize +from nox import session as nox_session +from nox.command import CommandFailed + from constants_nox import ( COMPOSE_SERVICE_NAME, EXEC_IT, @@ -11,9 +15,6 @@ START_APP_REMOTE_DEBUG, ) from docker_nox import build -from nox import Session, param, parametrize -from nox import session as nox_session -from nox.command import CommandFailed from run_infrastructure import ALL_DATASTORES, run_infrastructure from utils_nox import install_requirements, teardown diff --git a/noxfiles/docker_nox.py b/noxfiles/docker_nox.py index dc87033ca1..64ca738ccf 100644 --- a/noxfiles/docker_nox.py +++ b/noxfiles/docker_nox.py @@ -1,8 +1,9 @@ """Contains the nox sessions for docker-related tasks.""" import platform -from typing import List +from typing import List, Tuple import nox + from constants_nox import ( IMAGE, IMAGE_DEV, @@ -15,6 +16,59 @@ ) from git_nox import get_current_tag, recognized_tag +DOCKER_PLATFORM_MAP = { + "amd64": "linux/amd64", + "arm64": "linux/arm64", + "x86_64": "linux/amd64", +} +DOCKER_PLATFORMS = "linux/amd64,linux/arm64" + + +def verify_git_tag(session: nox.Session) -> str: + """ + Get the git tag for HEAD and validate it before using it. + """ + existing_commit_tag = get_current_tag(existing=True) + if existing_commit_tag is None: + session.error( + "Did not find an existing git tag on the current commit, not pushing git-tag images" + ) + + if not recognized_tag(existing_commit_tag): + session.error( + f"Existing git tag {existing_commit_tag} is not a recognized tag, not pushing git-tag images" + ) + + session.log( + f"Found git tag {existing_commit_tag} on the current commit, pushing corresponding git-tag images!" + ) + return existing_commit_tag + + +def generate_multiplatform_buildx_command( + image_tags: List[str], docker_build_target: str, dockerfile_path: str = "." +) -> Tuple[str, ...]: + """ + Generate the command for building and publishing a multiplatform image. + + See tests for example usage. + """ + buildx_command: Tuple[str, ...] = ( + "docker", + "buildx", + "build", + "--push", + f"--target={docker_build_target}", + "--platform", + DOCKER_PLATFORMS, + dockerfile_path, + ) + + for tag in image_tags: + buildx_command += ("--tag", tag) + + return buildx_command + def get_current_image() -> str: """Returns the current image tag""" @@ -26,17 +80,11 @@ def get_platform(posargs: List[str]) -> str: Calculate the CPU platform or get it from the positional arguments. """ - # Support Intel Macs - docker_platforms = { - "amd64": "linux/amd64", - "arm64": "linux/arm64", - "x86_64": "linux/amd64", - } if "amd64" in posargs: - return docker_platforms["amd64"] + return DOCKER_PLATFORM_MAP["amd64"] if "arm64" in posargs: - return docker_platforms["arm64"] - return docker_platforms[platform.machine().lower()] + return DOCKER_PLATFORM_MAP["arm64"] + return DOCKER_PLATFORM_MAP[platform.machine().lower()] @nox.session() @@ -56,7 +104,7 @@ def build(session: nox.Session, image: str, machine_type: str = "") -> None: Params: admin-ui = Build the Next.js Admin UI application. - dev = Build the fides webserver/CLI, tagged as `local`. + dev = Build the fides webserver/CLI, tagged as `local` and with an editable pip install of Fides. privacy-center = Build the Next.js Privacy Center application. prod = Build the fides webserver/CLI and tag it as the current application version. test = Build the fides webserver/CLI the same as `prod`, but tag it as `local`. @@ -97,6 +145,8 @@ def build(session: nox.Session, image: str, machine_type: str = "") -> None: "docker", "build", "--target=prod_pc", + "--platform", + build_platform, "--tag", privacy_center_image_tag, ".", @@ -108,6 +158,8 @@ def build(session: nox.Session, image: str, machine_type: str = "") -> None: "docker", "build", "clients/sample-app", + "--platform", + build_platform, "--tag", sample_app_image_tag, external=True, @@ -132,6 +184,118 @@ def build(session: nox.Session, image: str, machine_type: str = "") -> None: session.run(*build_command, external=True) +def push_prod(session: nox.Session) -> None: + """ + Contains the logic for pushing the suite of 'prod' images. + + Pushed Image Examples: + - ethyca/fides:2.0.0 + - ethyca/fides:latest + + - ethyca/fides-privacy-center:2.0.0 + - ethyca/fides-privacy-center:latest + + - ethyca/fides-sample-app:2.0.0 + - ethyca/fides-sample-app:latest + """ + fides_image_name = get_current_image() + privacy_center_image_name = f"{PRIVACY_CENTER_IMAGE}:{get_current_tag()}" + sample_app_image_name = f"{SAMPLE_APP_IMAGE}:{get_current_tag()}" + + privacy_center_latest = f"{PRIVACY_CENTER_IMAGE}:latest" + sample_app_latest = f"{SAMPLE_APP_IMAGE}:latest" + + fides_buildx_command = generate_multiplatform_buildx_command( + [fides_image_name, IMAGE_LATEST], docker_build_target="prod" + ) + session.run(*fides_buildx_command, external=True) + + privacy_center_buildx_command = generate_multiplatform_buildx_command( + [privacy_center_image_name, privacy_center_latest], + docker_build_target="prod_pc", + ) + session.run(*privacy_center_buildx_command, external=True) + + sample_app_buildx_command = generate_multiplatform_buildx_command( + [sample_app_image_name, sample_app_latest], + docker_build_target="prod", + dockerfile_path="clients/sample-app", + ) + session.run(*sample_app_buildx_command, external=True) + + +def push_dev(session: nox.Session) -> None: + """ + Push the bleeding-edge `dev` images. + + Pushed Image Examples: + - ethyca/fides:dev + - ethyca/fides-privacy-center:dev + - ethyca/fides-sample-app:dev + """ + privacy_center_dev = f"{PRIVACY_CENTER_IMAGE}:dev" + sample_app_dev = f"{SAMPLE_APP_IMAGE}:dev" + + fides_buildx_command = generate_multiplatform_buildx_command( + [IMAGE_DEV], docker_build_target="prod" + ) + session.run(*fides_buildx_command, external=True) + + privacy_center_buildx_command = generate_multiplatform_buildx_command( + [privacy_center_dev], + docker_build_target="prod_pc", + ) + session.run(*privacy_center_buildx_command, external=True) + + sample_app_buildx_command = generate_multiplatform_buildx_command( + [sample_app_dev], + docker_build_target="prod", + dockerfile_path="clients/sample-app", + ) + session.run(*sample_app_buildx_command, external=True) + + +def push_git_tag(session: nox.Session) -> None: + """ + Push an image with the tag of our current commit. + + If we have an existing git tag on the current commit, we push up + a set of images that's tagged specifically with this git tag. + + This publishes images that correspond to commits that have been explicitly tagged, + e.g. RC builds, `beta` tags on `main`, `alpha` tags for feature branch builds. + + Pushed Image Examples: + - ethyca/fides:{current_head_git_tag} + - ethyca/fides-privacy-center:{current_head_git_tag} + - ethyca/fides-sample-app:{current_head_git_tag} + """ + + existing_commit_tag = verify_git_tag(session) + custom_image_tag = f"{IMAGE}:{existing_commit_tag}" + privacy_center_dev = f"{PRIVACY_CENTER_IMAGE}:{existing_commit_tag}" + sample_app_dev = f"{SAMPLE_APP_IMAGE}:{existing_commit_tag}" + + # Publish + fides_buildx_command = generate_multiplatform_buildx_command( + [custom_image_tag], docker_build_target="prod" + ) + session.run(*fides_buildx_command, external=True) + + privacy_center_buildx_command = generate_multiplatform_buildx_command( + [privacy_center_dev], + docker_build_target="prod_pc", + ) + session.run(*privacy_center_buildx_command, external=True) + + sample_app_buildx_command = generate_multiplatform_buildx_command( + [sample_app_dev], + docker_build_target="prod", + dockerfile_path="clients/sample-app", + ) + session.run(*sample_app_buildx_command, external=True) + + @nox.session() @nox.parametrize( "tag", @@ -154,85 +318,27 @@ def push(session: nox.Session, tag: str) -> None: dev - Tags images with `dev` git-tag - Tags images with the git tag of the current commit, if it exists - NOTE: Expects these to first be built via 'build(prod)' + NOTE: This command also handles building images, including for multiple supported architectures. """ - fides_image_prod = get_current_image() - privacy_center_prod = f"{PRIVACY_CENTER_IMAGE}:{get_current_tag()}" - sample_app_prod = f"{SAMPLE_APP_IMAGE}:{get_current_tag()}" + + # Create the buildx builder + session.run( + "docker", + "buildx", + "create", + "--name", + "fides_builder", + "--bootstrap", + "--use", + external=True, + success_codes=[0, 1], + ) if tag == "dev": - # Push the ethyca/fides image, tagging with :dev - session.run("docker", "tag", fides_image_prod, IMAGE_DEV, external=True) - session.run("docker", "push", IMAGE_DEV, external=True) - - # Push the extra images, tagging with :dev - # - ethyca/fides-privacy-center:dev - # - ethyca/fides-sample-app:dev - privacy_center_dev = f"{PRIVACY_CENTER_IMAGE}:dev" - sample_app_dev = f"{SAMPLE_APP_IMAGE}:dev" - session.run( - "docker", "tag", privacy_center_prod, privacy_center_dev, external=True - ) - session.run("docker", "push", privacy_center_dev, external=True) - session.run("docker", "tag", sample_app_prod, sample_app_dev, external=True) - session.run("docker", "push", sample_app_dev, external=True) + push_dev(session=session) if tag == "git-tag": - # if we have an existing git tag on the current commit, we push up - # a set of images that's tagged specifically with this git tag. - # this publishes images that correspond to commits that have been explicitly tagged, - # e.g. RC builds, `beta` tags on `main`, `alpha` tags for feature branch builds. - existing_commit_tag = get_current_tag(existing=True) - if existing_commit_tag is None: - session.log( - "Did not find an existing git tag on the current commit, not pushing git-tag images" - ) - return - - if not recognized_tag(existing_commit_tag): - session.log( - f"Existing git tag {existing_commit_tag} is not a recognized tag, not pushing git-tag images" - ) - return - - session.log( - f"Found git tag {existing_commit_tag} on the current commit, pushing corresponding git-tag images!" - ) - custom_image_tag = f"{IMAGE}:{existing_commit_tag}" - # Push the ethyca/fides image, tagging with :{current_head_git_tag} - session.run("docker", "tag", fides_image_prod, custom_image_tag, external=True) - session.run("docker", "push", custom_image_tag, external=True) - - # Push the extra images, tagging with :{current_head_git_tag} - # - ethyca/fides-privacy-center:{current_head_git_tag} - # - ethyca/fides-sample-app:{current_head_git_tag} - privacy_center_dev = f"{PRIVACY_CENTER_IMAGE}:{existing_commit_tag}" - sample_app_dev = f"{SAMPLE_APP_IMAGE}:{existing_commit_tag}" - session.run( - "docker", "tag", privacy_center_prod, privacy_center_dev, external=True - ) - session.run("docker", "push", privacy_center_dev, external=True) - session.run("docker", "tag", sample_app_prod, sample_app_dev, external=True) - session.run("docker", "push", sample_app_dev, external=True) + push_git_tag(session=session) if tag == "prod": - # Example: "ethyca/fides:2.0.0" and "ethyca/fides:latest" - session.run("docker", "tag", fides_image_prod, IMAGE_LATEST, external=True) - session.run("docker", "push", IMAGE_LATEST, external=True) - session.run("docker", "push", fides_image_prod, external=True) - - # Example: - # - ethyca/fides-privacy-center:2.0.0 - # - ethyca/fides-privacy-center:latest - # - ethyca/fides-sample-app:2.0.0 - # - ethyca/fides-sample-app:latest - privacy_center_latest = f"{PRIVACY_CENTER_IMAGE}:latest" - sample_app_latest = f"{SAMPLE_APP_IMAGE}:latest" - session.run( - "docker", "tag", privacy_center_prod, privacy_center_latest, external=True - ) - session.run("docker", "push", privacy_center_latest, external=True) - session.run("docker", "push", privacy_center_prod, external=True) - session.run("docker", "tag", sample_app_prod, sample_app_latest, external=True) - session.run("docker", "push", sample_app_latest, external=True) - session.run("docker", "push", sample_app_prod, external=True) + push_prod(session=session) diff --git a/noxfiles/docs_nox.py b/noxfiles/docs_nox.py index 286cabd9b6..e47906c8f5 100644 --- a/noxfiles/docs_nox.py +++ b/noxfiles/docs_nox.py @@ -1,5 +1,6 @@ """Contains the nox sessions for developing docs.""" import nox + from constants_nox import CI_ARGS diff --git a/noxfiles/git_nox.py b/noxfiles/git_nox.py index eeb1dcc9c0..b4a7af3b57 100644 --- a/noxfiles/git_nox.py +++ b/noxfiles/git_nox.py @@ -4,9 +4,8 @@ from enum import Enum from typing import List, Optional -from packaging.version import Version - import nox +from packaging.version import Version RELEASE_BRANCH_REGEX = r"release-(([0-9]+\.)+[0-9]+)" RELEASE_TAG_REGEX = r"(([0-9]+\.)+[0-9]+)" diff --git a/noxfiles/test_setup_nox.py b/noxfiles/setup_tests_nox.py similarity index 97% rename from noxfiles/test_setup_nox.py rename to noxfiles/setup_tests_nox.py index 9d30a70a27..8013eb3d61 100644 --- a/noxfiles/test_setup_nox.py +++ b/noxfiles/setup_tests_nox.py @@ -1,3 +1,5 @@ +from nox import Session + from constants_nox import ( CI_ARGS_EXEC, COMPOSE_FILE, @@ -9,7 +11,6 @@ START_APP, START_APP_WITH_EXTERNAL_POSTGRES, ) -from nox import Session from run_infrastructure import OPS_TEST_DIR, run_infrastructure @@ -30,9 +31,7 @@ def pytest_nox(session: Session, coverage_arg: str) -> None: """Runs any tests of nox commands themselves.""" # the nox tests don't run with coverage, override the provided arg coverage_arg = "--no-cov" - session.notify("teardown") - session.run(*START_APP, external=True) - run_command = (*EXEC, "pytest", coverage_arg, "--noconftest", "tests/nox/") + run_command = ("pytest", coverage_arg, "noxfiles/") session.run(*run_command, external=True) diff --git a/noxfiles/test_docker_nox.py b/noxfiles/test_docker_nox.py new file mode 100644 index 0000000000..885d4f40e3 --- /dev/null +++ b/noxfiles/test_docker_nox.py @@ -0,0 +1,57 @@ +from docker_nox import generate_multiplatform_buildx_command + + +class TestBuildxPrivacyCenter: + def test_single_tag(self) -> None: + actual_result = generate_multiplatform_buildx_command(["foo"], "prod") + expected_result = ( + "docker", + "buildx", + "build", + "--push", + "--target=prod", + "--platform", + "linux/amd64,linux/arm64", + ".", + "--tag", + "foo", + ) + assert actual_result == expected_result + + def test_multiplte_tags(self) -> None: + actual_result = generate_multiplatform_buildx_command(["foo", "bar"], "prod") + expected_result = ( + "docker", + "buildx", + "build", + "--push", + "--target=prod", + "--platform", + "linux/amd64,linux/arm64", + ".", + "--tag", + "foo", + "--tag", + "bar", + ) + assert actual_result == expected_result + + def test_different_path(self) -> None: + actual_result = generate_multiplatform_buildx_command( + ["foo", "bar"], "prod", "other_path" + ) + expected_result = ( + "docker", + "buildx", + "build", + "--push", + "--target=prod", + "--platform", + "linux/amd64,linux/arm64", + "other_path", + "--tag", + "foo", + "--tag", + "bar", + ) + assert actual_result == expected_result diff --git a/tests/nox/test_git_nox.py b/noxfiles/test_git_nox.py similarity index 98% rename from tests/nox/test_git_nox.py rename to noxfiles/test_git_nox.py index 5a490a1daf..861725e986 100644 --- a/tests/nox/test_git_nox.py +++ b/noxfiles/test_git_nox.py @@ -3,7 +3,7 @@ import pytest from git import Repo -from noxfiles.git_nox import generate_tag +from git_nox import generate_tag class TestGitNox: diff --git a/noxfiles/utils_nox.py b/noxfiles/utils_nox.py index 043ed17fb1..aea75f4dfc 100644 --- a/noxfiles/utils_nox.py +++ b/noxfiles/utils_nox.py @@ -2,6 +2,7 @@ from pathlib import Path import nox + from constants_nox import COMPOSE_FILE_LIST from run_infrastructure import run_infrastructure diff --git a/tests/nox/__init__.py b/tests/nox/__init__.py deleted file mode 100644 index e69de29bb2..0000000000