From d26e56f39303804170b44ffab2e466ac607f4627 Mon Sep 17 00:00:00 2001 From: Brady Todhunter Date: Thu, 4 Nov 2021 11:56:22 -0700 Subject: [PATCH 01/13] add PHONY to ironbank target Signed-off-by: Brady Todhunter --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 7acab3dd3..f476b0ea9 100644 --- a/Makefile +++ b/Makefile @@ -196,6 +196,7 @@ push-redhat: setup-test-infra ## (Not available outside of CI) Push prod Anchore push-rebuild: setup-test-infra ## Rebuild and push prod Anchore Engine docker image to Docker Hub (not available outside of CI) @$(CI_CMD) push-prod-image-rebuild "$(COMMIT_SHA)" "$(DEV_IMAGE_REPO)" "$(GIT_TAG)" +.PHONY: ironbank-artifacts ironbank-artifacts: anchore-ci ## (Not available outside of CI) Create and upload ironbank buildblob artifacts @$(CI_CMD) create-ironbank-artifacts anchore-engine "$(GIT_TAG)" From 46583229ab9fab9de1dab9c5c3f59390e4a09816 Mon Sep 17 00:00:00 2001 From: Brady Todhunter Date: Thu, 4 Nov 2021 11:58:04 -0700 Subject: [PATCH 02/13] use args for base images & cleanup labels Signed-off-by: Brady Todhunter --- Dockerfile | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8e455f319..b44884542 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,10 @@ -FROM registry.access.redhat.com/ubi8/ubi:8.4 as anchore-engine-builder +ARG BASE_REGISTRY=registry.access.redhat.com +ARG BASE_IMAGE=ubi8/ubi +ARG BASE_TAG=8.4 -######## This is stage1 where anchore wheels, binary deps, and any items from the source tree get staged to /build_output ######## +#### Start first stage +#### Anchore wheels, binary dependencies, etc. are staged to /build_output for second stage +FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} as anchore-engine-builder ARG CLI_COMMIT @@ -55,11 +59,9 @@ RUN set -ex && \ tar -z -c -v -C /build_output -f /anchore-buildblob.tgz . && \ sha256sum /anchore-buildblob.tgz > /buildblob.tgz.sha256sum -# Build setup section - -FROM registry.access.redhat.com/ubi8/ubi:8.4 as anchore-engine-final - -######## This is stage2 which does setup and install entirely from items from stage1's /build_output ######## +#### Start second stage +#### Setup and install using first stage artifacts in /build_output +FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} as anchore-engine-final ARG CLI_COMMIT ARG ANCHORE_COMMIT @@ -70,16 +72,13 @@ ARG ANCHORE_ENGINE_RELEASE="r0" COPY --from=anchore-engine-builder /build_output /build_output # Container metadata section - -MAINTAINER dev@anchore.com - -LABEL anchore_cli_commit="$CLI_COMMIT" \ - anchore_commit="$ANCHORE_COMMIT" \ +LABEL anchore_cli_commit="${CLI_COMMIT}" \ + anchore_commit="${ANCHORE_COMMIT}" \ name="anchore-engine" \ maintainer="dev@anchore.com" \ vendor="Anchore Inc." \ - version="$ANCHORE_ENGINE_VERSION" \ - release="$ANCHORE_ENGINE_RELEASE" \ + version="${ANCHORE_ENGINE_VERSION}" \ + release="${ANCHORE_ENGINE_RELEASE}" \ summary="Anchore Engine - container image scanning service for policy-based security, best-practice and compliance enforcement." \ description="Anchore is an open platform for container security and compliance that allows developers, operations, and security teams to discover, analyze, and certify container images on-premises or in the cloud. Anchore Engine is the on-prem, OSS, API accessible service that allows ops and developers to perform detailed analysis, run queries, produce reports and define policies on container images that can be used in CI/CD pipelines to ensure that only containers that meet your organization’s requirements are deployed into production." From afcb9f4a28bd0a445f3ec67e93afa8018af1e6f4 Mon Sep 17 00:00:00 2001 From: Brady Todhunter Date: Thu, 4 Nov 2021 12:06:05 -0700 Subject: [PATCH 03/13] clean up env vars Signed-off-by: Brady Todhunter --- Dockerfile | 82 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 43 insertions(+), 39 deletions(-) diff --git a/Dockerfile b/Dockerfile index b44884542..5ffcdd680 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,8 +8,10 @@ FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} as anchore-engine-builder ARG CLI_COMMIT -ENV LANG=en_US.UTF-8 LC_ALL=C.UTF-8 -ENV GOPATH=/go +ENV LANG=en_US.UTF-8 +ENV LC_ALL=C.UTF-8 + +# environment variables for dependent binary versions ENV SYFT_VERSION=v0.26.0 ENV GRYPE_VERSION=v0.22.0 ENV PIP_VERSION=21.0.1 @@ -83,57 +85,59 @@ LABEL anchore_cli_commit="${CLI_COMMIT}" \ description="Anchore is an open platform for container security and compliance that allows developers, operations, and security teams to discover, analyze, and certify container images on-premises or in the cloud. Anchore Engine is the on-prem, OSS, API accessible service that allows ops and developers to perform detailed analysis, run queries, produce reports and define policies on container images that can be used in CI/CD pipelines to ensure that only containers that meet your organization’s requirements are deployed into production." # Environment variables to be present in running environment -ENV LANG=en_US.UTF-8 LC_ALL=C.UTF-8 +ENV AUTHLIB_INSECURE_TRANSPORT=true +ENV LANG=en_US.UTF-8 +ENV LC_ALL=C.UTF-8 +ENV PATH="${PATH}:/anchore-cli/bin" +ENV SET_HOSTID_TO_HOSTNAME=false # Default values overrideable at runtime of the container -ENV ANCHORE_CONFIG_DIR=/config \ - ANCHORE_SERVICE_DIR=/anchore_service \ - ANCHORE_LOG_LEVEL=INFO \ - ANCHORE_ENABLE_METRICS=false \ +ENV ANCHORE_ADMIN_EMAIL=admin@myanchore \ + ANCHORE_ADMIN_PASSWORD=null \ + ANCHORE_AUTH_ENABLE_HASHED_PASSWORDS=false \ + ANCHORE_AUTH_PRIVKEY=null \ + ANCHORE_AUTH_PUBKEY=null \ + ANCHORE_AUTH_SECRET=null \ + ANCHORE_AUTHZ_HANDLER=native \ + ANCHORE_CATALOG_NOTIFICATION_INTERVAL_SEC=30 \ + ANCHORE_CLI_PASS=foobar \ + ANCHORE_CLI_USER=admin \ + ANCHORE_CLI_URL="http://localhost:8228" \ + ANCHORE_CONFIG_DIR=/config \ + ANCHORE_DB_NAME=postgres \ + ANCHORE_DB_PORT=5432 \ + ANCHORE_DB_USER=postgres \ ANCHORE_DISABLE_METRICS_AUTH=false \ - ANCHORE_INTERNAL_SSL_VERIFY=false \ - ANCHORE_WEBHOOK_DESTINATION_URL=null \ - ANCHORE_HINTS_ENABLED=false \ - ANCHORE_FEEDS_ENABLED=true \ - ANCHORE_FEEDS_SSL_VERIFY=true \ + ANCHORE_ENABLE_METRICS=false \ + ANCHORE_ENABLE_PACKAGE_FILTERING="true" \ ANCHORE_ENDPOINT_HOSTNAME=localhost \ ANCHORE_EVENTS_NOTIFICATIONS_ENABLED=false \ - ANCHORE_CATALOG_NOTIFICATION_INTERVAL_SEC=30 \ - ANCHORE_FEED_SYNC_INTERVAL_SEC=21600 \ + ANCHORE_EXTERNAL_AUTHZ_ENDPOINT=null \ ANCHORE_EXTERNAL_PORT=null \ ANCHORE_EXTERNAL_TLS=false \ - ANCHORE_AUTHZ_HANDLER=native \ - ANCHORE_EXTERNAL_AUTHZ_ENDPOINT=null \ - ANCHORE_ADMIN_PASSWORD=null \ - ANCHORE_ADMIN_EMAIL=admin@myanchore \ - ANCHORE_HOST_ID="anchore-quickstart" \ - ANCHORE_DB_PORT=5432 \ - ANCHORE_DB_NAME=postgres \ - ANCHORE_DB_USER=postgres \ - SET_HOSTID_TO_HOSTNAME=false \ - ANCHORE_CLI_USER=admin \ - ANCHORE_CLI_PASS=foobar \ - ANCHORE_SERVICE_PORT=8228 \ - ANCHORE_CLI_URL="http://localhost:8228" \ - ANCHORE_FEEDS_URL="https://ancho.re/v1/service/feeds" \ ANCHORE_FEEDS_CLIENT_URL="https://ancho.re/v1/account/users" \ + ANCHORE_FEEDS_ENABLED=true \ + ANCHORE_FEEDS_SSL_VERIFY=true \ + ANCHORE_FEED_SYNC_INTERVAL_SEC=21600 \ ANCHORE_FEEDS_TOKEN_URL="https://ancho.re/oauth/token" \ - ANCHORE_GLOBAL_CLIENT_READ_TIMEOUT=0 \ + ANCHORE_FEEDS_URL="https://ancho.re/v1/service/feeds" \ ANCHORE_GLOBAL_CLIENT_CONNECT_TIMEOUT=0 \ - ANCHORE_AUTH_PUBKEY=null \ - ANCHORE_AUTH_PRIVKEY=null \ - ANCHORE_AUTH_SECRET=null \ + ANCHORE_GLOBAL_CLIENT_READ_TIMEOUT=0 \ + ANCHORE_GLOBAL_SERVER_REQUEST_TIMEOUT_SEC=180 \ + ANCHORE_GRYPE_DB_URL="https://toolbox-data.anchore.io/grype/databases/listing.json" \ + ANCHORE_HINTS_ENABLED=false \ + ANCHORE_HOST_ID="anchore-quickstart" \ + ANCHORE_INTERNAL_SSL_VERIFY=false \ + ANCHORE_LOG_LEVEL=INFO \ + ANCHORE_MAX_COMPRESSED_IMAGE_SIZE_MB=-1 \ ANCHORE_OAUTH_ENABLED=false \ ANCHORE_OAUTH_TOKEN_EXPIRATION=3600 \ - ANCHORE_AUTH_ENABLE_HASHED_PASSWORDS=false \ - AUTHLIB_INSECURE_TRANSPORT=true \ - ANCHORE_MAX_COMPRESSED_IMAGE_SIZE_MB=-1 \ - ANCHORE_GLOBAL_SERVER_REQUEST_TIMEOUT_SEC=180 \ + ANCHORE_SERVICE_DIR=/anchore_service \ + ANCHORE_SERVICE_PORT=8228 \ ANCHORE_VULNERABILITIES_PROVIDER=null \ - ANCHORE_GRYPE_DB_URL="https://toolbox-data.anchore.io/grype/databases/listing.json" \ - ANCHORE_ENABLE_PACKAGE_FILTERING="true" + ANCHORE_WEBHOOK_DESTINATION_URL=null -ENV PATH "${PATH}:/anchore-cli/bin" +#### Perform OS setup # Insecure transport required in case for things like tls sidecars From a8f40921b940cae1e0b9e7a50ffed5edbaa953f4 Mon Sep 17 00:00:00 2001 From: Brady Todhunter Date: Thu, 4 Nov 2021 12:11:31 -0700 Subject: [PATCH 04/13] reorganize first stage for easier readability Signed-off-by: Brady Todhunter --- Dockerfile | 65 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 43 insertions(+), 22 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5ffcdd680..ea8389834 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,42 +19,63 @@ ENV PIP_VERSION=21.0.1 COPY . /buildsource WORKDIR /buildsource +# setup build artifact directory RUN set -ex && \ - mkdir -p /build_output /build_output/deps /build_output/configs /build_output/wheels /build_output/cli_wheels + mkdir -p \ + /build_output/configs \ + /build_output/cli_wheels \ + /build_output/deps \ + /build_output/wheels +# installing build dependencies RUN set -ex && \ - echo "installing OS dependencies" && \ + echo "installing build dependencies" && \ + # keepcache is used so that subsequent invocations of yum do not remove the cached RPMs in --downloaddir echo "keepcache = 1" >> /etc/yum.conf && \ yum update -y && \ - yum module disable -y python36 && yum module enable -y python38 && \ - yum install -y gcc make python38 git python38-wheel python38-devel python38-psycopg2 go && \ - pip3 install pip=="${PIP_VERSION}" && \ - pip3 download -d /build_output/wheels pip=="${PIP_VERSION}" && \ + yum module disable -y python36 && \ + yum module enable -y python38 && \ + yum install -y \ + gcc \ + git \ + go \ + make \ + python38 \ + python38-devel \ + python38-psycopg2 \ + python38-wheel && \ yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \ - yum install -y --downloadonly --downloaddir=/build_output/build_deps/ dpkg clamav clamav-update + pip3 install pip=="${PIP_VERSION}" -# create anchore binaries +# stage dependent binaries into /build_output RUN set -ex && \ - echo "installing anchore" && \ - pip3 wheel --wheel-dir=/build_output/wheels . && \ - pip3 wheel --wheel-dir=/build_output/cli_wheels/ git+git://github.com/anchore/anchore-cli.git@"$CLI_COMMIT"\#egg=anchorecli && \ - cp ./LICENSE /build_output/ && \ - cp ./conf/default_config.yaml /build_output/configs/default_config.yaml && \ - cp ./docker-entrypoint.sh /build_output/configs/docker-entrypoint.sh && \ - cp -R ./anchore_engine/conf/clamav /build_output/configs/ + echo "downloading OS dependencies" && \ + pip3 download -d /build_output/wheels pip=="${PIP_VERSION}" && \ + yum install -y --downloadonly --downloaddir=/build_output/build_deps/ \ + clamav \ + clamav-update \ + dpkg -# stage anchore dependency binaries RUN set -ex && \ - echo "installing GO" && \ - mkdir -p /go + echo "downloading anchore-cli" && \ + pip3 wheel --wheel-dir=/build_output/cli_wheels/ git+git://github.com/anchore/anchore-cli.git@"${CLI_COMMIT}"\#egg=anchorecli RUN set -ex && \ - echo "installing Syft" && \ - curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /build_output/deps "$SYFT_VERSION" + echo "downloading Syft" && \ + curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /build_output/deps "${SYFT_VERSION}" RUN set -ex && \ - echo "installing Grype" && \ - curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /build_output/deps "$GRYPE_VERSION" + echo "downloading Grype" && \ + curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /build_output/deps "${GRYPE_VERSION}" + +# stage anchore-engine wheels and default application configs into /build_output +RUN set -ex && \ + echo "creating anchore-engine wheels" && \ + pip3 wheel --wheel-dir=/build_output/wheels . && \ + cp ./LICENSE /build_output/ && \ + cp ./conf/default_config.yaml /build_output/configs/default_config.yaml && \ + cp ./docker-entrypoint.sh /build_output/configs/docker-entrypoint.sh && \ + cp -R ./anchore_engine/conf/clamav /build_output/configs/ # create p1 buildblob & checksum RUN set -ex && \ From c53a9c8247ee19027f6a9b17d6cdf5a5f249eb81 Mon Sep 17 00:00:00 2001 From: Brady Todhunter Date: Thu, 4 Nov 2021 12:14:25 -0700 Subject: [PATCH 05/13] reorder first stage for better caching Signed-off-by: Brady Todhunter --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index ea8389834..4712bf5d5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,9 +16,6 @@ ENV SYFT_VERSION=v0.26.0 ENV GRYPE_VERSION=v0.22.0 ENV PIP_VERSION=21.0.1 -COPY . /buildsource -WORKDIR /buildsource - # setup build artifact directory RUN set -ex && \ mkdir -p \ @@ -68,6 +65,9 @@ RUN set -ex && \ echo "downloading Grype" && \ curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /build_output/deps "${GRYPE_VERSION}" +COPY . /buildsource +WORKDIR /buildsource + # stage anchore-engine wheels and default application configs into /build_output RUN set -ex && \ echo "creating anchore-engine wheels" && \ From f03acc45862969060174fdfcc4af83b16ee5bc78 Mon Sep 17 00:00:00 2001 From: Brady Todhunter Date: Thu, 4 Nov 2021 12:16:11 -0700 Subject: [PATCH 06/13] reorder second stage for better readability Signed-off-by: Brady Todhunter --- Dockerfile | 75 +++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 52 insertions(+), 23 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4712bf5d5..db9699213 100644 --- a/Dockerfile +++ b/Dockerfile @@ -160,33 +160,58 @@ ENV ANCHORE_ADMIN_EMAIL=admin@myanchore \ #### Perform OS setup -# Insecure transport required in case for things like tls sidecars - -# Container run environment settings - -#VOLUME /analysis_scratch -EXPOSE "${ANCHORE_SERVICE_PORT}" - -# Build dependencies - +# Setup container user/group and required application directories +RUN set -ex && \ + groupadd --gid 1000 anchore && \ + useradd --uid 1000 --gid anchore --shell /bin/bash --create-home anchore && \ + mkdir -p \ + /analysis_scratch \ + "${ANCHORE_SERVICE_DIR}"/bundles \ + /config \ + /home/anchore/clamav/db \ + /licenses \ + /var/log/anchore \ + /var/run/anchore \ + /workspace \ + /workspace_preload && \ + chown -R 1000:0 \ + /analysis_scratch \ + "${ANCHORE_SERVICE_DIR}" \ + /config \ + /home/anchore \ + /licenses \ + /var/log/anchore \ + /var/run/anchore \ + /workspace \ + /workspace_preload && \ + chmod -R g+rwX \ + /analysis_scratch \ + "${ANCHORE_SERVICE_DIR}" \ + /config \ + /home/anchore \ + /licenses \ + /var/log/anchore \ + /var/run/anchore \ + /workspace \ + /workspace_preload + +# Install build dependencies RUN set -ex && \ yum update -y && \ - yum module disable -y python36 && yum module enable -y python38 && \ - yum install -y python38 python38-wheel procps psmisc python38-psycopg2 skopeo && \ - pip3 install --upgrade --no-index --find-links=/build_output/wheels/ pip - -# Setup container default configs and directories - -WORKDIR /anchore-engine - -# Perform OS setup + yum module disable -y python36 && \ + yum module enable -y python38 && \ + yum install -y \ + procps \ + psmisc \ + python38 \ + python38-psycopg2 \ + python38-wheel \ + skopeo && \ + yum clean all +# Copy default application configuration files RUN set -ex && \ - groupadd --gid 1000 anchore && \ - useradd --uid 1000 --gid anchore --shell /bin/bash --create-home anchore && \ - mkdir /config && \ - mkdir /licenses && \ - mkdir -p /workspace_preload /var/log/anchore /var/run/anchore /analysis_scratch /workspace /anchore_service/bundles "${ANCHORE_SERVICE_DIR}"/bundles /home/anchore/clamav/db && \ + echo "copying default application config files" && \ cp /build_output/LICENSE /licenses/ && \ cp /build_output/configs/default_config.yaml /config/config.yaml && \ cp /build_output/configs/docker-entrypoint.sh /docker-entrypoint.sh && \ @@ -223,5 +248,9 @@ HEALTHCHECK --start-period=20s \ USER 1000 +EXPOSE "${ANCHORE_SERVICE_PORT}" + +WORKDIR /anchore-engine + ENTRYPOINT ["/docker-entrypoint.sh"] CMD ["anchore-manager", "service", "start", "--all"] From 6f7d592bf9f3c7269af2a9dbe49758882b1ae41d Mon Sep 17 00:00:00 2001 From: Brady Todhunter Date: Thu, 4 Nov 2021 12:21:47 -0700 Subject: [PATCH 07/13] move anchore-engine & deps installation to single layer below COPY command for better caching & smaller image size Signed-off-by: Brady Todhunter --- Dockerfile | 54 +++++++++++++++++++++++++++++------------------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/Dockerfile b/Dockerfile index db9699213..5195c6256 100644 --- a/Dockerfile +++ b/Dockerfile @@ -91,9 +91,6 @@ ARG ANCHORE_COMMIT ARG ANCHORE_ENGINE_VERSION="1.0.1" ARG ANCHORE_ENGINE_RELEASE="r0" -# Copy skopeo artifacts from build step -COPY --from=anchore-engine-builder /build_output /build_output - # Container metadata section LABEL anchore_cli_commit="${CLI_COMMIT}" \ anchore_commit="${ANCHORE_COMMIT}" \ @@ -209,37 +206,44 @@ RUN set -ex && \ skopeo && \ yum clean all -# Copy default application configuration files -RUN set -ex && \ - echo "copying default application config files" && \ - cp /build_output/LICENSE /licenses/ && \ - cp /build_output/configs/default_config.yaml /config/config.yaml && \ - cp /build_output/configs/docker-entrypoint.sh /docker-entrypoint.sh && \ - cp /build_output/configs/clamav/freshclam.conf /home/anchore/clamav/ && \ - chown -R 1000:0 /workspace_preload /var/log/anchore /var/run/anchore /analysis_scratch /workspace /anchore_service "${ANCHORE_SERVICE_DIR}" /home/anchore && \ - chmod -R g+rwX /workspace_preload /var/log/anchore /var/run/anchore /analysis_scratch /workspace /anchore_service "${ANCHORE_SERVICE_DIR}" /home/anchore && \ - chmod -R ug+rw /home/anchore/clamav && \ - md5sum /config/config.yaml > /config/build_installed && \ - chmod +x /docker-entrypoint.sh - - -# Perform any base OS specific setup +# Copy the installed artifacts from the first stage +COPY --from=anchore-engine-builder /build_output /build_output -# Perform the cli install into a virtual env +# install application & all dependencies from /build_output RUN set -ex && \ + echo "updating pip" && \ + pip3 install --upgrade --no-index --find-links=/build_output/wheels/ pip && \ + # Install anchore-cli into a virtual environment + echo "installing anchore-cli into virtual environment" && \ python3 -m venv /anchore-cli && \ source /anchore-cli/bin/activate && \ pip3 install --no-index --find-links=/build_output/cli_wheels/ anchorecli && \ - deactivate - -# Perform the anchore-engine build and install - -RUN set -ex && \ + deactivate && \ + # Install anchore-engine & required dependencies + echo "installing anchore-engine and required dependencies" && \ pip3 install --no-index --find-links=/build_output/wheels/ anchore-engine && \ + # copy all staged dependent binaries to PATH cp /build_output/deps/syft /usr/bin/syft && \ cp /build_output/deps/grype /usr/bin/grype && \ yum install -y /build_output/build_deps/*.rpm && \ - rm -rf /build_output /root/.cache + # Copy default application configuration files + echo "copying default application config files" && \ + cp /build_output/LICENSE /licenses/ && \ + cp /build_output/configs/default_config.yaml /config/config.yaml && \ + md5sum /config/config.yaml > /config/build_installed && \ + cp /build_output/configs/docker-entrypoint.sh /docker-entrypoint.sh && \ + chmod +x /docker-entrypoint.sh && \ + cp /build_output/configs/clamav/freshclam.conf /home/anchore/clamav/ && \ + chmod -R ug+rw /home/anchore/clamav && \ + # clean up unnecessary artifacts from filesystem + yum clean all && \ + echo "cleaning up unneccesary files used for testing/cache/build" && \ + rm -rf \ + /build_output \ + /root/.cache \ + /usr/local/lib64/python3.8/site-packages/twisted/test \ + /usr/local/lib64/python3.8/site-packages/Crypto/SelfTest \ + /usr/share/doc # Container runtime instructions From 35cc5326a359d2c7a588a5c6e9d0f54b0d97b7a6 Mon Sep 17 00:00:00 2001 From: Brady Todhunter Date: Thu, 4 Nov 2021 12:25:48 -0700 Subject: [PATCH 08/13] make CI jobs NEVER use cached layers when building (to ensure yum updates get run) Signed-off-by: Brady Todhunter --- scripts/ci/build | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/ci/build b/scripts/ci/build index 00cd4510a..a22935fe3 100755 --- a/scripts/ci/build +++ b/scripts/ci/build @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -euvo pipefail +set -exuvo pipefail COMMIT_SHA="${1:?'Missing required parameter: COMMIT_SHA'}" GIT_TAG="${2:?'Missing required parameter: GIT_TAG'}" @@ -20,10 +20,12 @@ fi print_colorized INFO "Building Anchore Engine image ${TEST_IMAGE_NAME}; installing anchore-cli from git@${anchore_cli_commit}."; echo -# A targeted build can be used to populate the cache in circleci if you're using the docker_layer_caching option in the job. -# Since we're not using docker_layer_caching, this build step can be ignored. -# docker build --target anchore-engine-builder -t anchore-engine:builder -f ./Dockerfile . +if [[ ${CI:-false} == "true" ]]; then + BUILD_CACHE_ARG="--no-cache" +else + BUILD_CACHE_ARG="" +fi -docker build --build-arg ANCHORE_COMMIT="${COMMIT_SHA}" --build-arg CLI_COMMIT="${anchore_cli_commit}" -t "${TEST_IMAGE_NAME}" -f ./Dockerfile . +docker build ${BUILD_CACHE_ARG} --build-arg ANCHORE_COMMIT="${COMMIT_SHA}" --build-arg CLI_COMMIT="${anchore_cli_commit}" -t "${TEST_IMAGE_NAME}" -f ./Dockerfile . print_colorized INFO "Built Anchore Engine image ${TEST_IMAGE_NAME}."; echo From 1ad5aeaeb59f7cba2e298d7ccf97335cfe54c8b6 Mon Sep 17 00:00:00 2001 From: Samuel Dacanay Date: Wed, 10 Nov 2021 16:13:17 +0200 Subject: [PATCH 09/13] Add unit test for make_response_error Signed-off-by: Samuel Dacanay --- .../anchore_engine/common/test_helpers.py | 161 ++++++++++++++++++ 1 file changed, 161 insertions(+) diff --git a/tests/unit/anchore_engine/common/test_helpers.py b/tests/unit/anchore_engine/common/test_helpers.py index df45e71b2..60b54914c 100644 --- a/tests/unit/anchore_engine/common/test_helpers.py +++ b/tests/unit/anchore_engine/common/test_helpers.py @@ -45,3 +45,164 @@ def test_valid_data(self): assert len(extracted_content[key]["licenses"]) == 1 assert extracted_content[key]["licenses"][0] == "MIT" assert len(extracted_content[key]["cpes"]) > 0 + + +class TestMakeResponseError: + class TestException(Exception): + def __init__(self, msg, anchore_error_json=None): + super().__init__(msg) + if anchore_error_json is not None: + self.anchore_error_json = anchore_error_json + + params = [ + pytest.param( + { + "errmsg": "basic-test-case", + "in_httpcode": None, + "details": None, + "expected": { + "message": "basic-test-case", + "httpcode": 500, + "detail": {"error_codes": []}, + }, + }, + id="basic", + ), + pytest.param( + { + "errmsg": "basic-test-case", + "in_httpcode": 400, + "details": None, + "expected": { + "message": "basic-test-case", + "httpcode": 400, + "detail": {"error_codes": []}, + }, + }, + id="basic-with-httpcode", + ), + pytest.param( + { + "errmsg": "basic-test-case", + "in_httpcode": None, + "details": {"test": "value"}, + "expected": { + "message": "basic-test-case", + "httpcode": 500, + "detail": { + "test": "value", + "error_codes": [], + }, + }, + }, + id="basic-with-details", + ), + pytest.param( + { + "errmsg": "basic-test-case", + "in_httpcode": None, + "details": {"error_codes": [500, 404]}, + "expected": { + "message": "basic-test-case", + "httpcode": 500, + "detail": {"error_codes": [500, 404]}, + }, + }, + id="basic-with-error-codes", + ), + pytest.param( + { + "errmsg": Exception("thisisatest"), + "in_httpcode": None, + "details": None, + "expected": { + "message": "thisisatest", + "httpcode": 500, + "detail": {"error_codes": []}, + }, + }, + id="basic-exception", + ), + pytest.param( + { + "errmsg": TestException( + "testexception", + anchore_error_json={ + "message": "test", + "httpcode": 500, + "detail": {"error_codes": [404]}, + }, + ), + "in_httpcode": 400, + "details": None, + "expected": { + "message": "test", + "httpcode": 500, + "detail": {"error_codes": [404]}, + }, + }, + id="basic-exception-with-anchore-error-json", + ), + pytest.param( + { + "errmsg": TestException( + "testexception", + anchore_error_json={ + "message": "test", + "httpcode": 500, + "detail": {"error_codes": [404]}, + "error_code": 401, + }, + ), + "in_httpcode": 400, + "details": None, + "expected": { + "message": "test", + "httpcode": 500, + "detail": {"error_codes": [404, 401]}, + }, + }, + id="basic-exception-with-anchore-error-json-and-error-code", + ), + pytest.param( + { + "errmsg": TestException( + "testexception", + anchore_error_json='{"message": "test", "httpcode": 500, "detail": {"error_codes": [404]}}', + ), + "in_httpcode": 400, + "details": None, + "expected": { + "message": "test", + "httpcode": 500, + "detail": {"error_codes": [404]}, + }, + }, + id="basic-exception-with-json-string", + ), + pytest.param( + { + "errmsg": TestException( + "testexception", + anchore_error_json='{"message" "test", "httpcode": 500, "detail": {"error_codes": [404]}}', + ), + "in_httpcode": 400, + "details": None, + "expected": { + "message": "testexception", + "httpcode": 400, + "detail": {"error_codes": []}, + }, + }, + id="basic-exception-with-bad-json-string", + ), + ] + + @pytest.mark.parametrize("param", params) + def test_make_response_error(self, param): + actual = helpers.make_response_error( + param["errmsg"], param["in_httpcode"], param["details"] + ) + assert actual["message"] == param["expected"]["message"] + assert actual["httpcode"] == param["expected"]["httpcode"] + assert actual["detail"] == param["expected"]["detail"] From 4afb7447262b5a8cc5733820c22f34dca6117415 Mon Sep 17 00:00:00 2001 From: Brady Todhunter Date: Wed, 10 Nov 2021 17:04:18 -0800 Subject: [PATCH 10/13] update UBI base image to v8.5 Signed-off-by: Brady Todhunter --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5195c6256..8d616e409 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ ARG BASE_REGISTRY=registry.access.redhat.com ARG BASE_IMAGE=ubi8/ubi -ARG BASE_TAG=8.4 +ARG BASE_TAG=8.5 #### Start first stage #### Anchore wheels, binary dependencies, etc. are staged to /build_output for second stage From 23d249852da652e8cb1d40e0fb9d88ccd8af7c5b Mon Sep 17 00:00:00 2001 From: Brady Todhunter Date: Wed, 10 Nov 2021 17:05:22 -0800 Subject: [PATCH 11/13] get freshclam config file from anchore-engine installation path Signed-off-by: Brady Todhunter --- Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8d616e409..7cb97edcc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -74,8 +74,7 @@ RUN set -ex && \ pip3 wheel --wheel-dir=/build_output/wheels . && \ cp ./LICENSE /build_output/ && \ cp ./conf/default_config.yaml /build_output/configs/default_config.yaml && \ - cp ./docker-entrypoint.sh /build_output/configs/docker-entrypoint.sh && \ - cp -R ./anchore_engine/conf/clamav /build_output/configs/ + cp ./docker-entrypoint.sh /build_output/configs/docker-entrypoint.sh # create p1 buildblob & checksum RUN set -ex && \ @@ -233,7 +232,7 @@ RUN set -ex && \ md5sum /config/config.yaml > /config/build_installed && \ cp /build_output/configs/docker-entrypoint.sh /docker-entrypoint.sh && \ chmod +x /docker-entrypoint.sh && \ - cp /build_output/configs/clamav/freshclam.conf /home/anchore/clamav/ && \ + cp -R $(pip3 show anchore-engine | grep Location: | cut -c 11-)/anchore_engine/conf/clamav/freshclam.conf /home/anchore/clamav/ && \ chmod -R ug+rw /home/anchore/clamav && \ # clean up unnecessary artifacts from filesystem yum clean all && \ From 721c8e6f1be3b6d1b091ead54702553fae8bcfbc Mon Sep 17 00:00:00 2001 From: Brady Todhunter Date: Wed, 10 Nov 2021 17:13:06 -0800 Subject: [PATCH 12/13] split last layer into multiple layers. Using a single layer doesnt save enough space to justify how much harder it is to read Signed-off-by: Brady Todhunter --- Dockerfile | 29 +++++++++++++++-------------- scripts/ci/build | 2 +- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7cb97edcc..a8403c503 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,7 +48,7 @@ RUN set -ex && \ RUN set -ex && \ echo "downloading OS dependencies" && \ pip3 download -d /build_output/wheels pip=="${PIP_VERSION}" && \ - yum install -y --downloadonly --downloaddir=/build_output/build_deps/ \ + yum install -y --downloadonly --downloaddir=/build_output/deps/ \ clamav \ clamav-update \ dpkg @@ -57,11 +57,11 @@ RUN set -ex && \ echo "downloading anchore-cli" && \ pip3 wheel --wheel-dir=/build_output/cli_wheels/ git+git://github.com/anchore/anchore-cli.git@"${CLI_COMMIT}"\#egg=anchorecli -RUN set -ex && \ +RUN set -exo pipefail && \ echo "downloading Syft" && \ curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /build_output/deps "${SYFT_VERSION}" -RUN set -ex && \ +RUN set -exo pipefail && \ echo "downloading Grype" && \ curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /build_output/deps "${GRYPE_VERSION}" @@ -208,24 +208,27 @@ RUN set -ex && \ # Copy the installed artifacts from the first stage COPY --from=anchore-engine-builder /build_output /build_output -# install application & all dependencies from /build_output +# Install anchore-cli into a virtual environment RUN set -ex && \ echo "updating pip" && \ pip3 install --upgrade --no-index --find-links=/build_output/wheels/ pip && \ - # Install anchore-cli into a virtual environment echo "installing anchore-cli into virtual environment" && \ python3 -m venv /anchore-cli && \ source /anchore-cli/bin/activate && \ pip3 install --no-index --find-links=/build_output/cli_wheels/ anchorecli && \ - deactivate && \ - # Install anchore-engine & required dependencies - echo "installing anchore-engine and required dependencies" && \ - pip3 install --no-index --find-links=/build_output/wheels/ anchore-engine && \ - # copy all staged dependent binaries to PATH + deactivate + +# Install required OS deps & application config files +RUN set -exo pipefail && \ cp /build_output/deps/syft /usr/bin/syft && \ cp /build_output/deps/grype /usr/bin/grype && \ - yum install -y /build_output/build_deps/*.rpm && \ - # Copy default application configuration files + yum install -y /build_output/deps/*.rpm && \ + yum clean all + +# Install anchore-engine & cleanup filesystem +RUN set -ex && \ + echo "installing anchore-engine and required dependencies" && \ + pip3 install --no-index --find-links=/build_output/wheels/ anchore-engine && \ echo "copying default application config files" && \ cp /build_output/LICENSE /licenses/ && \ cp /build_output/configs/default_config.yaml /config/config.yaml && \ @@ -234,8 +237,6 @@ RUN set -ex && \ chmod +x /docker-entrypoint.sh && \ cp -R $(pip3 show anchore-engine | grep Location: | cut -c 11-)/anchore_engine/conf/clamav/freshclam.conf /home/anchore/clamav/ && \ chmod -R ug+rw /home/anchore/clamav && \ - # clean up unnecessary artifacts from filesystem - yum clean all && \ echo "cleaning up unneccesary files used for testing/cache/build" && \ rm -rf \ /build_output \ diff --git a/scripts/ci/build b/scripts/ci/build index a22935fe3..be10ceb92 100755 --- a/scripts/ci/build +++ b/scripts/ci/build @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -exuvo pipefail +set -euvo pipefail COMMIT_SHA="${1:?'Missing required parameter: COMMIT_SHA'}" GIT_TAG="${2:?'Missing required parameter: GIT_TAG'}" From 71c75447e76a14140c35e47579cd51c773c04c8f Mon Sep 17 00:00:00 2001 From: Zach Hill Date: Mon, 27 Sep 2021 15:10:03 -0700 Subject: [PATCH 13/13] [tech-debt] Removes unused to_grype() methods from PackageMapper type hierarchy Signed-off-by: Zach Hill --- .../policy_engine/engine/vulns/mappers.py | 156 +----------------- 1 file changed, 1 insertion(+), 155 deletions(-) diff --git a/anchore_engine/services/policy_engine/engine/vulns/mappers.py b/anchore_engine/services/policy_engine/engine/vulns/mappers.py index ebacd0230..f2862f93f 100644 --- a/anchore_engine/services/policy_engine/engine/vulns/mappers.py +++ b/anchore_engine/services/policy_engine/engine/vulns/mappers.py @@ -15,7 +15,7 @@ Vulnerability, VulnerabilityMatch, ) -from anchore_engine.db import Image, ImageCpe, ImagePackage +from anchore_engine.db import Image from anchore_engine.services.policy_engine.engine.vulns.utils import get_api_endpoint from anchore_engine.subsys import logger as log from anchore_engine.util.cpe_generators import ( @@ -51,26 +51,6 @@ def __init__(self, engine_type, grype_type, grype_language=None): # default language to blank string self.grype_language = grype_language if grype_language else "" - def to_grype( - self, - image_package: ImagePackage, - location_cpes_dict: Dict[str, List[str]] = None, - ): - artifact = { - "id": str(uuid.uuid4()), - "name": image_package.name, - "version": image_package.fullversion, - "type": self.grype_type, - "locations": [ - { - "path": image_package.pkg_path, - } - ], - # Package type specific mappers add metadata attribute - } - - return artifact - def image_content_to_grype_sbom(self, record: Dict): """ Creates a grype sbom formatted record from a single image content API response record @@ -98,25 +78,6 @@ class KBMapper(PackageMapper): def __init__(self): super(KBMapper, self).__init__(engine_type="kb", grype_type="msrc-kb") - def to_grype( - self, - image_package: ImagePackage, - location_cpes_dict: Dict[str, List[str]] = None, - ): - """ - Convert this image package to a sbom artifact suitable for Grype input. - The name is the Microsoft Product ID, which is the Windows OS version. - The version is the KB (Knowledge Base) ID, which is the identifier for the patch. - """ - artifact = { - "id": str(uuid.uuid4()), - "name": image_package.src_pkg, - "version": image_package.version, - "type": self.grype_type, - "locations": [{"path": image_package.pkg_path}], - } - return artifact - def image_content_to_grype_sbom(self, record: Dict): artifact = { "id": str(uuid.uuid4()), @@ -142,39 +103,6 @@ class RpmMapper(LinuxDistroPackageMapper): def __init__(self): super(RpmMapper, self).__init__(engine_type="rpm", grype_type="rpm") - def to_grype( - self, - image_package: ImagePackage, - location_cpes_dict: Dict[str, List[str]] = None, - ): - """ - Adds the source package information to grype sbom - - Source package has already been through 2 transformations before this point - 1. From syft sbom to analyzer manifest in anchore_engine/analyzers/syft/handlers/rpm.py - 2. From analyzer manifest to policy-engine ImagePackage in anchore_engine/services/policy_engine/engine/loaders.py - - After the above transformations ImagePackage.src_pkg is the equivalent of syft/grype sourceRpm - - """ - artifact = super().to_grype(image_package, location_cpes_dict) - if image_package.normalized_src_pkg != "N/A": - artifact["metadataType"] = "RpmdbMetadata" - artifact["metadata"] = {"sourceRpm": image_package.src_pkg} - - # This epoch handling is necessary because in RPMs the epoch of the binary package is often not part of the - # sourceRpm name, but Grype needs it to do the version comparison correctly. - # Without this step Grype will use the wrong version string for the vulnerability match - epoch_str, version, release = split_fullversion(image_package.fullversion) - if epoch_str: - try: - artifact["epoch"] = int(epoch_str) - except ValueError: - # not a valid epoch, something went wrong - pass - - return artifact - def image_content_to_grype_sbom(self, record: Dict): """ Adds the source package information to grype sbom @@ -221,48 +149,6 @@ class DpkgMapper(LinuxDistroPackageMapper): def __init__(self): super(DpkgMapper, self).__init__(engine_type="dpkg", grype_type="deb") - def to_grype( - self, - image_package: ImagePackage, - location_cpes_dict: Dict[str, List[str]] = None, - ): - """ - Adds the source package information to grype sbom - - Source package has already been through 2 transformations before this point. These transformations make the final result more error prone unlike rpm - 1. From syft sbom to analyzer manifest in anchore_engine/analyzers/syft/handlers/debian.py. combines source package and source/package version into one value - 2. From analyzer manifest to policy-engine ImagePackage in anchore_engine/services/policy_engine/engine/loaders.py. attempts to split the source package and version - - After the above transformations ImagePackage.normalized_src_pkg is the closest approximation to syft/grype source. - Closest because the corner cases are not handled correctly by the above transformations - - Example of not-working case - Syft output of bsdutils package - { - "name": "bsdutils", - "version": "1:2.20.1-5.3", - "type": "deb", - "foundBy": "dpkgdb-cataloger", - "metadataType": "DpkgMetadata", - "metadata": { - "package": "bsdutils", - "source": "util-linux", - "version": "1:2.20.1-5.3", - "sourceVersion": "2.20.1-5.3", - ... - - Notice version and sourceVersion are different because of the epoch - - Step 1 processes this into sourcepkg=util-linux-2.20.1-5.3 - - Step 2 falters in it's processing because of the epoch difference and saves util-linux-2.20.1-5.3 to both - ImagePackage.src_pkg and ImagePackage.normalized_src_pkg. The correct value for normalized_src_pkg is util-linux - - """ - artifact = super().to_grype(image_package, location_cpes_dict) - if image_package.normalized_src_pkg != "N/A": - artifact["metadataType"] = "DpkgMetadata" - artifact["metadata"] = {"source": image_package.normalized_src_pkg} - return artifact - def image_content_to_grype_sbom(self, record: Dict): """ Adds the source package information to grype sbom @@ -284,21 +170,6 @@ class ApkgMapper(LinuxDistroPackageMapper): def __init__(self): super(ApkgMapper, self).__init__(engine_type="APKG", grype_type="apk") - def to_grype( - self, - image_package: ImagePackage, - location_cpes_dict: Dict[str, List[str]] = None, - ): - artifact = super().to_grype(image_package, location_cpes_dict) - - # pkgdb/ prefix is added to all os package locations, it's the only way to associate a package with it's cpes - cpes = location_cpes_dict.get(f"pkgdb/{image_package.name}") - if cpes: - # populate cpes for os packages - artifact["cpes"] = [cpe.get_cpe23_fs_for_sbom() for cpe in cpes] - - return artifact - def image_content_to_grype_sbom(self, record: Dict): """ Adds the origin package information to grype sbom @@ -316,31 +187,6 @@ def image_content_to_grype_sbom(self, record: Dict): class CPEMapper(PackageMapper): - def to_grype( - self, - image_package: ImagePackage, - location_cpes_dict: Dict[str, List[ImageCpe]] = None, - ): - cpes = location_cpes_dict.get(image_package.pkg_path) - if cpes: - artifact = { - "id": str(uuid.uuid4()), - "name": image_package.name, - "type": self.grype_type, - "language": self.grype_language, - "locations": [ - { - "path": image_package.pkg_path, - } - ], - "cpes": [cpe.get_cpe23_fs_for_sbom() for cpe in cpes], - "version": cpes[0].version, # set the version explicitly for grype - } - - return artifact - else: - raise ValueError("No CPEs found for package={}".format(image_package.name)) - def fallback_cpe_generator(self, record: Dict) -> List[str]: return generate_fuzzy_cpes( record.get("package"), record.get("version"), self.engine_type