Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

build: build all images + run tests using them #1069

Merged
merged 4 commits into from
Apr 8, 2024
Merged

build: build all images + run tests using them #1069

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ad38273
ci: switch keycloak out of containers (#1070)
mnaser Apr 8, 2024
2174fa5
ci: use merge-mode squash
mnaser Apr 8, 2024
893c0c6
ci: fix merge-mode
mnaser Apr 8, 2024
c690f93
zed: build all images + run tests using them
mnaser Apr 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
188 changes: 27 additions & 161 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,161 +1,27 @@
FROM ubuntu:jammy-20240227 AS ubuntu
LABEL org.opencontainers.image.source=https://github.com/vexxhost/atmosphere

FROM ubuntu AS helm
ARG TARGETOS
ARG TARGETARCH
ARG HELM_VERSION=3.14.0
ADD https://get.helm.sh/helm-v${HELM_VERSION}-${TARGETOS}-${TARGETARCH}.tar.gz /helm.tar.gz
RUN tar -xzf /helm.tar.gz
RUN mv /${TARGETOS}-${TARGETARCH}/helm /usr/bin/helm

FROM ubuntu AS ubuntu-cloud-archive
ADD --chmod=644 https://git.launchpad.net/ubuntu/+source/ubuntu-keyring/plain/keyrings/ubuntu-cloud-keyring.gpg /etc/apt/trusted.gpg.d/ubuntu-cloud-keyring.gpg
ARG RELEASE
RUN <<EOF bash -xe
source /etc/os-release
if [ "\${VERSION_CODENAME}" = "jammy" ]; then \
if [ "${RELEASE}" = "yoga" ]; then \
# NOTE: Yoga shipped with 22.04, so no need to add an extra repository.
echo "" > /etc/apt/sources.list.d/cloudarchive.list; \
elif [ "${RELEASE}" = "zed" ]; then \
echo "deb http://ubuntu-cloud.archive.canonical.com/ubuntu \${VERSION_CODENAME}-updates/${RELEASE} main" > /etc/apt/sources.list.d/cloudarchive.list; \
elif [ "${RELEASE}" = "2023.1" ]; then \
echo "deb http://ubuntu-cloud.archive.canonical.com/ubuntu \${VERSION_CODENAME}-updates/antelope main" > /etc/apt/sources.list.d/cloudarchive.list; \
elif [ "${RELEASE}" = "2023.2" ]; then \
echo "deb http://ubuntu-cloud.archive.canonical.com/ubuntu \${VERSION_CODENAME}-updates/bobcat main" > /etc/apt/sources.list.d/cloudarchive.list; \
elif [ "${RELEASE}" = "master" ]; then \
echo "deb http://ubuntu-cloud.archive.canonical.com/ubuntu \${VERSION_CODENAME}-updates/caracal main" > /etc/apt/sources.list.d/cloudarchive.list; \
else \
echo "${RELEASE} is not supported on \${VERSION_CODENAME}"; \
exit 1; \
fi; \
else
echo "Unsupported release"; \
exit 1; \
fi
EOF

FROM alpine/git AS requirements
ARG BRANCH
ADD https://opendev.org/openstack/requirements.git#${BRANCH} /src
RUN <<EOF sh -xe
sed -i 's/cryptography===36.0.2/cryptography===42.0.4/' /src/upper-constraints.txt
sed -i 's/cryptography===40.0.2/cryptography===42.0.4/' /src/upper-constraints.txt
sed -i 's/cryptography===41.0.7/cryptography===42.0.4/' /src/upper-constraints.txt
sed -i 's/Django===3.2.18/Django===3.2.24/' /src/upper-constraints.txt
sed -i 's/Flask===2.2.3/Flask===2.2.5/' /src/upper-constraints.txt
sed -i 's/Jinja2===3.1.2/Jinja2===3.1.3/' /src/upper-constraints.txt
sed -i 's/oauthlib===3.2.0/oauthlib===3.2.2/' /src/upper-constraints.txt
sed -i 's/paramiko===2.11.0/paramiko===3.4.0/' /src/upper-constraints.txt
sed -i 's/paramiko===3.1.0/paramiko===3.4.0/' /src/upper-constraints.txt
sed -i 's/protobuf===4.21.5/protobuf===4.21.6/' /src/upper-constraints.txt
sed -i 's/pyOpenSSL===22.0.0/pyOpenSSL===24.0.0/' /src/upper-constraints.txt
sed -i 's/pyOpenSSL===23.1.1/pyOpenSSL===24.0.0/' /src/upper-constraints.txt
sed -i 's/requests===2.28.1/requests===2.31.0/' /src/upper-constraints.txt
sed -i 's/requests===2.28.2/requests===2.31.0/' /src/upper-constraints.txt
sed -i 's/sqlparse===0.4.2/sqlparse===0.4.4/' /src/upper-constraints.txt
sed -i 's/urllib3===1.26.12/urllib3===1.26.18/' /src/upper-constraints.txt
sed -i 's/urllib3===1.26.15/urllib3===1.26.18/' /src/upper-constraints.txt
sed -i 's/Werkzeug===2.2.2/Werkzeug===2.3.8/' /src/upper-constraints.txt
sed -i 's/Werkzeug===2.2.3/Werkzeug===2.3.8/' /src/upper-constraints.txt
sed -i 's/zstd===1.5.2.5/zstd===1.5.4.0/' /src/upper-constraints.txt
sed -i '/glance-store/d' /src/upper-constraints.txt
sed -i '/horizon/d' /src/upper-constraints.txt
EOF

FROM ubuntu-cloud-archive AS openstack-venv-builder
RUN <<EOF bash -xe
apt-get update -qq
apt-get install -qq -y --no-install-recommends \
build-essential \
git \
libldap2-dev \
libpcre3-dev \
libsasl2-dev \
libssl-dev \
lsb-release \
openssh-client \
python3 \
python3-dev \
python3-pip \
python3-venv
EOF
RUN <<EOF bash -xe
python3 -m venv --upgrade-deps --system-site-packages /var/lib/openstack
EOF
ENV PATH=/var/lib/openstack/bin:$PATH
COPY --link --from=requirements /src/upper-constraints.txt /upper-constraints.txt
RUN <<EOF bash -xe
pip3 install \
--constraint /upper-constraints.txt \
cryptography \
pymysql \
python-binary-memcached \
python-memcached \
uwsgi
EOF

FROM ubuntu-cloud-archive AS openstack-runtime
RUN <<EOF bash -xe
apt-get update -qq
apt-get install -qq -y --no-install-recommends \
ca-certificates \
libpython3.10 \
lsb-release \
python3-distutils \
sudo
EOF
ARG PROJECT
ARG SHELL=/usr/sbin/nologin
RUN \
groupadd -g 42424 ${PROJECT} && \
useradd -u 42424 -g 42424 -M -d /var/lib/${PROJECT} -s ${SHELL} -c "${PROJECT} User" ${PROJECT} && \
mkdir -p /etc/${PROJECT} /var/log/${PROJECT} /var/lib/${PROJECT} /var/cache/${PROJECT} && \
chown -Rv ${PROJECT}:${PROJECT} /etc/${PROJECT} /var/log/${PROJECT} /var/lib/${PROJECT} /var/cache/${PROJECT}
ENV PATH=/var/lib/openstack/bin:$PATH

FROM alpine/git AS barbican-src
ARG BARBICAN_GIT_REF
ADD --keep-git-dir=true https://opendev.org/openstack/barbican.git#${BARBICAN_GIT_REF} /src
RUN git -C /src fetch --unshallow

FROM openstack-venv-builder AS barbican-build
COPY --from=barbican-src --link /src /src/barbican
RUN <<EOF bash -xe
pip3 install \
--constraint /upper-constraints.txt \
/src/barbican \
pykmip
EOF

FROM openstack-runtime AS barbican
COPY --from=barbican-build --link /var/lib/openstack /var/lib/openstack

FROM alpine/git AS magnum-src
ARG MAGNUM_GIT_REF
ADD --keep-git-dir=true https://opendev.org/openstack/magnum.git#${MAGNUM_GIT_REF} /src
RUN git -C /src fetch --unshallow
ARG RELEASE
COPY patches/${RELEASE}/magnum /patches
RUN if [ -n "$(ls -A /patches/*.patch)" ]; then git -C /src apply --verbose /patches/*; fi

FROM openstack-venv-builder AS magnum-build
COPY --from=magnum-src --link /src /src/magnum
RUN <<EOF bash -xe
pip3 install \
--constraint /upper-constraints.txt \
/src/magnum \
magnum-cluster-api==0.16.0
EOF

FROM openstack-runtime AS magnum
RUN <<EOF bash -xe
apt-get update -qq
apt-get install -qq -y --no-install-recommends \
haproxy
apt-get clean
rm -rf /var/lib/apt/lists/*
EOF
COPY --from=helm --link /usr/bin/helm /usr/local/bin/helm
COPY --from=magnum-build --link /var/lib/openstack /var/lib/openstack
# Copyright (c) 2024 VEXXHOST, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

FROM golang:1.21 AS go-builder
COPY go.mod go.sum /src/
WORKDIR /src
RUN go mod download

FROM go-builder AS libvirt-tls-sidecar-builder
COPY cmd/ /src/cmd/
COPY internal/ /src/internal/
RUN go build -o main ./cmd/libvirt-tls-sidecar/main.go

FROM registry.atmosphere.dev/library/ubuntu:zed AS libvirt-tls-sidecar
COPY --from=libvirt-tls-sidecar-builder /src/main /usr/bin/libvirt-tls-sidecar
ENTRYPOINT ["/usr/bin/libvirt-tls-sidecar"]
108 changes: 17 additions & 91 deletions Earthfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,47 +36,23 @@ unit.go:
SAVE ARTIFACT /src/junit-go.xml AS LOCAL junit-go.xml
END

builder:
FROM ubuntu:jammy
RUN apt-get update -qq
RUN \
apt-get install -qq -y --no-install-recommends \
build-essential git python3-dev python3-pip python3-venv
ARG POETRY_VERSION=1.4.2
RUN pip3 install --no-cache-dir poetry==${POETRY_VERSION}

build.collection:
FROM registry.gitlab.com/pipeline-components/ansible-lint:latest
COPY . /src
RUN ansible-galaxy collection build /src
SAVE ARTIFACT /code/*.tar.gz AS LOCAL dist/

go.build:
FROM golang:1.21
WORKDIR /src
ARG GOOS=linux
ARG GOARCH=amd64
ARG VARIANT
COPY --dir go.mod go.sum ./
RUN go mod download

libvirt-tls-sidecar.build:
FROM +go.build
ARG GOOS=linux
ARG GOARCH=amd64
ARG VARIANT
COPY --dir cmd internal ./
RUN GOARM=${VARIANT#"v"} go build -o main cmd/libvirt-tls-sidecar/main.go
SAVE ARTIFACT ./main

libvirt-tls-sidecar.platform-image:
ARG TARGETPLATFORM
ARG TARGETARCH
ARG TARGETVARIANT
FROM --platform=$TARGETPLATFORM ./images/base+image
COPY \
--platform=linux/amd64 \
(+libvirt-tls-sidecar.build/main --GOARCH=$TARGETARCH --VARIANT=$TARGETVARIANT) /usr/bin/libvirt-tls-sidecar
ENTRYPOINT ["/usr/bin/libvirt-tls-sidecar"]
ARG REGISTRY=ghcr.io/vexxhost/atmosphere
SAVE IMAGE --push ${REGISTRY}/libvirt-tls-sidecar:latest

libvirt-tls-sidecar.image:
BUILD --platform=linux/amd64 --platform=linux/arm64 +libvirt-tls-sidecar.platform-image

build.wheels:
FROM ./images/builder+image
FROM +builder
COPY pyproject.toml poetry.lock ./
ARG --required only
RUN poetry export --only=${only} -f requirements.txt --without-hashes > requirements.txt
Expand Down Expand Up @@ -114,71 +90,21 @@ build.collections:
SAVE IMAGE --cache-hint

image:
ARG RELEASE=2023.1
FROM ./images/cloud-archive-base+image --RELEASE ${RELEASE}
FROM ubuntu:jammy
ENV ANSIBLE_PIPELINING=True
DO ./images+APT_INSTALL --PACKAGES "rsync openssh-client"
RUN \
apt-get update -qq && \
apt-get install -qq -y --no-install-recommends \
rsync openssh-client && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
COPY +build.venv.runtime/venv /venv
ENV PATH=/venv/bin:$PATH
COPY +build.collections/ /usr/share/ansible
ARG tag=latest
ARG REGISTRY=ghcr.io/vexxhost/atmosphere
SAVE IMAGE --push ${REGISTRY}:${tag}

images:
ARG REGISTRY=ghcr.io/vexxhost/atmosphere
BUILD +libvirt-tls-sidecar.image --REGISTRY=${REGISTRY}
BUILD ./images/cinder+image --REGISTRY=${REGISTRY}
BUILD ./images/cluster-api-provider-openstack+image --REGISTRY=${REGISTRY}
BUILD ./images/designate+image --REGISTRY=${REGISTRY}
BUILD ./images/glance+image --REGISTRY=${REGISTRY}
BUILD ./images/heat+image --REGISTRY=${REGISTRY}
BUILD ./images/horizon+image --REGISTRY=${REGISTRY}
BUILD ./images/ironic+image --REGISTRY=${REGISTRY}
BUILD ./images/keystone+image --REGISTRY=${REGISTRY}
BUILD ./images/kubernetes-entrypoint+image --REGISTRY=${REGISTRY}
BUILD ./images/libvirtd+image --REGISTRY=${REGISTRY}
BUILD ./images/magnum+image --REGISTRY=${REGISTRY}
BUILD ./images/manila+image --REGISTRY=${REGISTRY}
BUILD ./images/netoffload+image --REGISTRY=${REGISTRY}
BUILD ./images/neutron+image --REGISTRY=${REGISTRY}
BUILD ./images/nova-ssh+image --REGISTRY=${REGISTRY}
BUILD ./images/nova+image --REGISTRY=${REGISTRY}
BUILD ./images/octavia+image --REGISTRY=${REGISTRY}
BUILD ./images/openvswitch+image --REGISTRY=${REGISTRY}
BUILD ./images/ovn+images --REGISTRY=${REGISTRY}
BUILD ./images/placement+image --REGISTRY=${REGISTRY}
BUILD ./images/senlin+image --REGISTRY=${REGISTRY}
BUILD ./images/staffeln+image --REGISTRY=${REGISTRY}
BUILD ./images/tempest+image --REGISTRY=${REGISTRY}

SCAN_IMAGE:
FUNCTION
ARG --required IMAGE
# TODO(mnaser): Include secret scanning when it's more reliable.
RUN \
trivy image \
--skip-db-update \
--skip-java-db-update \
--scanners vuln \
--exit-code 1 \
--ignore-unfixed \
--timeout 10m \
${IMAGE}

scan-image:
FROM ./images/trivy+image
ARG --required IMAGE
DO +SCAN_IMAGE --IMAGE ${IMAGE}

scan-images:
FROM ./images/trivy+image
COPY roles/defaults/vars/main.yml /defaults.yml
# TODO(mnaser): Scan all images eventually
FOR IMAGE IN $(cat /defaults.yml | egrep -E 'ghcr.io/vexxhost|registry.atmosphere.dev' | cut -d' ' -f4 | sort | uniq)
BUILD +scan-image --IMAGE ${IMAGE}
END

pin-images:
FROM +build.venv.dev
COPY roles/defaults/vars/main.yml /defaults.yml
Expand Down
Loading