From 6694f0a1266f3cf6cdd9e1a4040e5fd106d97efc Mon Sep 17 00:00:00 2001 From: Jordan Dubrick Date: Tue, 21 May 2024 12:10:08 -0400 Subject: [PATCH] Multi-Arch Support for oci-registry builds and CI (#240) * add multi-arch build support for build.sh script & update doc for new functionality Signed-off-by: Jordan Dubrick * test new workflow for buildx Signed-off-by: Jordan Dubrick * add multi arch script Signed-off-by: Jordan Dubrick * add base repository Signed-off-by: Jordan Dubrick * update ci to test both amd64 and arm64 image builds Signed-off-by: Jordan Dubrick * add multi arch building stage for oci registry Signed-off-by: Jordan Dubrick * add qemu for arm64 image building Signed-off-by: Jordan Dubrick * add base tag for image Signed-off-by: Jordan Dubrick * fix podman condition Signed-off-by: Jordan Dubrick * pin login action to hash Signed-off-by: Jordan Dubrick * make default manifest image a var Signed-off-by: Jordan Dubrick * add troubleshooting section Signed-off-by: Jordan Dubrick --------- Signed-off-by: Jordan Dubrick --- .github/workflows/ci.yaml | 5 +- .github/workflows/pushimage-next.yaml | 15 +++-- oci-registry/README.md | 49 +++++++++++++++- oci-registry/build-multi-arch.sh | 81 +++++++++++++++++++++++++++ oci-registry/build.sh | 13 ++++- 5 files changed, 151 insertions(+), 12 deletions(-) create mode 100644 oci-registry/build-multi-arch.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6f7974fbf..9402e69db 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -88,6 +88,9 @@ jobs: uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0 with: go-version: ${{ env.GO_VERSION }} + + - name: Set up QEMU # Enables arm64 image building + uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 #v3.0.0 - name: Check license run: | @@ -113,7 +116,7 @@ jobs: run: cd index/server && go test ./... -coverprofile cover.out - name: Check if oci server build is working - run: cd oci-registry && bash ./build.sh + run: cd oci-registry && bash ./build.sh && bash ./build.sh linux/arm64 - name: Check if devfile-registry-integration build is working run: cd tests/integration && bash ./docker-build.sh diff --git a/.github/workflows/pushimage-next.yaml b/.github/workflows/pushimage-next.yaml index 3d6a91821..b943ab65a 100644 --- a/.github/workflows/pushimage-next.yaml +++ b/.github/workflows/pushimage-next.yaml @@ -89,17 +89,16 @@ jobs: steps: - name: Check out registry support source code uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - - name: Build and push oci-registry docker image - uses: docker/build-push-action@v1.1.0 + - name: Set up QEMU # Enables arm64 image building + uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 #v3.0.0 + - name: Login to Quay.io + uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2.2.0 with: - path: ./oci-registry + registry: quay.io username: ${{ secrets.QUAY_USERNAME }} password: ${{ secrets.QUAY_PASSWORD }} - registry: quay.io - repository: devfile/oci-registry - dockerfile: ./oci-registry/Dockerfile - tags: next - tag_with_sha: true + - name: Build and push oci-registry docker image + run: bash ./oci-registry/build-multi-arch.sh devfileRegistryIntegrationBuild: runs-on: ubuntu-latest diff --git a/oci-registry/README.md b/oci-registry/README.md index 064b99cdc..44a74ea20 100644 --- a/oci-registry/README.md +++ b/oci-registry/README.md @@ -5,7 +5,7 @@ This folder contains the Dockerfile for the OCI registry server. It is based off ## Build The scripts in this project support both `Docker` and `Podman` container engines. By default the scripts will run using `Docker`, to use `Podman` first run `export USE_PODMAN=true`. -To build the image, run `bash build.sh`. +The build script enables users to build for different architectures, by default running `bash build.sh` will build for `linux/amd64`. If you would like to build for a different architecture simply add it as an argument to the script. E.g. `bash build.sh linux/arm64` for `linux/arm64` builds. To push the image to a repository of your choice, you can run `bash push.sh `. @@ -16,4 +16,49 @@ To deploy this image as part of a Devfile registry: 1. Build and push this image to an image registry. 2. Install the [Devfile Registry Operator](https://github.com/devfile/registry-operator) on a Kubernetes cluster. 3. Create a `DevfileRegistry` yaml file and set `spec.ociRegistryImage` to the name of your pushed image from the previous step. -4. Run `kubectl apply -f ` \ No newline at end of file +4. Run `kubectl apply -f ` + +## Troubleshooting + +If you are trying to run `build-multi-arch.sh`, or you are trying to run `build.sh` for an architecture your machine is **not running** on, you may encounter an error similiar to this in your build: +``` +Dockerfile:18 +-------------------- + 16 | + 17 | FROM registry.access.redhat.com/ubi8-minimal:8.2 + 18 | >>> RUN microdnf update -y && rm -rf /var/cache/yum && microdnf install ca-certificates httpd-tools + 19 | + 20 | # Create a non-root user to run the server as +-------------------- +ERROR: failed to solve: process "/bin/sh -c microdnf update -y && rm -rf /var/cache/yum && microdnf install ca-certificates httpd-tools" did not complete successfully: exit code: 1 +``` + +This error can occur because your container engine is not properly supporting emulation. To check if this is the case you can run: +``` +podman run --rm --privileged tonistiigi/binfmt +``` +or if you are using docker: +``` +docker run --rm --privileged tonistiigi/binfmt +``` + +This command will output something similar to the following where you can observe what architectures are currently supported for you: +``` +"supported": [ + "linux/arm64", + "linux/amd64", + "linux/riscv64", + "linux/ppc64le", + "linux/s390x", + "linux/386", + "linux/mips64le", + "linux/mips64" + ] +``` + +If you do not see the architecture you are trying to build for in the output list you are probably missing emulation capabilities. The following command has been known to fix this issue and allow for emulation: +``` +sudo apt-get install -y gcc-arm-linux-gnueabihf libc6-dev-armhf-cross qemu-user-static qemu-system-i386 +``` + +The CI environments we call these scripts in are incorporating the use of QEMU. If the above command does not solve the issue for you it would be best to investigate into if QEMU is working properly in your environment. \ No newline at end of file diff --git a/oci-registry/build-multi-arch.sh b/oci-registry/build-multi-arch.sh new file mode 100644 index 000000000..f8857f4e8 --- /dev/null +++ b/oci-registry/build-multi-arch.sh @@ -0,0 +1,81 @@ +#!/bin/sh + +# +# Copyright Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +buildfolder="$(basename "$(dirname "$0")")" +# Due to command differences between podman and docker we need to separate the process +# for creating and adding images to a multi-arch manifest +podman=${USE_PODMAN:-false} +# Stores all created image tags +images=() +# Base Repository +BASE_REPO="quay.io/devfile/oci-registry" +BASE_TAG="next" +DEFAULT_MANIFEST="$BASE_REPO:$BASE_TAG" + +function build { + IMAGE="$BASE_REPO:$2" + + echo "Building: ${IMAGE}" + $1 build -t $IMAGE --platform "linux/$2" "$buildfolder" + + echo "Tagging: ${IMAGE}" + $1 tag "$IMAGE" "$IMAGE" + + echo "Pushing: ${IMAGE}" + $1 push "$IMAGE" + + # Add image to list of all images to be added to a manifest + images+=("${IMAGE}") +} + +function engine-handler { + for arch in amd64 arm64 ; do + build "$1" "$arch" + done +} + + +if [ ${podman} == true ]; then + echo "Executing with podman" + + # Build and push multi-arch images + engine-handler podman + + # Create manifest and add images + podman manifest create oci-registry-manifest + for img in "${images[@]}" ; do + podman manifest add oci-registry-manifest "$img" + done + + # Push and delete local manifest + podman manifest push oci-registry-manifest "$DEFAULT_MANIFEST" + podman manifest rm oci-registry-manifest + +else + echo "Executing with docker" + + # Build and push multi-arch images + engine-handler docker + + # Create manifest and add images + docker manifest create "$DEFAULT_MANIFEST" "${images[@]}" + + # Push and delete local manifest + docker manifest push "$DEFAULT_MANIFEST" + docker manifest rm "$$DEFAULT_MANIFEST" + +fi \ No newline at end of file diff --git a/oci-registry/build.sh b/oci-registry/build.sh index d5759b1fa..ca8f7eeb0 100755 --- a/oci-registry/build.sh +++ b/oci-registry/build.sh @@ -18,7 +18,18 @@ # Build the index container for the registry buildfolder="$(basename "$(dirname "$0")")" +DEFAULT_ARCH="linux/amd64" + +# Check if different architecture was passed for image build +# Will default to $DEFAULT_ARCH if unset +if [ ! -z "$1" ] + then + arch="$1" +else + arch="$DEFAULT_ARCH" +fi + # set podman alias if necessary . ${buildfolder}/../setenv.sh -docker build -t oci-registry:next $buildfolder +docker build -t oci-registry:next --platform "${arch}" "$buildfolder"