From 6a7c255550bc2a3260b8935f787cd8b0b86114d0 Mon Sep 17 00:00:00 2001 From: Jordan Dubrick Date: Thu, 30 May 2024 10:09:00 -0400 Subject: [PATCH 1/9] update projects to use multi arch build Signed-off-by: Jordan Dubrick --- apps/landing-page/project.json | 3 ++- apps/registry-viewer/project.json | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/apps/landing-page/project.json b/apps/landing-page/project.json index bbe95a5a..a8a4c4c9 100644 --- a/apps/landing-page/project.json +++ b/apps/landing-page/project.json @@ -61,8 +61,9 @@ }, "docker-build": { "executor": "nx:run-commands", + "description": "Build a multi-architecture image", "options": { - "command": "docker build -t nextjs-docker . --build-arg PROJECT_NAME=landing-page" + "command": "docker buildx create --name landing-builder && docker buildx use landing-builder && docker buildx build --platform=linux/amd64,linux/arm64 -t nextjs-docker . --build-arg PROJECT_NAME=landing-page && docker buildx rm landing-builder" } }, "test": { diff --git a/apps/registry-viewer/project.json b/apps/registry-viewer/project.json index 09ec22d2..77ca5ee6 100644 --- a/apps/registry-viewer/project.json +++ b/apps/registry-viewer/project.json @@ -58,10 +58,11 @@ "command": "cd ./apps/registry-viewer/ && next-sitemap --config ./next-sitemap.config.mjs && cp -a ./dist/public/. ./dist/exported" } }, - "docker-build": { + "docker-build": { "executor": "nx:run-commands", + "description": "Build a multi-architecture image", "options": { - "command": "docker build -t nextjs-docker . --build-arg PROJECT_NAME=registry-viewer" + "command": "docker buildx create --name viewer-builder && docker buildx use viewer-builder && docker buildx build --platform=linux/amd64,linux/arm64 -t nextjs-docker . --build-arg PROJECT_NAME=registry-viewer && docker buildx rm viewer-builder" } }, "test": { From a85bec1b9de99a0ad238b2b3f18d388dff98926f Mon Sep 17 00:00:00 2001 From: Jordan Dubrick Date: Thu, 30 May 2024 10:09:48 -0400 Subject: [PATCH 2/9] update dockerfile to customize timeout value Signed-off-by: Jordan Dubrick --- Dockerfile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8223272a..66009a25 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,6 +33,12 @@ ARG NEXT_PUBLIC_DOCSEARCH_APP_ID ARG NEXT_PUBLIC_DOCSEARCH_API_KEY ARG NEXT_PUBLIC_DOCSEARCH_INDEX_NAME +# Building different architectures via emulation is slow with Yarn +# This increases the timeout period so it can properly download dependencies +# Value is in milliseconds and is set to 60 minutes +# To increase/decrease you can override via --build-arg YARN_TIMEOUT=x in your build command +ARG YARN_TIMEOUT=3600000 + # Check if the PROJECT_NAME build argument is set RUN \ if [ "$PROJECT_NAME" == "landing-page" ] || [ "$PROJECT_NAME" == "registry-viewer" ]; then echo "Building project \"${PROJECT_NAME}\"."; \ @@ -43,7 +49,7 @@ WORKDIR /app # Install dependencies COPY package.json yarn.lock* ./ RUN \ - if [ -f yarn.lock ]; then yarn --frozen-lockfile; \ + if [ -f yarn.lock ]; then yarn --frozen-lockfile --network-timeout $YARN_TIMEOUT; \ else echo "Lockfile not found." && exit 1; \ fi From b48c1fd2de67b5e0e40416d937674441607d0e59 Mon Sep 17 00:00:00 2001 From: Jordan Dubrick Date: Thu, 30 May 2024 10:14:58 -0400 Subject: [PATCH 3/9] add new script for building/pushing multi arch registry viewer Signed-off-by: Jordan Dubrick --- scripts/build_multi_arch.sh | 59 +++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 scripts/build_multi_arch.sh diff --git a/scripts/build_multi_arch.sh b/scripts/build_multi_arch.sh new file mode 100644 index 00000000..053e58e5 --- /dev/null +++ b/scripts/build_multi_arch.sh @@ -0,0 +1,59 @@ +#!/bin/sh + +# +# Copyright Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ABSOLUTE_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BUILD_DIR=$ABSOLUTE_PATH/.. +# Due to command differences between podman and docker we need to separate the process +# for creating and adding images to a multi-arch manifest +podman=${USE_PODMAN:-false} +# Base Repository +BASE_REPO="quay.io/devfile/registry-viewer" +BASE_TAG="next" +DEFAULT_IMG="$BASE_REPO:$BASE_TAG" +# Platforms to build for +PLATFORMS="linux/amd64,linux/arm64" + +if [ ${podman} == true ]; then + echo "Executing with podman" + + podman manifest create "$DEFAULT_IMG" + + podman build --platform="$PLATFORMS" --manifest "$DEFAULT_IMG" "$BUILD_DIR" \ + --no-cache \ + --build-arg PROJECT_NAME=registry-viewer \ + --build-arg NEXT_PUBLIC_BASE_PATH=${NEXT_PUBLIC_BASE_PATH:-"/viewer"} + + podman manifest push "$DEFAULT_IMG" + + podman manifest rm "$DEFAULT_IMG" + +else + echo "Executing with docker" + + docker buildx create --name registry-viewer-builder + + docker buildx use registry-viewer-builder + + docker buildx build --push --platform="$PLATFORMS" --tag "$DEFAULT_IMG" "$BUILD_DIR" \ + --no-cache \ + --provenance=false \ + --build-arg PROJECT_NAME=registry-viewer \ + --build-arg NEXT_PUBLIC_BASE_PATH=${NEXT_PUBLIC_BASE_PATH:-"/viewer"} + + docker buildx rm registry-viewer-builder + +fi \ No newline at end of file From 443094f5fb47d182b8da4ca371c8f2fd87b33e42 Mon Sep 17 00:00:00 2001 From: Jordan Dubrick Date: Thu, 30 May 2024 10:16:00 -0400 Subject: [PATCH 4/9] free disk space and add qemu emulator Signed-off-by: Jordan Dubrick --- .github/workflows/ci.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index df398e94..47a867d9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -99,6 +99,20 @@ jobs: uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 with: fetch-depth: 0 + + - name: Free Disk Space + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be #v1.3.1 + with: + tool-cache: false + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: true + swap-storage: true + + - name: Set up QEMU # Enables arm64 image building + uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 #v3.0.0 - uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 #v4.0.0 with: From ed16040feed7fc895ea13890182e5e5731fb5e56 Mon Sep 17 00:00:00 2001 From: Jordan Dubrick Date: Thu, 30 May 2024 10:17:06 -0400 Subject: [PATCH 5/9] use new build/push script and add qemu emulator Signed-off-by: Jordan Dubrick --- .github/workflows/pushimage-next.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pushimage-next.yml b/.github/workflows/pushimage-next.yml index d4c59bfc..2f03d101 100644 --- a/.github/workflows/pushimage-next.yml +++ b/.github/workflows/pushimage-next.yml @@ -31,16 +31,16 @@ jobs: steps: - name: Check out devfile web source code uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + - name: Set up QEMU # Enables arm64 image building + uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 #v3.0.0 - name: Login to Quay uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2.2.0 with: registry: quay.io username: ${{ secrets.QUAY_USERNAME }} password: ${{ secrets.QUAY_PASSWORD }} - - name: Build the registry viewer image - run: bash ./scripts/build_viewer.sh - - name: Push the registry viewer image - run: bash ./scripts/push.sh registry-viewer:latest quay.io/devfile/registry-viewer:next + - name: Build and push the registry viewer image + run: bash ./scripts/build_multi_arch.sh dispatch: needs: registry-viewer-build strategy: From c04cf6fa2753ab4955765ad68e3aadaeb887788b Mon Sep 17 00:00:00 2001 From: Jordan Dubrick Date: Thu, 30 May 2024 14:46:34 -0400 Subject: [PATCH 6/9] add buildx setup to actions Signed-off-by: Jordan Dubrick --- .github/workflows/ci.yaml | 3 +++ .github/workflows/pushimage-next.yml | 2 ++ 2 files changed, 5 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 47a867d9..e8a54768 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -114,6 +114,9 @@ jobs: - name: Set up QEMU # Enables arm64 image building uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 #v3.0.0 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 #v4.0.0 with: node-version-file: 'package.json' diff --git a/.github/workflows/pushimage-next.yml b/.github/workflows/pushimage-next.yml index 2f03d101..2f9a3f3b 100644 --- a/.github/workflows/pushimage-next.yml +++ b/.github/workflows/pushimage-next.yml @@ -33,6 +33,8 @@ jobs: uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - name: Set up QEMU # Enables arm64 image building uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 #v3.0.0 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - name: Login to Quay uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2.2.0 with: From 8a80f1756fe98172f6c79ff8b429105d0127690f Mon Sep 17 00:00:00 2001 From: Jordan Dubrick Date: Thu, 30 May 2024 14:49:59 -0400 Subject: [PATCH 7/9] pin sha for buildx setup Signed-off-by: Jordan Dubrick --- .github/workflows/ci.yaml | 2 +- .github/workflows/pushimage-next.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e8a54768..a331c896 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -115,7 +115,7 @@ jobs: uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 #v3.0.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb #v3.3.0 - uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 #v4.0.0 with: diff --git a/.github/workflows/pushimage-next.yml b/.github/workflows/pushimage-next.yml index 2f9a3f3b..90ecebb7 100644 --- a/.github/workflows/pushimage-next.yml +++ b/.github/workflows/pushimage-next.yml @@ -34,7 +34,7 @@ jobs: - name: Set up QEMU # Enables arm64 image building uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 #v3.0.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb #v3.3.0 - name: Login to Quay uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2.2.0 with: From fedf3f431231b47c0a576861bd611303ccc013e7 Mon Sep 17 00:00:00 2001 From: Jordan Dubrick Date: Thu, 30 May 2024 16:24:59 -0400 Subject: [PATCH 8/9] increase parallel to 4 Signed-off-by: Jordan Dubrick --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a331c896..4f72bbb0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -130,7 +130,7 @@ jobs: - name: Docker Build run: | - yarn nx affected --target=docker-build --parallel=3 + yarn nx affected --target=docker-build --parallel=4 e2e: name: E2E From c8ef2d3931500add19ce9ccd561f7207aa890991 Mon Sep 17 00:00:00 2001 From: Jordan Dubrick Date: Fri, 31 May 2024 11:08:55 -0400 Subject: [PATCH 9/9] break nx command into 2 separate commands Signed-off-by: Jordan Dubrick --- .github/workflows/ci.yaml | 9 +++++++-- apps/landing-page/project.json | 13 ++++++++++--- apps/registry-viewer/project.json | 13 ++++++++++--- 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4f72bbb0..97144189 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -128,9 +128,14 @@ jobs: run: | yarn install --frozen-lockfile - - name: Docker Build + - name: Docker Build for linux/amd64 run: | - yarn nx affected --target=docker-build --parallel=4 + yarn nx affected --target=docker-build-amd64 --parallel=3 + + - name: Docker Build for linux/arm64 + run: | + yarn nx affected --target=docker-build-arm64 --parallel=3 + e2e: name: E2E diff --git a/apps/landing-page/project.json b/apps/landing-page/project.json index a8a4c4c9..a3f64bc2 100644 --- a/apps/landing-page/project.json +++ b/apps/landing-page/project.json @@ -59,11 +59,18 @@ "command": "cd ./apps/landing-page/ && next-sitemap --config ./next-sitemap.config.mjs && cp -a ./dist/public/. ./dist/exported" } }, - "docker-build": { + "docker-build-amd64": { "executor": "nx:run-commands", - "description": "Build a multi-architecture image", + "description": "Build docker image for the linux/amd64 architecture", "options": { - "command": "docker buildx create --name landing-builder && docker buildx use landing-builder && docker buildx build --platform=linux/amd64,linux/arm64 -t nextjs-docker . --build-arg PROJECT_NAME=landing-page && docker buildx rm landing-builder" + "command": "docker build --platform=linux/amd64 -t nextjs-docker-amd64 . --build-arg PROJECT_NAME=landing-page" + } + }, + "docker-build-arm64": { + "executor": "nx:run-commands", + "description": "Build docker image for the linux/arm64 architecture", + "options": { + "command": "docker build --platform=linux/arm64 -t nextjs-docker-arm64 . --build-arg PROJECT_NAME=landing-page" } }, "test": { diff --git a/apps/registry-viewer/project.json b/apps/registry-viewer/project.json index 77ca5ee6..a08de6c4 100644 --- a/apps/registry-viewer/project.json +++ b/apps/registry-viewer/project.json @@ -58,11 +58,18 @@ "command": "cd ./apps/registry-viewer/ && next-sitemap --config ./next-sitemap.config.mjs && cp -a ./dist/public/. ./dist/exported" } }, - "docker-build": { + "docker-build-amd64": { "executor": "nx:run-commands", - "description": "Build a multi-architecture image", + "description": "Build docker image for the linux/amd64 architecture", "options": { - "command": "docker buildx create --name viewer-builder && docker buildx use viewer-builder && docker buildx build --platform=linux/amd64,linux/arm64 -t nextjs-docker . --build-arg PROJECT_NAME=registry-viewer && docker buildx rm viewer-builder" + "command": "docker build --platform=linux/amd64 -t nextjs-docker-amd64 . --build-arg PROJECT_NAME=registry-viewer" + } + }, + "docker-build-arm64": { + "executor": "nx:run-commands", + "description": "Build docker image for the linux/arm64 architecture", + "options": { + "command": "docker build --platform=linux/arm64 -t nextjs-docker-arm64 . --build-arg PROJECT_NAME=registry-viewer" } }, "test": {