Add caching for Dockerfile build (#94)

* Add caching for Dockerfile build * Add caching for Dockerfile build * Add caching for Dockerfile build * Add caching for Dockerfile build * Add caching for Dockerfile build * Add caching for Dockerfile build * Add caching for Dockerfile build * Add caching for Dockerfile build * Add caching for Dockerfile build * no podman * gawk * env? * env? * env? * path? * path? * grab write all? * grab write all? * grab write all? * grab write all? * grab write all? * grab write all? * grab write all? * dockar * dockar * dockar * dockar * dockar * dockar * dockar * dockar * dockar * dockar * dockar * dockar * dockar * cache dir * ccache dir again? * fix ccache dir name in container * change cache dir * update dockerfile
rh-messaging · May 5, 2023 · 180d788 · 180d788
1 parent 53dfc76
commit 180d788
Show file tree

Hide file tree

Showing 3 changed files with 75 additions and 12 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -2,4 +2,5 @@
 .idea
 /bazel-*
 /cmake-*
+/qpid-proton
 Dockerfile
diff --git a/.github/workflows/image_build.yaml b/.github/workflows/image_build.yaml
@@ -6,20 +6,34 @@ on:
   workflow_dispatch:
     branches: [ main ]
 
-permissions: read-all
+permissions:
+  packages: write
 
 env:
-  IMAGE_NAME: cli-cpp
   IMAGE_REGISTRY: quay.io
   IMAGE_NAMESPACE: rhmessagingqe
+  IMAGE_NAME: cli-cpp
+  # https://github.com/actions/cache/issues/814
+  CCACHE_DIR: /ccache
 
 jobs:
   build:
     name: Build and push image
     runs-on: ubuntu-22.04
+    # https://github.com/redhat-actions/buildah-build/issues/90#issuecomment-1500807385
+    container:
+      image: quay.io/buildah/stable:latest
+      options: --privileged
 
     steps:
+
       - uses: actions/checkout@v3
+        with:
+          path: cli-cpp
+
+      - name: Install dependencies
+        run: |
+          sudo dnf install -y perl podman qemu-user-static gawk
 
       # https://www.integralist.co.uk/posts/github-actions/
       - name: Prepare ref name
@@ -28,8 +42,34 @@ jobs:
           ref_name=$(echo ${{ github.ref_name }} | perl -pe 's/[^a-zA-Z0-9]+/-/g' | perl -pe 's/(\A-|-\Z)//g' | awk '{print tolower($0)}')
           echo "ref_name=${ref_name}" >> $GITHUB_OUTPUT
 
+      - name: Initialize ccache
+        run: mkdir -p "${{ env.CCACHE_DIR }}"
+
+      - name: Log in to ghcr.io
+        run: echo "${{ secrets.GITHUB_TOKEN }}" | buildah login ghcr.io --username="${{ github.actor }}" --password-stdin
+
+      # https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry
+      - name: Prepare registry cache vars
+        id: cache
+        run: |
+          echo "cache-from=--cache-from=ghcr.io/${{ github.repository_owner }}/cli-cpp/buildah-cache" >> $GITHUB_OUTPUT
+          # don't pollute cache by output from pull request builds
+          if [[ ${{ github.ref }} == 'refs/heads/main' ]]; then
+            echo "cache-to=--cache-to=ghcr.io/${{ github.repository_owner }}/cli-cpp/buildah-cache" >> $GITHUB_OUTPUT
+          fi
+
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        run: podman run --privileged --rm docker.io/tonistiigi/binfmt --install all
+
+      - name: Cache CCACHE_DIR (restore)
+        uses: actions/cache/restore@v3
+        id: restore-ccache-cache
+        with:
+          path: "${{ env.CCACHE_DIR }}"
+          key: ${{ runner.os }}-ccache-${{ matrix.os }}-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-ccache-${{ matrix.os }}-
+            ${{ runner.os }}-ccache-
 
       - name: Build Image
         id: build-image
@@ -38,8 +78,19 @@ jobs:
           image: ${{ env.IMAGE_NAME }}
           tags: latest ${{ github.sha }} ${{ steps.cleaned_ref_name.outputs.ref_name }}
           archs: amd64, arm64, arm64, s390x
-          containerfiles: |
-            ./Dockerfile
+          layers: true
+          extra-args: |
+            --volume=${{ env.CCACHE_DIR }}:/ccache
+            ${{ steps.cache.outputs.cache-from }}
+            ${{ steps.cache.outputs.cache-to }}
+          context: cli-cpp
+          containerfiles: cli-cpp/Dockerfile
+
+      - name: Cache CCACHE_DIR (save)
+        uses: actions/cache/save@v3
+        with:
+          path: "${{ env.CCACHE_DIR }}"
+          key: ${{ steps.restore-ccache-cache.outputs.cache-primary-key }}
 
       - name: Push To quay.io
         if: github.ref == 'refs/heads/main'

diff --git a/Dockerfile b/Dockerfile
@@ -5,6 +5,8 @@ ARG UBI_RUNTIME_TAG=latest
 ARG IMAGE_BUILD=registry.access.redhat.com/ubi${UBI_VERSION}/ubi-minimal:${UBI_TAG}
 ARG IMAGE_BASE=registry.access.redhat.com/ubi${UBI_VERSION}/ubi-minimal:${UBI_RUNTIME_TAG}
 
+# Build this with something like buildah bud --arch arm64 --volume=/tmp/ccache:/ccache
+
 #DEV FROM $IMAGE_BUILD
 FROM quay.io/centos/centos:stream9 as build
 
@@ -16,7 +18,7 @@ RUN /usr/bin/crb enable
 RUN curl -L https://copr.fedorainfracloud.org/coprs/kpvdr/opentelemetry-cpp/repo/epel-9/kpvdr-opentelemetry-cpp-epel-9.repo > /etc/yum.repos.d/kpvdr-opentelemetry-cpp-epel-9.repo
 
 RUN dnf -y --setopt=install_weak_deps=0 --setopt=tsflags=nodocs install \
-    findutils git patchelf \
+    ccache findutils git patchelf \
     \
     cmake ninja-build \
     gcc gcc-c++ \
@@ -29,24 +31,33 @@ RUN dnf -y --setopt=install_weak_deps=0 --setopt=tsflags=nodocs install \
 COPY . /src
 WORKDIR /src
 
+# can't put $(arch) to CCACHE_DIR, https://github.com/moby/moby/issues/29110
+# ENV CCACHE_DIR=
+ENV CCACHE_COMPRESS=true
+ENV CCACHE_MAXSIZE=400MB
+
 RUN git clone --depth=1 https://github.com/apache/qpid-proton.git
-RUN cmake -S qpid-proton -B cmake-build-qpid-proton -GNinja \
+RUN CCACHE_DIR=/ccache/$(arch) cmake -S qpid-proton -B cmake-build-qpid-proton -GNinja \
+    -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+    -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
     -DCMAKE_INSTALL_PREFIX=cmake-install \
     -DBUILD_BINDINGS=cpp \
     -DBUILD_EXAMPLES=OFF \
     -DBUILD_TESTING=OFF \
     -DSASL_IMPL=cyrus \
     -DSSL_IMPL=openssl \
     -DPROACTOR=epoll
-RUN cmake --build cmake-build-qpid-proton
-RUN cmake --install cmake-build-qpid-proton --config RelWithDebInfo
+RUN CCACHE_DIR=/ccache/$(arch) cmake --build cmake-build-qpid-proton
+RUN CCACHE_DIR=/ccache/$(arch) cmake --install cmake-build-qpid-proton --config RelWithDebInfo
 
-RUN cmake -S . -B cmake-build-cli-cpp -GNinja \
+RUN CCACHE_DIR=/ccache/$(arch) cmake -S . -B cmake-build-cli-cpp -GNinja \
+    -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+    -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
     -DCMAKE_INSTALL_PREFIX=cmake-install
     #-DCMAKE_BUILD_WITH_INSTALL_RPATH=TRUE \
     #-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=TRUE
-RUN cmake --build cmake-build-cli-cpp
-RUN cmake --install cmake-build-cli-cpp --config RelWithDebInfo
+RUN CCACHE_DIR=/ccache/$(arch) cmake --build cmake-build-cli-cpp
+RUN CCACHE_DIR=/ccache/$(arch) cmake --install cmake-build-cli-cpp --config RelWithDebInfo
 
 # fixup finding the proton library, this is something that our build should handle by itself, though
 RUN for i in cmake-install/bin/*; do patchelf --add-rpath /usr/local/lib64 $i; done