ci(debug): debug docker build

scylladb · Feb 7, 2025 · f362162 · f362162
1 parent 926784d
commit f362162
Show file tree

Hide file tree

Showing 3 changed files with 137 additions and 43 deletions.
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -8,11 +8,10 @@ on:
         required: false
         type: string
     secrets:
-      DOCKERHUB_TOKEN:
+      DOCKERHUB_TOKEN_VP:
         required: true
-      DOCKERHUB_USERNAME:
+      DOCKERHUB_USERNAME_VP:
         required: true
-
   workflow_dispatch:
     inputs:
       version:
@@ -21,37 +20,126 @@ on:
         type: string
 
 jobs:
-  docker:
-    runs-on: ubuntu-24.04
+  build-with-matrix:
+    name: Build ${{ matrix.build.platform }}
+    runs-on: ubuntu-24.04${{ matrix.build.platform == 'linux/arm64/v8' && '-arm' || '' }}
+    strategy:
+      matrix:
+        # platform:
+        #   - linux/amd64
+        #   - linux/arm64/v8
+        build:
+          - platform: linux/amd64
+            cache: cache-amd64
+            arch_short: amd
+          - platform: linux/arm64/v8
+            cache: cache-arm64
+            arch_short: arm
     steps:
-      - name: Checkout
+      - name: Checkout Repository
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
-      # Needed for ARM64 Docker builds
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+      - name: Log in to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME_VP }}
+          password: ${{ secrets.DOCKERHUB_TOKEN_VP }}
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Login to dockerhub
+      - name: Enable BuildKit
+        run: echo "DOCKER_BUILDKIT=1" >> $GITHUB_ENV
+
+      # --cache-from=type=registry,ref=vponomarovatscylladb/hydra-loaders:${{ matrix.build.cache }} \
+      # --cache-to=type=registry,ref=vponomarovatscylladb/hydra-loaders:${{ matrix.build.cache }} \
+      # --load \
+      - name: Build and Cache Image for the ${{ matrix.build.platform }} platform
+        run: |
+          docker buildx build \
+            -t vponomarovatscylladb/hydra-loaders \
+            --platform ${{ matrix.build.platform }} \
+            --provenance false \
+            --load \
+            --output push-by-digest=true,type=image,push=true \
+            .
+
+      - name: Get Image Digest
+        run: |
+          docker images --digests
+          docker inspect --format='{{index .Id}}' $(docker images --digests --format "{{.ID}} {{ .Repository }}" | grep "hydra" | cut -d" " -f1)
+          CURRENT_SHA=$(docker inspect --format='{{index .Id}}' $(docker images --digests --format "{{.ID}} {{ .Repository }}" | grep "hydra" | cut -d" " -f1))
+          echo CURRENT_SHA=$CURRENT_SHA
+          echo $CURRENT_SHA > digest-${{ matrix.build.arch_short }}
+
+      - name: Upload Digest Artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: digest-${{ matrix.build.arch_short }}
+          path: digest-${{ matrix.build.arch_short }}
+
+  push-after-matrix:
+    name: Push multiarch images
+    needs: build-with-matrix
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Log in to DockerHub
         uses: docker/login-action@v3
         with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Build and push API
-        uses: docker/build-push-action@v6
+          username: ${{ secrets.DOCKERHUB_USERNAME_VP }}
+          password: ${{ secrets.DOCKERHUB_TOKEN_VP }}
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Enable BuildKit
+        run: echo "DOCKER_BUILDKIT=1" >> $GITHUB_ENV
+
+      - name: Download Digests
+        uses: actions/download-artifact@v4
         with:
-          file: ./Dockerfile
-          context: .
-          push: true
-          pull: true
-          platforms: "linux/amd64,linux/arm64/v8"
-          target: production
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-          tags: |
-            scylladb/latte:latest
-            scylladb/latte:${{ inputs.version }}
+          path: digests
+
+      - name: Create Multi-Arch Manifest and Push
+        run: |
+          SHA_AMD=$(cat digests/digest-amd*)
+          SHA_ARM=$(cat digests/digest-arm*)
+          docker manifest create vponomarovatscylladb/hydra-loaders:latte-latest \
+            --amend vponomarovatscylladb/hydra-loaders@$SHA_AMD \
+            --amend vponomarovatscylladb/hydra-loaders@$SHA_ARM
+          docker manifest push vponomarovatscylladb/hydra-loaders:latte-latest
+          docker manifest inspect vponomarovatscylladb/hydra-loaders:latte-latest
+
+          docker manifest create vponomarovatscylladb/hydra-loaders:latte-${{ inputs.version }} \
+            --amend vponomarovatscylladb/hydra-loaders@$SHA_AMD \
+            --amend vponomarovatscylladb/hydra-loaders@$SHA_ARM
+          docker manifest push vponomarovatscylladb/hydra-loaders:latte-${{ inputs.version }}
+          docker manifest inspect vponomarovatscylladb/hydra-loaders:latte-${{ inputs.version }}
+
+          # docker buildx imagetools create \
+          #   --tag vponomarovatscylladb/hydra-loaders:latte-latest \
+          #   vponomarovatscylladb/hydra-loaders:cache-amd64 \
+          #   vponomarovatscylladb/hydra-loaders:cache-arm64
+          # docker buildx imagetools create \
+          #   --tag vponomarovatscylladb/hydra-loaders:latte-${{ inputs.version }} \
+          #   vponomarovatscylladb/hydra-loaders:cache-amd64 \
+          #   vponomarovatscylladb/hydra-loaders:cache-arm64
+          # docker buildx build \
+          #   --cache-from=type=registry,ref=vponomarovatscylladb/hydra-loaders:cache-amd64 \
+          #   --cache-from=type=registry,ref=vponomarovatscylladb/hydra-loaders:cache-arm64 \
+          #   --platform linux/amd64,linux/arm64/v8 \
+          #   --output=type=image \
+          #   -t vponomarovatscylladb/hydra-loaders:latte-latest \
+          #   --push .
+          # docker buildx build \
+          #   --cache-from=type=registry,ref=vponomarovatscylladb/hydra-loaders:cache-amd64 \
+          #   --cache-from=type=registry,ref=vponomarovatscylladb/hydra-loaders:cache-arm64 \
+          #   --platform linux/amd64,linux/arm64/v8 \
+          #   --output=type=image \
+          #   -t vponomarovatscylladb/hydra-loaders:latte-${{ inputs.version }} \
+          #   --push .
diff --git a/.github/workflows/dockerhub-description.yml b/.github/workflows/dockerhub-description.yml
@@ -16,8 +16,8 @@ jobs:
     - name: Docker Hub Description
       uses: peter-evans/dockerhub-description@v4
       with:
-        username: ${{ secrets.DOCKERHUB_USERNAME }}
-        password:  ${{ secrets.DOCKERHUB_TOKEN }}
-        repository: scylladb/latte
+        username: ${{ secrets.DOCKERHUB_USERNAME_VP }}
+        password:  ${{ secrets.DOCKERHUB_TOKEN_VP }}
+        repository: vponomarovatscylladb/hydra-loaders
         short-description: ${{ github.event.repository.description }}
         enable-url-completion: true
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -4,11 +4,15 @@ on:
   push:
     tags:
       - '*'
+    branches:
+      - debug
+
 jobs:
   get_tag:
     runs-on: ubuntu-latest
     outputs:
-      tag: ${{ steps.version_tag.outputs.tag }}
+      # tag: ${{ steps.version_tag.outputs.tag }}
+      tag: "debug"
     steps:
       - uses: actions/checkout@v4
         with:
@@ -25,8 +29,8 @@ jobs:
     with:
       version: ${{ needs.get_tag.outputs.tag }}
     secrets:
-      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
-      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+      DOCKERHUB_USERNAME_VP: ${{ secrets.DOCKERHUB_USERNAME_VP }}
+      DOCKERHUB_TOKEN_VP: ${{ secrets.DOCKERHUB_TOKEN_VP }}
 
   release:
     needs: [get_tag, docker_build]
@@ -45,24 +49,26 @@ jobs:
           command: build
           args: --release
 
-      - name: Create Release
-        id: create_release
-        uses: actions/create-release@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          tag_name: v${{ needs.get_tag.outputs.tag }}
-          release_name: Release ${{ github.ref }}
-          draft: false
-          prerelease: false
+      # NOTE: here we depend on the fact that user created release with automatic tag creation.
+      # So, 'tag' and 'release' must exist at this step
+      # - name: Create Release
+      #   id: create_release
+      #   uses: actions/create-release@v1
+      #   env:
+      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      #   with:
+      #     tag_name: v${{ needs.get_tag.outputs.tag }}
+      #     release_name: Release ${{ github.ref }}
+      #     draft: false
+      #     prerelease: false
 
       - name: Upload Release Asset
         id: upload-release-asset-unix
         uses: actions/upload-release-asset@v1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          upload_url: ${{ github.event.release.upload_url }}
           asset_path: ./target/release/latte
-          asset_name: latte-${{ matrix.os }}
+          asset_name: latte-${{ needs.get_tag.outputs.tag }}--${{ matrix.os }}
           asset_content_type: application/octet-stream