[HYD-62] Improve build process to use bake

.github/workflows/ci.yaml

@@ -18,10 +18,11 @@ jobs:
         ports:
           - 5000:5000
     steps:
+      # Checkout
       # Note: the following folder layout matters:
       # parent_dir:
       #   - hydra
-      #   - Hydras
+      #   - hydra-extension
       #   - citus
       #   - spilo
       - name: Checkout hydra
@@ -47,6 +48,8 @@ jobs:
           token: ${{ secrets.GH_TOKEN }}
           ref: 2.1-p6
           path: spilo
+
+      # Set versions
       - name: Set versions
         run: |
           cd ./hydra-extension && HYDRA_EXT_SHA=$(git rev-parse --short HEAD) && echo "HYDRA_EXT_SHA=${HYDRA_EXT_SHA}" >> $GITHUB_ENV && cd ../
@@ -59,61 +62,17 @@ jobs:
         uses: docker/setup-buildx-action@v2
         with:
           driver-opts: network=host
-      - name: Docker Build Hydra Extension
-        uses: docker/build-push-action@v3
-        with:
-          context: ./hydra-extension
-          push: true
-          tags: localhost:5000/hydra/hydra-extension:${{ env.TAG }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-      - name: Docker Build Columnar Extension
-        uses: docker/build-push-action@v3
-        with:
-          context: ./citus
-          push: true
-          tags: localhost:5000/hydra/columnar:${{ env.TAG }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-      - name: Docker Build Spilo
-        uses: docker/build-push-action@v3
-        with:
-          context: ./spilo/postgres-appliance
-          file: ./spilo/postgres-appliance/Dockerfile
-          push: true
-          tags: |
-            localhost:5000/hydra/spilo:${{ env.TAG }}
-          build-args: |
-            TIMESCALEDB=
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-      - name: Docker Build Hydra
-        uses: docker/build-push-action@v3
-        with:
-          context: ./hydra
-          file: ./hydra/Dockerfile
-          push: true
-          tags: |
-            localhost:5000/hydra/hydra:${{ env.TAG }}
-          build-args: |
-            COLUMNAR_EXT_IMAGE=localhost:5000/hydra/columnar:${{ env.TAG }}
-            SPILO_IMAGE=localhost:5000/hydra/spilo:${{ env.TAG }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-      - name: Docker Build Hydra All
-        uses: docker/build-push-action@v3
+
+      - name: Docker build
+        uses: docker/bake-action@master
         with:
-          context: ./hydra
-          file: ./hydra/Dockerfile.all
+          workdir: ./hydra
           push: true
-          tags: |
-            localhost:5000/hydra/hydra-all:${{ env.TAG }}
-          build-args: |
-            HYDRA_EXT_IMAGE=localhost:5000/hydra/hydra-extension:${{ env.TAG }}
-            COLUMNAR_EXT_IMAGE=localhost:5000/hydra/columnar:${{ env.TAG }}
-            SPILO_IMAGE=localhost:5000/hydra/spilo:${{ env.TAG }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          set: |
+            hydra.tags=localhost:5000/hydra/hydra:${{ env.TAG }}
+            hydra-all.tags=localhost:5000/hydra/hydra-all:${{ env.TAG }}
+            *.cache-from=type=gha
+            *.cache-to=type=gha,mode=max
 
       # Acceptance tests
       - uses: actions/setup-go@v2
@@ -132,26 +91,19 @@ jobs:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Docker Push Hydra
+
+      - name: Docker push hydra to ghcr.io
         if: github.ref == 'refs/heads/master'
-        uses: docker/build-push-action@v3
+        uses: docker/bake-action@master
         with:
-          context: ./hydra
-          file: ./hydra/Dockerfile
+          workdir: ./hydra
+          targets: hydra
           push: true
-          tags: |
-            ghcr.io/hydrasco/hydra:${{ env.TAG }}
-            ghcr.io/hydrasco/hydra:latest
-          build-args: |
-            COLUMNAR_EXT_IMAGE=localhost:5000/hydra/columnar:${{ env.TAG }}
-            SPILO_IMAGE=localhost:5000/hydra/spilo:${{ env.TAG }}
-          labels:
-            maintainer=HydrasCo
-            org.opencontainers.image.title=hydra
-            org.opencontainers.image.source=https://github.com/HydrasCo/hydra
-            org.opencontainers.image.vendor=HydrasCo
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          set: |
+            hydra.tags=ghcr.io/hydrasco/hydra:latest
+            hydra.tags=ghcr.io/hydrasco/hydra:${{ env.TAG }}
+            *.cache-from=type=gha
+            *.cache-to=type=gha,mode=max
 
       # Docker push hydra-all
       - name: Configure AWS credentials
@@ -162,26 +114,24 @@ jobs:
           aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY }}
           aws-region: us-east-1
           mask-aws-account-id: no
+
       - name: Login to Amazon ECR
         if: github.ref == 'refs/heads/master'
         id: login-ecr
         uses: aws-actions/amazon-ecr-login@v1
         with:
           registries: "011789831835"
-      - name: Docker Push Hydra All
+
+      - name: Docker push hydra-all to ECR
         if: github.ref == 'refs/heads/master'
-        uses: docker/build-push-action@v3
+        uses: docker/bake-action@master
         with:
-          context: ./hydra
-          file: ./hydra/Dockerfile.all
+          workdir: ./hydra
+          targets: hydra-all
           push: true
           # TODO: create ECR repo to HOST/hydra and change the refs here
-          tags: |
-            ${{ steps.login-ecr.outputs.registry }}/spilo:${{ env.TAG }}
-            ${{ steps.login-ecr.outputs.registry }}/spilo:latest
-          build-args: |
-            HYDRA_EXT_IMAGE=localhost:5000/hydra/hydra-extension:${{ env.TAG }}
-            COLUMNAR_EXT_IMAGE=localhost:5000/hydra/columnar:${{ env.TAG }}
-            SPILO_IMAGE=localhost:5000/hydra/spilo:${{ env.TAG }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          set: |
+            hydra-all.tags=${{ steps.login-ecr.outputs.registry }}/spilo:latest
+            hydra-all.tags=${{ steps.login-ecr.outputs.registry }}/spilo:${{ env.TAG }}
+            *.cache-from=type=gha
+            *.cache-to=type=gha,mode=max

Dockerfile

@@ -1,9 +1,8 @@
-ARG COLUMNAR_EXT_IMAGE
-ARG SPILO_IMAGE
+#syntax=docker/dockerfile-upstream:latest
 
-FROM $COLUMNAR_EXT_IMAGE as columnar-ext
+FROM columnar_ext as columnar-ext
 
-FROM $SPILO_IMAGE
+FROM spilobase
 
 COPY --from=columnar-ext /pg_ext /
 COPY files/default/postgres-appliance/scripts /scripts/

Dockerfile.all

@@ -1,11 +1,9 @@
-ARG HYDRA_EXT_IMAGE
-ARG COLUMNAR_EXT_IMAGE
-ARG SPILO_IMAGE
+#syntax=docker/dockerfile-upstream:latest
 
-FROM $HYDRA_EXT_IMAGE as hydra-ext
-FROM $COLUMNAR_EXT_IMAGE as columnar-ext
+FROM hydra_ext as hydra-ext
+FROM columnar_ext as columnar-ext
 
-FROM $SPILO_IMAGE
+FROM spilobase
 
 COPY --from=hydra-ext /pg_ext /
 COPY --from=columnar-ext /pg_ext /

Makefile

@@ -5,64 +5,25 @@ define clone_if_not_exist
 endef
 
 DOCKER_OPTS ?=
+TARGET ?= default
 TAG ?= latest
-HYDRA_EXT_REPO ?= ghcr.io/hydrasco/hydra_ext
-COLUMNAR_EXT_REPO ?= ghcr.io/hydrasco/columnar_ext
-SPILO_REPO ?= ghcr.io/hydrasco/spilo
 HYDRA_REPO ?= ghcr.io/hydrasco/hydra
 HYDRA_ALL_REPO ?= ghcr.io/hydrasco/hydra-all
 
 .PHONY: docker_push
 docker_push: docker_build
-	docker push $(HYDRA_REPO):$(TAG)
+	TAG=$(TAG) HYDRA_REPO=$(HYDRA_REPO) HYDRA_ALL_REPO=$(HYDRA_ALL_REPO) docker buildx bake $(DOCKER_OPTS) $(TARGET) --push
 
 .PHONY: docker_build
-docker_build: clone_projects docker_build_columnar_ext docker_build_spilo
-	docker build \
-		$(DOCKER_OPTS) \
-		--build-arg COLUMNAR_EXT_IMAGE=$(COLUMNAR_EXT_REPO):$(TAG) \
-		--build-arg SPILO_IMAGE=$(SPILO_REPO):$(TAG) \
-		-t $(HYDRA_REPO):$(TAG) \
-		.
-
-.PHONY: docker_push_all
-docker_push_all: docker_build_all
-	docker push $(HYDRA_ALL_REPO):$(TAG)
-
-.PHONY: docker_build_all
-docker_build_all: clone_projects docker_build_hydra_ext docker_build_columnar_ext docker_build_spilo
-	docker build \
-		$(DOCKER_OPTS) \
-		--build-arg HYDRA_EXT_IMAGE=$(HYDRA_EXT_REPO):$(TAG) \
-		--build-arg COLUMNAR_EXT_IMAGE=$(COLUMNAR_EXT_REPO):$(TAG) \
-		--build-arg SPILO_IMAGE=$(SPILO_REPO):$(TAG) \
-		-f Dockerfile.all \
-		-t $(HYDRA_ALL_REPO):$(TAG) \
-		.
+docker_build: clone_projects
+	TAG=$(TAG) HYDRA_REPO=$(HYDRA_REPO) HYDRA_ALL_REPO=$(HYDRA_ALL_REPO) docker buildx bake $(DOCKER_OPTS) $(TARGET)
 
 .PHONY: clone_projects
 clone_projects:
 	@$(call clone_if_not_exist,[email protected]:HydrasCo/hydra-extension.git,$(CURDIR)/../hydra-extension,main)
 	@$(call clone_if_not_exist,[email protected]:HydrasCo/citus.git,$(CURDIR)/../citus,master)
 	@$(call clone_if_not_exist,[email protected]:zalando/spilo.git,$(CURDIR)/../spilo,2.1-p6)
 
-.PHONY: docker_build_spilo
-docker_build_spilo:
-	cd $(CURDIR)/../spilo/postgres-appliance && \
-		docker build \
-		$(DOCKER_OPTS) \
-		--build-arg TIMESCALEDB="" \
-		-t $(SPILO_REPO):$(TAG) \
-		.
-
-.PHONY: docker_build_hydra_ext
-docker_build_hydra_ext:
-	cd $(CURDIR)/../Hydras && docker build $(DOCKER_OPTS) -t $(HYDRA_EXT_REPO):$(TAG) .
-
-.PHONY: docker_build_columnar_ext
-docker_build_columnar_ext:
-	cd $(CURDIR)/../citus && docker build $(DOCKER_OPTS) -t $(COLUMNAR_EXT_REPO):$(TAG) .
-
 .PHONY: acceptance_test
 acceptance_test:
 	go test ./... -count=1 -race -v

README.md

@@ -28,13 +28,13 @@ There are two image distributions:
 To build the first distribution, run:
 
 ```
-TAG=1234 make docker_build
+TAG=1234 TARGET=hydra make docker_build
 ```
 
 To build the second distribution, run:
 
 ```
-TAG=1234 make docker_build_all
+TAG=1234 TARGET=hydra-all make docker_build
 ```
 
 ## Image Build Tags

docker-bake.hcl

@@ -0,0 +1,48 @@
+variable "TAG" {
+  default = "latest"
+}
+
+variable "HYDRA_REPO" {
+  default = "ghcr.io/hydrasco/hydra"
+}
+
+variable "HYDRA_ALL_REPO" {
+  default = "ghcr.io/hydrasco/hydra-all"
+}
+
+group "default" {
+  targets = ["hydra", "hydra-all"]
+}
+
+target "hydra-all" {
+  dockerfile = "Dockerfile.all"
+  contexts = {
+    spilobase = "target:spilo"
+    columnar_ext = "target:columnar_ext"
+    hydra_ext = "target:hydra_ext"
+  }
+  tags = ["${HYDRA_ALL_REPO}:${TAG}"]
+}
+
+target "hydra" {
+  contexts = {
+    spilobase = "target:spilo"
+    columnar_ext = "target:columnar_ext"
+  }
+  tags = ["${HYDRA_REPO}:${TAG}"]
+}
+
+target "spilo" {
+  context = "../spilo/postgres-appliance"
+  args = {
+    TIMESCALEDB = ""
+  }
+}
+
+target "columnar_ext" {
+  context = "../citus"
+}
+
+target "hydra_ext" {
+  context = "../hydra-extension"
+}