chore: reuse aggregation from ctf during component build

.github/workflows/components.yaml

@@ -1,4 +1,4 @@
-name: component CTFs
+name: Components
 
 on:
   pull_request:
@@ -11,62 +11,44 @@ permissions:
   contents: read
   pull-requests: read
 
+env:
+  CTF_TYPE: directory
+  components: '["ocmcli", "helminstaller", "helmdemo", "subchartsdemo", "ecrplugin"]'
+
 jobs:
-  build-cli:
-    name: Build CLI
-    runs-on: large_runner
+  define-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      components: ${{ steps.componentMatrix.outputs.matrix }}
     steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Setup Go
-        uses: actions/setup-go@v5
-        with:
-          go-version-file: '${{ github.workspace }}/go.mod'
-          cache: false
-
-      - name: Get go environment for use with cache
+      - id: componentMatrix
+        name: Set Components to be used for Build
         run: |
-          echo "go_cache=$(go env GOCACHE)" >> $GITHUB_ENV
-          echo "go_modcache=$(go env GOMODCACHE)" >> $GITHUB_ENV
-      # This step will only reuse the go mod and build cache from main made during the Build,
-      # see push_ocm.yaml => "ocm-cli-latest" Job
-      # This means it never caches by itself and PRs cannot cause cache pollution / thrashing
-      # This is because we have huge storage requirements for our cache because of the mass of dependencies
-      - name: Restore / Reuse Cache from central build
-        id: cache-golang-restore
-        uses: actions/cache/restore@v4 # Only Restore, not build another cache (too big)
-        with:
-          path: |
-            ${{ env.go_cache }}
-            ${{ env.go_modcache }}
-          key: ${{ env.cache_name }}-${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}-${{ hashFiles('**/go.mod') }}
-          restore-keys: |
-            ${{ env.cache_name }}-${{ runner.os }}-go-
+          echo "matrix=$input" >> $GITHUB_OUTPUT
         env:
-          cache_name: ocm-cli-latest-go-cache # needs to be the same key in the end as in the build step
-
-      - name: CTF
-        run: |
-          cd components/ocmcli
-          PATH=$PATH:$(go env GOPATH)/bin make ctf
-
-  build-helminstaller:
-    name: Build HelmInstaller
+          input: ${{ env.components }}
+
+  build:
+    name: "Build"
+    needs: define-matrix
+    strategy:
+      matrix:
+        component: ${{fromJSON(needs.define-matrix.outputs.components)}}
     runs-on: large_runner
     steps:
+      - name: Self Hosted Runner Post Job Cleanup Action
+        uses: TooMuch4U/[email protected]
       - name: Checkout
         uses: actions/checkout@v4
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
           go-version-file: '${{ github.workspace }}/go.mod'
           cache: false
-
       - name: Get go environment for use with cache
         run: |
           echo "go_cache=$(go env GOCACHE)" >> $GITHUB_ENV
           echo "go_modcache=$(go env GOMODCACHE)" >> $GITHUB_ENV
-
       # This step will only reuse the go mod and build cache from main made during the Build,
       # see push_ocm.yaml => "ocm-cli-latest" Job
       # This means it never caches by itself and PRs cannot cause cache pollution / thrashing
@@ -83,29 +65,39 @@ jobs:
             ${{ env.cache_name }}-${{ runner.os }}-go-
         env:
           cache_name: ocm-cli-latest-go-cache # needs to be the same key in the end as in the build step
-
       - name: CTF
         run: |
-          cd components/helminstaller
-          PATH=$PATH:$(go env GOPATH)/bin make ctf
-
-  build-helmdemo:
-    name: Build HelmDemo
+          cd components/${{ matrix.component }}
+          PATH=$PATH:$(go env GOPATH)/bin CTF_TYPE=${{ env.CTF_TYPE }} make ctf descriptor describe
+      - name: Upload CTF
+        uses: actions/upload-artifact@v4
+        with:
+          if-no-files-found: error
+          overwrite: true
+          retention-days: 1
+          name: ctf-component-${{ matrix.component }}
+          path: gen/${{ matrix.component }}/ctf
+
+  aggregate:
+    name: "Aggregate"
     runs-on: large_runner
+    needs: [build, define-matrix]
+    env:
+      components: ${{ join(fromJSON(needs.define-matrix.outputs.components), ' ') }}
     steps:
+      - name: Self Hosted Runner Post Job Cleanup Action
+        uses: TooMuch4U/[email protected]
       - name: Checkout
         uses: actions/checkout@v4
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
           go-version-file: '${{ github.workspace }}/go.mod'
           cache: false
-
       - name: Get go environment for use with cache
         run: |
           echo "go_cache=$(go env GOCACHE)" >> $GITHUB_ENV
           echo "go_modcache=$(go env GOMODCACHE)" >> $GITHUB_ENV
-
       # This step will only reuse the go mod and build cache from main made during the Build,
       # see push_ocm.yaml => "ocm-cli-latest" Job
       # This means it never caches by itself and PRs cannot cause cache pollution / thrashing
@@ -122,89 +114,37 @@ jobs:
             ${{ env.cache_name }}-${{ runner.os }}-go-
         env:
           cache_name: ocm-cli-latest-go-cache # needs to be the same key in the end as in the build step
-
-      - name: CTF
-        run: |
-          cd components/helmdemo
-          PATH=$PATH:$(go env GOPATH)/bin make ctf
-
-  build-subchartsdemo:
-    name: Build Helm SubChartsDemo
-    runs-on: large_runner
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Setup Go
-        uses: actions/setup-go@v5
+      - name: Download CTFs
+        uses: actions/download-artifact@v4
         with:
-          go-version-file: '${{ github.workspace }}/go.mod'
-          cache: false
-
-      - name: Get go environment for use with cache
+          pattern: 'ctf-component-*'
+          path: gen/downloaded-ctfs
+      - name: Move CTFs into correct directory for aggregation
         run: |
-          echo "go_cache=$(go env GOCACHE)" >> $GITHUB_ENV
-          echo "go_modcache=$(go env GOMODCACHE)" >> $GITHUB_ENV
-
-      # This step will only reuse the go mod and build cache from main made during the Build,
-      # see push_ocm.yaml => "ocm-cli-latest" Job
-      # This means it never caches by itself and PRs cannot cause cache pollution / thrashing
-      # This is because we have huge storage requirements for our cache because of the mass of dependencies
-      - name: Restore / Reuse Cache from central build
-        id: cache-golang-restore
-        uses: actions/cache/restore@v4 # Only Restore, not build another cache (too big)
-        with:
-          path: |
-            ${{ env.go_cache }}
-            ${{ env.go_modcache }}
-          key: ${{ env.cache_name }}-${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}-${{ hashFiles('**/go.mod') }}
-          restore-keys: |
-            ${{ env.cache_name }}-${{ runner.os }}-go-
-        env:
-          cache_name: ocm-cli-latest-go-cache # needs to be the same key in the end as in the build step
-
-      - name: CTF
+          IFS=" " read -a COMPONENTS <<< "${{ env.components }}"
+          for i in "${COMPONENTS[@]}"; do
+            mkdir -p ${{ github.workspace }}/gen/${i}
+            mv ${{ github.workspace }}/gen/downloaded-ctfs/ctf-component-${i} ${{ github.workspace }}/gen/${i}/ctf
+            ls -R ${{ github.workspace }}/gen/${i}
+          done
+      - name: Create aggregated CTF
         run: |
-          cd components/subchartsdemo
-          PATH=$PATH:$(go env GOPATH)/bin make ctf
-
-  build-ecrplugin:
-    name: Build ECR Plugin
-    runs-on: large_runner
-    steps:
-      - name: Self Hosted Runner Post Job Cleanup Action
-        uses: TooMuch4U/[email protected]
-
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Setup Go
-        uses: actions/setup-go@v5
+          PATH=$PATH:$(go env GOPATH)/bin \
+          CTF_TYPE=${{ env.CTF_TYPE }} \
+          COMPONENTS="${{ env.components }}" \
+          make plain-ctf
+      - name: Upload aggregated CTF
+        # only upload the artifact if we are not on a PR
+        if: needs.pr-check.outputs.number != 'null'
+        uses: actions/upload-artifact@v4
         with:
-          go-version-file: '${{ github.workspace }}/go.mod'
-          cache: false
-
-      - name: Get go environment for use with cache
-        run: |
-          echo "go_cache=$(go env GOCACHE)" >> $GITHUB_ENV
-          echo "go_modcache=$(go env GOMODCACHE)" >> $GITHUB_ENV
-
-      # This step will only reuse the go mod and build cache from main made during the Build,
-      # see push_ocm.yaml => "ocm-cli-latest" Job
-      # This means it never caches by itself and PRs cannot cause cache pollution / thrashing
-      # This is because we have huge storage requirements for our cache because of the mass of dependencies
-      - name: Restore / Reuse Cache from central build
-        id: cache-golang-restore
-        uses: actions/cache/restore@v4 # Only Restore, not build another cache (too big)
+          if-no-files-found: error
+          overwrite: true
+          retention-days: 60
+          name: ctf-aggregated
+          path: gen/ctf
+      - name: Delete old CTFs that lead up to aggregation
+        uses: geekyeggo/delete-artifact@v5
         with:
-          path: |
-            ${{ env.go_cache }}
-            ${{ env.go_modcache }}
-          key: ${{ env.cache_name }}-${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}-${{ hashFiles('**/go.mod') }}
-          restore-keys: |
-            ${{ env.cache_name }}-${{ runner.os }}-go-
-        env:
-          cache_name: ocm-cli-latest-go-cache # needs to be the same key in the end as in the build step
-
-      - name: CTF
-        run: |
-          cd components/ecrplugin
-          PATH=$PATH:$(go env GOPATH)/bin make ctf
+          name: |
+            ctf-component-*

Makefile

@@ -13,7 +13,7 @@ CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen
 PLATFORMS = windows/amd64 darwin/arm64 darwin/amd64 linux/amd64 linux/arm64
 
 CREDS    ?=
-OCM      := go run $(REPO_ROOT)/cmds/ocm $(CREDS)
+OCM := bin/ocm $(CREDS)
 CTF_TYPE ?= directory
 
 GEN := $(REPO_ROOT)/gen
@@ -165,7 +165,7 @@ $(GEN)/.comps: $(GEN)/.exists
 .PHONY: ctf
 ctf: $(GEN)/ctf
 
-$(GEN)/ctf: $(GEN)/.exists $(GEN)/.comps
+$(GEN)/ctf: $(GEN)/.exists $(GEN)/.comps bin/ocm
 	@rm -rf "$(GEN)"/ctf
 	@for i in $(COMPONENTS); do \
       echo "transfering component $$i..."; \
@@ -174,19 +174,27 @@ $(GEN)/ctf: $(GEN)/.exists $(GEN)/.comps
 	done
 	@touch $@
 
+.PHONY: describe
+describe: $(GEN)/ctf bin/ocm
+	$(OCM) get resources --lookup $(OCMREPO) -r -o treewide $(GEN)/ctf
+
+.PHONY: descriptor
+descriptor: $(GEN)/ctf bin/ocm
+	$(OCM) get component -S v3alpha1 -o yaml $(GEN)/ctf
+
 .PHONY: push
 push: $(GEN)/ctf $(GEN)/.push.$(NAME)
 
-$(GEN)/.push.$(NAME): $(GEN)/ctf
+$(GEN)/.push.$(NAME): $(GEN)/ctf bin/ocm
 	$(OCM) transfer ctf -f $(GEN)/ctf $(OCMREPO)
 	@touch $@
 
 .PHONY: plain-push
-plain-push: $(GEN)
+plain-push: $(GEN) bin/ocm
 	$(OCM) transfer ctf -f $(GEN)/ctf $(OCMREPO)
 
 .PHONY: plain-ctf
-plain-ctf: $(GEN)
+plain-ctf: $(GEN) bin/ocm
 	@rm -rf "$(GEN)"/ctf
 	@for i in $(COMPONENTS); do \
        echo "transfering component $$i..."; \

components/demoplugin/Makefile

@@ -105,7 +105,7 @@ info:
 
 .PHONY: describe
 describe: $(GEN)/ctf $(OCM_BIN)
-	$(OCM) get resources --lookup $(OCMREPO) -c -o treewide $(GEN)/ctf
+	$(OCM) get resources --lookup $(OCMREPO) -r -o treewide $(GEN)/ctf
 
 .PHONY: descriptor
 descriptor: $(GEN)/ctf $(OCM_BIN)

components/ecrplugin/Makefile

@@ -107,7 +107,7 @@ info:
 
 .PHONY: describe
 describe: $(GEN)/ctf $(OCM_BIN)
-	$(OCM) get resources --lookup $(OCMREPO) -c -o treewide $(GEN)/ctf
+	$(OCM) get resources --lookup $(OCMREPO) -r -o treewide $(GEN)/ctf
 
 .PHONY: descriptor
 descriptor: $(GEN)/ctf $(OCM_BIN)

components/ocmcli/Makefile

@@ -156,7 +156,7 @@ info:
 
 .PHONY: describe
 describe: $(GEN)/ctf $(OCM_BIN)
-	$(OCM) get resources --lookup $(OCMREPO) -c -o treewide $(GEN)/ctf
+	$(OCM) get resources --lookup $(OCMREPO) -r -o treewide $(GEN)/ctf
 
 .PHONY: descriptor
 descriptor: $(GEN)/ctf $(OCM_BIN)

components/subchartsdemo/Makefile

@@ -77,12 +77,12 @@ info:
 	@echo "CREDS:    $(CREDS)"
 
 .PHONY: describe
-describe: $(GEN)/ctf
-	ocm get resources --lookup $(OCMREPO) -o treewide $(GEN)/ctf
+describe: $(GEN)/ctf $(OCM_BIN)
+	$(OCM) get resources --lookup $(OCMREPO) -r -o treewide $(GEN)/ctf
 
 .PHONY: descriptor
-descriptor: $(GEN)/ctf
-	ocm get component -S v3alpha1 -o yaml $(GEN)/ctf
+descriptor: $(GEN)/ctf $(OCM_BIN)
+	$(OCM) get component -S v3alpha1 -o yaml $(GEN)/ctf
 
 .PHONY: clean
 clean: