-
Notifications
You must be signed in to change notification settings - Fork 22
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
chore: reuse aggregation from ctf during component build (#1044)
<!-- markdownlint-disable MD041 --> #### What this PR does / why we need it this will create an aggregation post build that can be reused during releases if necessary #### Which issue(s) this PR fixes <!-- Usage: `Fixes #<issue number>`, or `Fixes (paste link of issue)`. --> This is a revision of the component build in preparation to reuse the final CTF artifacts during our release --------- Co-authored-by: Hilmar Falkenberg <[email protected]>
- Loading branch information
1 parent
d6a5994
commit 2188c1c
Showing
6 changed files
with
89 additions
and
141 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,4 +1,4 @@ | ||
| name: component CTFs | ||
| name: Components | ||
|
|
||
| on: | ||
| pull_request: | ||
|
|
@@ -11,62 +11,44 @@ permissions: | |
| contents: read | ||
| pull-requests: read | ||
|
|
||
| env: | ||
| CTF_TYPE: directory | ||
| components: '["ocmcli", "helminstaller", "helmdemo", "subchartsdemo", "ecrplugin"]' | ||
|
|
||
| jobs: | ||
| build-cli: | ||
| name: Build CLI | ||
| runs-on: large_runner | ||
| define-matrix: | ||
| runs-on: ubuntu-latest | ||
| outputs: | ||
| components: ${{ steps.componentMatrix.outputs.matrix }} | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v4 | ||
| - name: Setup Go | ||
| uses: actions/setup-go@v5 | ||
| with: | ||
| go-version-file: '${{ github.workspace }}/go.mod' | ||
| cache: false | ||
|
|
||
| - name: Get go environment for use with cache | ||
| - id: componentMatrix | ||
| name: Set Components to be used for Build | ||
| run: | | ||
| echo "go_cache=$(go env GOCACHE)" >> $GITHUB_ENV | ||
| echo "go_modcache=$(go env GOMODCACHE)" >> $GITHUB_ENV | ||
| # This step will only reuse the go mod and build cache from main made during the Build, | ||
| # see push_ocm.yaml => "ocm-cli-latest" Job | ||
| # This means it never caches by itself and PRs cannot cause cache pollution / thrashing | ||
| # This is because we have huge storage requirements for our cache because of the mass of dependencies | ||
| - name: Restore / Reuse Cache from central build | ||
| id: cache-golang-restore | ||
| uses: actions/cache/restore@v4 # Only Restore, not build another cache (too big) | ||
| with: | ||
| path: | | ||
| ${{ env.go_cache }} | ||
| ${{ env.go_modcache }} | ||
| key: ${{ env.cache_name }}-${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}-${{ hashFiles('**/go.mod') }} | ||
| restore-keys: | | ||
| ${{ env.cache_name }}-${{ runner.os }}-go- | ||
| echo "matrix=$input" >> $GITHUB_OUTPUT | ||
| env: | ||
| cache_name: ocm-cli-latest-go-cache # needs to be the same key in the end as in the build step | ||
|
|
||
| - name: CTF | ||
| run: | | ||
| cd components/ocmcli | ||
| PATH=$PATH:$(go env GOPATH)/bin make ctf | ||
| build-helminstaller: | ||
| name: Build HelmInstaller | ||
| input: ${{ env.components }} | ||
|
|
||
| build: | ||
| name: "Build" | ||
| needs: define-matrix | ||
| strategy: | ||
| matrix: | ||
| component: ${{fromJSON(needs.define-matrix.outputs.components)}} | ||
| runs-on: large_runner | ||
| steps: | ||
| - name: Self Hosted Runner Post Job Cleanup Action | ||
| uses: TooMuch4U/[email protected] | ||
| - name: Checkout | ||
| uses: actions/checkout@v4 | ||
| - name: Setup Go | ||
| uses: actions/setup-go@v5 | ||
| with: | ||
| go-version-file: '${{ github.workspace }}/go.mod' | ||
| cache: false | ||
|
|
||
| - name: Get go environment for use with cache | ||
| run: | | ||
| echo "go_cache=$(go env GOCACHE)" >> $GITHUB_ENV | ||
| echo "go_modcache=$(go env GOMODCACHE)" >> $GITHUB_ENV | ||
| # This step will only reuse the go mod and build cache from main made during the Build, | ||
| # see push_ocm.yaml => "ocm-cli-latest" Job | ||
| # This means it never caches by itself and PRs cannot cause cache pollution / thrashing | ||
|
|
@@ -83,29 +65,39 @@ jobs: | |
| ${{ env.cache_name }}-${{ runner.os }}-go- | ||
| env: | ||
| cache_name: ocm-cli-latest-go-cache # needs to be the same key in the end as in the build step | ||
|
|
||
| - name: CTF | ||
| run: | | ||
| cd components/helminstaller | ||
| PATH=$PATH:$(go env GOPATH)/bin make ctf | ||
| build-helmdemo: | ||
| name: Build HelmDemo | ||
| cd components/${{ matrix.component }} | ||
| PATH=$PATH:$(go env GOPATH)/bin CTF_TYPE=${{ env.CTF_TYPE }} make ctf descriptor describe | ||
| - name: Upload CTF | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| if-no-files-found: error | ||
| overwrite: true | ||
| retention-days: 1 | ||
| name: ctf-component-${{ matrix.component }} | ||
| path: gen/${{ matrix.component }}/ctf | ||
|
|
||
| aggregate: | ||
| name: "Aggregate" | ||
| runs-on: large_runner | ||
| needs: [build, define-matrix] | ||
| env: | ||
| components: ${{ join(fromJSON(needs.define-matrix.outputs.components), ' ') }} | ||
| steps: | ||
| - name: Self Hosted Runner Post Job Cleanup Action | ||
| uses: TooMuch4U/[email protected] | ||
| - name: Checkout | ||
| uses: actions/checkout@v4 | ||
| - name: Setup Go | ||
| uses: actions/setup-go@v5 | ||
| with: | ||
| go-version-file: '${{ github.workspace }}/go.mod' | ||
| cache: false | ||
|
|
||
| - name: Get go environment for use with cache | ||
| run: | | ||
| echo "go_cache=$(go env GOCACHE)" >> $GITHUB_ENV | ||
| echo "go_modcache=$(go env GOMODCACHE)" >> $GITHUB_ENV | ||
| # This step will only reuse the go mod and build cache from main made during the Build, | ||
| # see push_ocm.yaml => "ocm-cli-latest" Job | ||
| # This means it never caches by itself and PRs cannot cause cache pollution / thrashing | ||
|
|
@@ -122,89 +114,37 @@ jobs: | |
| ${{ env.cache_name }}-${{ runner.os }}-go- | ||
| env: | ||
| cache_name: ocm-cli-latest-go-cache # needs to be the same key in the end as in the build step | ||
|
|
||
| - name: CTF | ||
| run: | | ||
| cd components/helmdemo | ||
| PATH=$PATH:$(go env GOPATH)/bin make ctf | ||
| build-subchartsdemo: | ||
| name: Build Helm SubChartsDemo | ||
| runs-on: large_runner | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v4 | ||
| - name: Setup Go | ||
| uses: actions/setup-go@v5 | ||
| - name: Download CTFs | ||
| uses: actions/download-artifact@v4 | ||
| with: | ||
| go-version-file: '${{ github.workspace }}/go.mod' | ||
| cache: false | ||
|
|
||
| - name: Get go environment for use with cache | ||
| pattern: 'ctf-component-*' | ||
| path: gen/downloaded-ctfs | ||
| - name: Move CTFs into correct directory for aggregation | ||
| run: | | ||
| echo "go_cache=$(go env GOCACHE)" >> $GITHUB_ENV | ||
| echo "go_modcache=$(go env GOMODCACHE)" >> $GITHUB_ENV | ||
| # This step will only reuse the go mod and build cache from main made during the Build, | ||
| # see push_ocm.yaml => "ocm-cli-latest" Job | ||
| # This means it never caches by itself and PRs cannot cause cache pollution / thrashing | ||
| # This is because we have huge storage requirements for our cache because of the mass of dependencies | ||
| - name: Restore / Reuse Cache from central build | ||
| id: cache-golang-restore | ||
| uses: actions/cache/restore@v4 # Only Restore, not build another cache (too big) | ||
| with: | ||
| path: | | ||
| ${{ env.go_cache }} | ||
| ${{ env.go_modcache }} | ||
| key: ${{ env.cache_name }}-${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}-${{ hashFiles('**/go.mod') }} | ||
| restore-keys: | | ||
| ${{ env.cache_name }}-${{ runner.os }}-go- | ||
| env: | ||
| cache_name: ocm-cli-latest-go-cache # needs to be the same key in the end as in the build step | ||
|
|
||
| - name: CTF | ||
| IFS=" " read -a COMPONENTS <<< "${{ env.components }}" | ||
| for i in "${COMPONENTS[@]}"; do | ||
| mkdir -p ${{ github.workspace }}/gen/${i} | ||
| mv ${{ github.workspace }}/gen/downloaded-ctfs/ctf-component-${i} ${{ github.workspace }}/gen/${i}/ctf | ||
| ls -R ${{ github.workspace }}/gen/${i} | ||
| done | ||
| - name: Create aggregated CTF | ||
| run: | | ||
| cd components/subchartsdemo | ||
| PATH=$PATH:$(go env GOPATH)/bin make ctf | ||
| build-ecrplugin: | ||
| name: Build ECR Plugin | ||
| runs-on: large_runner | ||
| steps: | ||
| - name: Self Hosted Runner Post Job Cleanup Action | ||
| uses: TooMuch4U/[email protected] | ||
|
|
||
| - name: Checkout | ||
| uses: actions/checkout@v4 | ||
| - name: Setup Go | ||
| uses: actions/setup-go@v5 | ||
| PATH=$PATH:$(go env GOPATH)/bin \ | ||
| CTF_TYPE=${{ env.CTF_TYPE }} \ | ||
| COMPONENTS="${{ env.components }}" \ | ||
| make plain-ctf | ||
| - name: Upload aggregated CTF | ||
| # only upload the artifact if we are not on a PR | ||
| if: needs.pr-check.outputs.number != 'null' | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| go-version-file: '${{ github.workspace }}/go.mod' | ||
| cache: false | ||
|
|
||
| - name: Get go environment for use with cache | ||
| run: | | ||
| echo "go_cache=$(go env GOCACHE)" >> $GITHUB_ENV | ||
| echo "go_modcache=$(go env GOMODCACHE)" >> $GITHUB_ENV | ||
| # This step will only reuse the go mod and build cache from main made during the Build, | ||
| # see push_ocm.yaml => "ocm-cli-latest" Job | ||
| # This means it never caches by itself and PRs cannot cause cache pollution / thrashing | ||
| # This is because we have huge storage requirements for our cache because of the mass of dependencies | ||
| - name: Restore / Reuse Cache from central build | ||
| id: cache-golang-restore | ||
| uses: actions/cache/restore@v4 # Only Restore, not build another cache (too big) | ||
| if-no-files-found: error | ||
| overwrite: true | ||
| retention-days: 60 | ||
| name: ctf-aggregated | ||
| path: gen/ctf | ||
| - name: Delete old CTFs that lead up to aggregation | ||
| uses: geekyeggo/delete-artifact@v5 | ||
| with: | ||
| path: | | ||
| ${{ env.go_cache }} | ||
| ${{ env.go_modcache }} | ||
| key: ${{ env.cache_name }}-${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}-${{ hashFiles('**/go.mod') }} | ||
| restore-keys: | | ||
| ${{ env.cache_name }}-${{ runner.os }}-go- | ||
| env: | ||
| cache_name: ocm-cli-latest-go-cache # needs to be the same key in the end as in the build step | ||
|
|
||
| - name: CTF | ||
| run: | | ||
| cd components/ecrplugin | ||
| PATH=$PATH:$(go env GOPATH)/bin make ctf | ||
| name: | | ||
| ctf-component-* | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters