Refactor large workflows into multiple jobs (#1902)

## Description This PR refactors our larger workflows to use multiple jobs to avoid accumulation on disk. ## Related Issue Fixes #N/A ## Type of change - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [X] Other (security config, docs update, etc) ## Checklist before merging - [X] Test, docs, adr added or updated as needed - [X] [Contributor Guide Steps](https://github.com/defenseunicorns/zarf/blob/main/CONTRIBUTING.md#developer-workflow) followed
zarf-dev · Jul 12, 2023 · d863e00 · d863e00
1 parent 49d41ee
commit d863e00
Show file tree

Hide file tree

Showing 8 changed files with 147 additions and 47 deletions.
diff --git a/.github/actions/cleanup-files/action.yaml b/.github/actions/cleanup-files/action.yaml
@@ -0,0 +1,16 @@
+name: cleanup-files
+description: "Cleans up files on the local system to reduce disk pressure"
+
+runs:
+  using: composite
+  steps:
+    - run: |
+        lsblk -f
+
+        sudo rm -rf zarf-sbom /tmp/zarf-* src/ui/node_modules
+        sudo env "PATH=$PATH" CI=true make delete-packages
+        sudo build/zarf tools clear-cache
+        sudo docker system prune --all --force
+
+        lsblk -f
+      shell: bash
diff --git a/.github/actions/install-tools/action.yaml b/.github/actions/install-tools/action.yaml
@@ -6,6 +6,8 @@ runs:
   steps:
     - uses: sigstore/cosign-installer@9becc617647dfa20ae7b1151972e9b3a2c338a2b # v2.8.1
 
+    - uses: anchore/sbom-action/download-syft@78fc58e266e87a38d4194b2137a3d4e9bcaf7ca1 # v0.14.3
+
     - run: "curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin"
       shell: bash
 

diff --git a/.github/actions/packages/action.yaml b/.github/actions/packages/action.yaml
@@ -1,9 +1,28 @@
 name: packages
-description: "Create agent image, init package and example packages"
+description: "Build CLI, agent image, init package and example packages"
+
+inputs:
+  init-package:
+    description: 'Build the init package'
+    required: false
+    default: 'true'
+  build-examples:
+    description: 'Build the example packages'
+    required: false
+    default: 'true'
+
 
 runs:
   using: composite
   steps:
     - run: |
-        make build-cli-linux-amd init-package build-examples ARCH=amd64
+        make build-cli-linux-amd ARCH=amd64
+      shell: bash
+    - run: |
+        make init-package ARCH=amd64
+      shell: bash
+      if: ${{ inputs.init-package == 'true' }}
+    - run: |
+        make build-examples ARCH=amd64
       shell: bash
+      if: ${{ inputs.build-examples == 'true' }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -9,12 +9,12 @@ on:
       - "v*"
 
 jobs:
-  push-resources:
+  build:
     runs-on: ubuntu-latest
     permissions:
-      contents: write
       packages: write
     steps:
+      # Checkout the repo and setup the tooling for this job
       - name: Checkout
         uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
         with:
@@ -46,6 +46,8 @@ jobs:
           cp build/zarf build/zarf-linux-amd64
           cp build/zarf-arm build/zarf-linux-arm64
           docker buildx build --push --platform linux/arm64/v8,linux/amd64 --tag ghcr.io/defenseunicorns/zarf/agent:$GITHUB_REF_NAME .
+          rm build/zarf-linux-amd64
+          rm build/zarf-linux-arm64
 
       - name: "Zarf Agent: Sign the Image"
         run: cosign sign --key awskms:///${{ secrets.COSIGN_AWS_KMS_KEY }} -a release-engineer=https://github.com/${{ github.actor }} -a version=$GITHUB_REF_NAME ghcr.io/defenseunicorns/zarf/agent:$GITHUB_REF_NAME
@@ -61,19 +63,40 @@ jobs:
           make release-init-package ARCH=amd64 AGENT_IMAGE_TAG=$GITHUB_REF_NAME
           make release-init-package ARCH=arm64 AGENT_IMAGE_TAG=$GITHUB_REF_NAME
 
-      # Before we run the tests we need to aggressively cleanup files to reduce disk pressure
-      - name: Cleanup files
-        run: |
-          lsblk -f
+      # Create a CVE report based on this build
+      - name: Create release time CVE report
+        run: "make cve-report"
 
-          sudo rm -rf zarf-sbom /tmp/zarf-* src/ui/node_modules
-          sudo build/zarf tools clear-cache
-          sudo docker system prune --all --force
-          go clean -cache
+      # Upload the contents of the build directory for later stages to use
+      - name: Upload build artifacts
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+        with:
+          name: build-artifacts
+          path: build/
+          retention-days: 1
 
-          lsblk -f
+  validate:
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      # Checkout the repo and setup the tooling for this job
+      - name: Checkout
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+        with:
+          fetch-depth: 0
 
-      - name: Run Tests
+      - name: Download build artifacts
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
+        with:
+          name: build-artifacts
+          path: build/
+
+      - name: Make Zarf executable
+        run: |
+          chmod +x build/zarf
+
+      # Build the example packages and run the tests
+      - name: Build examples and run tests
         run: |
           sudo env "PATH=$PATH" CI=true APPLIANCE_MODE=true make test-e2e ARCH=amd64
           sudo chown $USER /tmp/zarf-*.log
@@ -82,22 +105,32 @@ jobs:
         if: always()
         uses: ./.github/actions/save-logs
 
-      # Builds init packages since GoReleaser won't handle this for us
-      - name: Create release time CVE report
-        run: "make cve-report"
+  push:
+    runs-on: ubuntu-latest
+    needs: test
+    permissions:
+      contents: write
+    steps:
+      # Checkout the repo and setup the tooling for this job
+      - name: Checkout
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+        with:
+          fetch-depth: 0
 
-      # Before we run GoReleaser we need to (again) aggressively cleanup files to reduce disk pressure
-      - name: Cleanup files
-        run: |
-          lsblk -f
+      - name: Setup golang
+        uses: ./.github/actions/golang
 
-          sudo rm -rf zarf-sbom /tmp/zarf-*
-          sudo env "PATH=$PATH" CI=true make delete-packages
-          sudo build/zarf tools clear-cache
-          sudo docker system prune --all --force
-          go clean -cache
+      - name: Setup NodeJS
+        uses: ./.github/actions/node
 
-          lsblk -f
+      - name: Install tools
+        uses: ./.github/actions/install-tools
+
+      - name: Download build artifacts
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
+        with:
+          name: build-artifacts
+          path: build/
 
       # Set up AWS credentials for GoReleaser to upload backups of artifacts to S3
       - name: Set AWS Credentials

diff --git a/.github/workflows/test-bigbang.yml b/.github/workflows/test-bigbang.yml
@@ -34,13 +34,15 @@ jobs:
       - name: Setup NodeJS
         uses: ./.github/actions/node
 
-      - name: Build binary and zarf packages
+      - name: Build Zarf binary
         uses: ./.github/actions/packages
+        with:
+          build-examples: 'false'
 
       - name: Setup K3d
         uses: ./.github/actions/k3d
 
-      - name: "Login to Iron Bank"
+      - name: Login to Iron Bank
         uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2.2.0
         if: ${{ env.IRONBANK_USERNAME != '' }}
         env:

diff --git a/.github/workflows/test-k3s.yml b/.github/workflows/test-k3s.yml
@@ -41,9 +41,10 @@ jobs:
         # NOTE: "PATH=$PATH" preserves the default user $PATH. This is needed to maintain the version of go installed
         #       in a previous step. This test run will use Zarf to create a K3s cluster, and a brand new cluster will be
         #       used for each test
-        # chown the logs since they were orignally created as root
+        # chown the logs since they were originally created as root
         run: |
           sudo env "PATH=$PATH" CI=true APPLIANCE_MODE=true make test-e2e ARCH=amd64
+          sudo chown $USER /tmp/zarf-*.log
 
       - name: Save logs
         if: always()

diff --git a/.github/workflows/test-upgrade.yml b/.github/workflows/test-upgrade.yml
@@ -22,10 +22,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  validate:
-    env:
-      # Reduce the kubelet eviction minimums reduce the chance for disk pressure causing evictions during the tests
-      ZARF_PACKAGE_DEPLOY_SET_K3S_ARGS: --disable traefik --kubelet-arg "eviction-hard=imagefs.available<1%,nodefs.available<1%" --kubelet-arg "eviction-minimum-reclaim=imagefs.available=1%,nodefs.available=1%"
+  build:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -37,6 +34,36 @@ jobs:
       - name: Setup NodeJS
         uses: ./.github/actions/node
 
+      - name: Build PR binary and zarf init package
+        uses: ./.github/actions/packages
+        with:
+          build-examples: 'false'
+
+      # Upload the contents of the build directory for later stages to use
+      - name: Upload build artifacts
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+        with:
+          name: build-artifacts
+          path: build/
+          retention-days: 1
+
+  validate:
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Checkout
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+
+      - name: Download build artifacts
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
+        with:
+          name: build-artifacts
+          path: build/
+
+      - name: Make Zarf executable
+        run: |
+          chmod +x build/zarf
+
       - name: Install release version of Zarf
         uses: defenseunicorns/setup-zarf@main
         with:
@@ -50,6 +77,10 @@ jobs:
           sudo env "PATH=$PATH" CI=true zarf init --components k3s,git-server,logging --confirm
           sudo chown $USER /tmp/zarf-*.log
 
+      # Before we run the regular tests we need to aggressively cleanup files to reduce disk pressure
+      - name: Cleanup files
+        uses: ./.github/actions/cleanup-files
+
       - name: Create and deploy the upgrade test packages
         # NOTE: "PATH=$PATH" preserves the default user $PATH. This is needed to maintain the version of zarf installed
         #       in a previous step. This test run will the current release to create a K3s cluster.
@@ -61,9 +92,6 @@ jobs:
           sudo env "PATH=$PATH" CI=true zarf tools kubectl describe pods -n=podinfo-upgrade
           sudo chown $USER /tmp/zarf-*.log
 
-      - name: Build PR binary and zarf packages
-        uses: ./.github/actions/packages
-
       - name: "Run the PR's tests"
         # NOTE: "PATH=$PATH" preserves the default user $PATH. This is needed to maintain the version of go installed
         #       in a previous step. This test run will use this PR's Zarf to create a K3s cluster.
@@ -72,27 +100,24 @@ jobs:
           sudo env "PATH=$PATH" CI=true APPLIANCE_MODE=true APPLIANCE_MODE_KEEP=true make test-e2e ARCH=amd64
           sudo chown $USER /tmp/zarf-*.log
 
-      - name: "Cleanup after running tests"
-        # NOTE: This reduces disk pressure before the upgrade-specific tests begin
+      - name: "Describe nodes, pods and deployments"
+        # NOTE: We describe nodes, pods and deployments here to help understand failures
         run: |
-          lsblk -f
           sudo env "PATH=$PATH" CI=true zarf tools kubectl describe nodes
           sudo env "PATH=$PATH" CI=true zarf tools kubectl describe deployments -n=podinfo-upgrade
           sudo env "PATH=$PATH" CI=true zarf tools kubectl describe pods -n=podinfo-upgrade
 
-          sudo rm -rf zarf-sbom /tmp/zarf-*/ src/ui/node_modules
-          sudo env "PATH=$PATH" CI=true make delete-packages
-          sudo build/zarf tools clear-cache
-          sudo docker system prune --all --force
-
-          lsblk -f
-          sudo env "PATH=$PATH" CI=true zarf tools kubectl describe nodes
+      # Before we run the upgrade tests we need to aggressively cleanup files to reduce disk pressure
+      - name: Cleanup files
+        uses: ./.github/actions/cleanup-files
 
       - name: Run the upgrade tests
         # NOTE: "PATH=$PATH" preserves the default user $PATH. This is needed to maintain the version of zarf installed
         #       in a previous step. This test run will the current release to create a K3s cluster.
         # chown the logs since they were originally created as root
         run: |
+          sudo env "PATH=$PATH" CI=true zarf tools kubectl describe nodes
+
           zarf package create src/test/upgrade --set PODINFO_VERSION=6.3.4 --confirm
 
           sudo env "PATH=$PATH" CI=true make test-upgrade ARCH=amd64

diff --git a/Makefile b/Makefile
@@ -108,6 +108,8 @@ build-local-agent-image: ## Build the Zarf agent image to be used in a locally b
 	@ if [ "$(ARCH)" = "arm64" ] && [ ! -s ./build/zarf-arm ]; then $(MAKE) build-cli-linux-arm; fi
 	@ if [ "$(ARCH)" = "arm64" ]; then cp build/zarf-arm build/zarf-linux-arm64; fi
 	docker buildx build --load --platform linux/$(ARCH) --tag ghcr.io/defenseunicorns/zarf/agent:local .
+	@ if [ "$(ARCH)" = "amd64" ]; then rm build/zarf-linux-amd64; fi
+	@ if [ "$(ARCH)" = "arm64" ]; then rm build/zarf-linux-arm64; fi
 
 init-package: ## Create the zarf init package (must `brew install coreutils` on macOS and have `docker` first)
 	@test -s $(ZARF_BIN) || $(MAKE) build-cli