From dbb7ad7eeef05f37c144d9cf532c161efb34bcc1 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Wed, 12 Jul 2023 10:18:25 -0500 Subject: [PATCH 1/7] Refactor large workflows into multiple jobs --- .github/actions/cleanup-files/action.yaml | 16 +++++ .github/actions/install-tools/action.yaml | 2 + .github/actions/packages/action.yaml | 23 ++++++- .github/workflows/release.yml | 77 +++++++++++++++-------- .github/workflows/test-bigbang.yml | 6 +- .github/workflows/test-k3s.yml | 3 +- .github/workflows/test-upgrade.yml | 54 +++++++++++----- 7 files changed, 134 insertions(+), 47 deletions(-) create mode 100644 .github/actions/cleanup-files/action.yaml diff --git a/.github/actions/cleanup-files/action.yaml b/.github/actions/cleanup-files/action.yaml new file mode 100644 index 0000000000..1ea4b72ebd --- /dev/null +++ b/.github/actions/cleanup-files/action.yaml @@ -0,0 +1,16 @@ +name: cleanup-files +description: "Cleans up files on the local system to reduce disk pressure" + +runs: + using: composite + steps: + - run: | + lsblk -f + + sudo rm -rf zarf-sbom /tmp/zarf-* src/ui/node_modules + sudo env "PATH=$PATH" CI=true make delete-packages + sudo build/zarf tools clear-cache + sudo docker system prune --all --force + + lsblk -f + shell: bash diff --git a/.github/actions/install-tools/action.yaml b/.github/actions/install-tools/action.yaml index 423e4aeab2..21e346048d 100644 --- a/.github/actions/install-tools/action.yaml +++ b/.github/actions/install-tools/action.yaml @@ -6,6 +6,8 @@ runs: steps: - uses: sigstore/cosign-installer@9becc617647dfa20ae7b1151972e9b3a2c338a2b # v2.8.1 + - uses: anchore/sbom-action/download-syft@78fc58e266e87a38d4194b2137a3d4e9bcaf7ca1 # v0.14.3 + - run: "curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin" shell: bash diff --git a/.github/actions/packages/action.yaml b/.github/actions/packages/action.yaml index a531a1b834..8ea3f42675 100644 --- a/.github/actions/packages/action.yaml +++ b/.github/actions/packages/action.yaml @@ -1,9 +1,28 @@ name: packages -description: "Create agent image, init package and example packages" +description: "Build CLI, agent image, init package and example packages" + +inputs: + init-package: + description: 'Build the init package' + required: false + default: 'true' + build-examples: + description: 'Build the example packages' + required: false + default: 'true' + runs: using: composite steps: - run: | - make build-cli-linux-amd init-package build-examples ARCH=amd64 + make build-cli-linux-amd ARCH=amd64 + shell: bash + - run: | + make init-package ARCH=amd64 + shell: bash + if: ${{ inputs.init-package == 'true' }} + - run: | + make build-examples ARCH=amd64 shell: bash + if: ${{ inputs.build-examples == 'true' }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9e73de4b60..e4cc5f85eb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,12 +9,12 @@ on: - "v*" jobs: - push-resources: + build: runs-on: ubuntu-latest permissions: - contents: write packages: write steps: + # Checkout the repo and setup the tooling for this job - name: Checkout uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 with: @@ -61,19 +61,36 @@ jobs: make release-init-package ARCH=amd64 AGENT_IMAGE_TAG=$GITHUB_REF_NAME make release-init-package ARCH=arm64 AGENT_IMAGE_TAG=$GITHUB_REF_NAME - # Before we run the tests we need to aggressively cleanup files to reduce disk pressure - - name: Cleanup files - run: | - lsblk -f + # Create a CVE report based on this build + - name: Create release time CVE report + run: "make cve-report" - sudo rm -rf zarf-sbom /tmp/zarf-* src/ui/node_modules - sudo build/zarf tools clear-cache - sudo docker system prune --all --force - go clean -cache + # Upload the contents of the build directory for later stages to use + - name: Upload build artifacts + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + with: + name: build-artifacts + path: build/ + retention-days: 1 + + validate: + runs-on: ubuntu-latest + needs: build + steps: + # Checkout the repo and setup the tooling for this job + - name: Checkout + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + fetch-depth: 0 - lsblk -f + - name: Download build artifacts + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: build-artifacts + path: build/ - - name: Run Tests + # Build the example packages and run the tests + - name: Build examples and run tests run: | sudo env "PATH=$PATH" CI=true APPLIANCE_MODE=true make test-e2e ARCH=amd64 sudo chown $USER /tmp/zarf-*.log @@ -82,22 +99,32 @@ jobs: if: always() uses: ./.github/actions/save-logs - # Builds init packages since GoReleaser won't handle this for us - - name: Create release time CVE report - run: "make cve-report" + push: + runs-on: ubuntu-latest + needs: test + permissions: + contents: write + steps: + # Checkout the repo and setup the tooling for this job + - name: Checkout + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + fetch-depth: 0 - # Before we run GoReleaser we need to (again) aggressively cleanup files to reduce disk pressure - - name: Cleanup files - run: | - lsblk -f + - name: Setup golang + uses: ./.github/actions/golang + + - name: Setup NodeJS + uses: ./.github/actions/node - sudo rm -rf zarf-sbom /tmp/zarf-* - sudo env "PATH=$PATH" CI=true make delete-packages - sudo build/zarf tools clear-cache - sudo docker system prune --all --force - go clean -cache + - name: Install tools + uses: ./.github/actions/install-tools - lsblk -f + - name: Download build artifacts + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: build-artifacts + path: build/ # Set up AWS credentials for GoReleaser to upload backups of artifacts to S3 - name: Set AWS Credentials diff --git a/.github/workflows/test-bigbang.yml b/.github/workflows/test-bigbang.yml index 69aae2c131..d8f252c3b9 100644 --- a/.github/workflows/test-bigbang.yml +++ b/.github/workflows/test-bigbang.yml @@ -34,13 +34,15 @@ jobs: - name: Setup NodeJS uses: ./.github/actions/node - - name: Build binary and zarf packages + - name: Build Zarf binary uses: ./.github/actions/packages + with: + build-examples: 'false' - name: Setup K3d uses: ./.github/actions/k3d - - name: "Login to Iron Bank" + - name: Login to Iron Bank uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2.2.0 if: ${{ env.IRONBANK_USERNAME != '' }} env: diff --git a/.github/workflows/test-k3s.yml b/.github/workflows/test-k3s.yml index a989fd0b49..c498996860 100644 --- a/.github/workflows/test-k3s.yml +++ b/.github/workflows/test-k3s.yml @@ -41,9 +41,10 @@ jobs: # NOTE: "PATH=$PATH" preserves the default user $PATH. This is needed to maintain the version of go installed # in a previous step. This test run will use Zarf to create a K3s cluster, and a brand new cluster will be # used for each test - # chown the logs since they were orignally created as root + # chown the logs since they were originally created as root run: | sudo env "PATH=$PATH" CI=true APPLIANCE_MODE=true make test-e2e ARCH=amd64 + sudo chown $USER /tmp/zarf-*.log - name: Save logs if: always() diff --git a/.github/workflows/test-upgrade.yml b/.github/workflows/test-upgrade.yml index 1d390346ec..d71a025910 100644 --- a/.github/workflows/test-upgrade.yml +++ b/.github/workflows/test-upgrade.yml @@ -22,10 +22,7 @@ concurrency: cancel-in-progress: true jobs: - validate: - env: - # Reduce the kubelet eviction minimums reduce the chance for disk pressure causing evictions during the tests - ZARF_PACKAGE_DEPLOY_SET_K3S_ARGS: --disable traefik --kubelet-arg "eviction-hard=imagefs.available<1%,nodefs.available<1%" --kubelet-arg "eviction-minimum-reclaim=imagefs.available=1%,nodefs.available=1%" + build: runs-on: ubuntu-latest steps: - name: Checkout @@ -42,6 +39,35 @@ jobs: with: download-init-package: true + - name: Build PR binary and zarf init package + uses: ./.github/actions/packages + with: + build-examples: 'false' + + # Upload the contents of the build directory for later stages to use + - name: Upload build artifacts + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + with: + name: build-artifacts + path: build/ + retention-days: 1 + + validate: + runs-on: ubuntu-latest + needs: build + env: + # Reduce the kubelet eviction minimums reduce the chance for disk pressure causing evictions during the tests + ZARF_PACKAGE_DEPLOY_SET_K3S_ARGS: --disable traefik --kubelet-arg "eviction-hard=imagefs.available<1%,nodefs.available<1%" --kubelet-arg "eviction-minimum-reclaim=imagefs.available=1%,nodefs.available=1%" + steps: + - name: Checkout + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + + - name: Download build artifacts + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: build-artifacts + path: build/ + - name: Initialize the cluster with the release version # NOTE: "PATH=$PATH" preserves the default user $PATH. This is needed to maintain the version of zarf installed # in a previous step. This test run will the current release to create a K3s cluster. @@ -61,9 +87,6 @@ jobs: sudo env "PATH=$PATH" CI=true zarf tools kubectl describe pods -n=podinfo-upgrade sudo chown $USER /tmp/zarf-*.log - - name: Build PR binary and zarf packages - uses: ./.github/actions/packages - - name: "Run the PR's tests" # NOTE: "PATH=$PATH" preserves the default user $PATH. This is needed to maintain the version of go installed # in a previous step. This test run will use this PR's Zarf to create a K3s cluster. @@ -72,27 +95,24 @@ jobs: sudo env "PATH=$PATH" CI=true APPLIANCE_MODE=true APPLIANCE_MODE_KEEP=true make test-e2e ARCH=amd64 sudo chown $USER /tmp/zarf-*.log - - name: "Cleanup after running tests" - # NOTE: This reduces disk pressure before the upgrade-specific tests begin + - name: "Describe nodes, pods and deployments" + # NOTE: We describe nodes, pods and deployments here to help understand failures run: | - lsblk -f sudo env "PATH=$PATH" CI=true zarf tools kubectl describe nodes sudo env "PATH=$PATH" CI=true zarf tools kubectl describe deployments -n=podinfo-upgrade sudo env "PATH=$PATH" CI=true zarf tools kubectl describe pods -n=podinfo-upgrade - sudo rm -rf zarf-sbom /tmp/zarf-*/ src/ui/node_modules - sudo env "PATH=$PATH" CI=true make delete-packages - sudo build/zarf tools clear-cache - sudo docker system prune --all --force - - lsblk -f - sudo env "PATH=$PATH" CI=true zarf tools kubectl describe nodes + # Before we run the upgrade tests we need to aggressively cleanup files to reduce disk pressure + - name: Cleanup files + uses: ./.github/actions/cleanup-files - name: Run the upgrade tests # NOTE: "PATH=$PATH" preserves the default user $PATH. This is needed to maintain the version of zarf installed # in a previous step. This test run will the current release to create a K3s cluster. # chown the logs since they were originally created as root run: | + sudo env "PATH=$PATH" CI=true zarf tools kubectl describe nodes + zarf package create src/test/upgrade --set PODINFO_VERSION=6.3.4 --confirm sudo env "PATH=$PATH" CI=true make test-upgrade ARCH=amd64 From 94e9c4c6e0b6a844d6239ec22424cad1443bac7f Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Wed, 12 Jul 2023 10:22:52 -0500 Subject: [PATCH 2/7] Remove kubelet hack from upgrade workflow --- .github/workflows/test-upgrade.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/test-upgrade.yml b/.github/workflows/test-upgrade.yml index d71a025910..379ca7dfa0 100644 --- a/.github/workflows/test-upgrade.yml +++ b/.github/workflows/test-upgrade.yml @@ -55,9 +55,6 @@ jobs: validate: runs-on: ubuntu-latest needs: build - env: - # Reduce the kubelet eviction minimums reduce the chance for disk pressure causing evictions during the tests - ZARF_PACKAGE_DEPLOY_SET_K3S_ARGS: --disable traefik --kubelet-arg "eviction-hard=imagefs.available<1%,nodefs.available<1%" --kubelet-arg "eviction-minimum-reclaim=imagefs.available=1%,nodefs.available=1%" steps: - name: Checkout uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 From ebc7e6f3b416ddff1b6c469ef52868d0c566d8e2 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Wed, 12 Jul 2023 10:28:11 -0500 Subject: [PATCH 3/7] Fixup cleanup of files --- .github/workflows/test-upgrade.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-upgrade.yml b/.github/workflows/test-upgrade.yml index 379ca7dfa0..76d3f2861c 100644 --- a/.github/workflows/test-upgrade.yml +++ b/.github/workflows/test-upgrade.yml @@ -34,11 +34,6 @@ jobs: - name: Setup NodeJS uses: ./.github/actions/node - - name: Install release version of Zarf - uses: defenseunicorns/setup-zarf@main - with: - download-init-package: true - - name: Build PR binary and zarf init package uses: ./.github/actions/packages with: @@ -65,6 +60,11 @@ jobs: name: build-artifacts path: build/ + - name: Install release version of Zarf + uses: defenseunicorns/setup-zarf@main + with: + download-init-package: true + - name: Initialize the cluster with the release version # NOTE: "PATH=$PATH" preserves the default user $PATH. This is needed to maintain the version of zarf installed # in a previous step. This test run will the current release to create a K3s cluster. @@ -73,6 +73,10 @@ jobs: sudo env "PATH=$PATH" CI=true zarf init --components k3s,git-server,logging --confirm sudo chown $USER /tmp/zarf-*.log + # Before we run the regular tests we need to aggressively cleanup files to reduce disk pressure + - name: Cleanup files + uses: ./.github/actions/cleanup-files + - name: Create and deploy the upgrade test packages # NOTE: "PATH=$PATH" preserves the default user $PATH. This is needed to maintain the version of zarf installed # in a previous step. This test run will the current release to create a K3s cluster. From 2b0778448cacbd6d76fb2265d0f96eb89261e1d1 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Wed, 12 Jul 2023 10:32:51 -0500 Subject: [PATCH 4/7] Remove extra zarf copies from build --- .github/workflows/release.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e4cc5f85eb..eb074d9d98 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -46,6 +46,8 @@ jobs: cp build/zarf build/zarf-linux-amd64 cp build/zarf-arm build/zarf-linux-arm64 docker buildx build --push --platform linux/arm64/v8,linux/amd64 --tag ghcr.io/defenseunicorns/zarf/agent:$GITHUB_REF_NAME . + rm build/zarf-linux-amd64 + rm build/zarf-linux-arm64 - name: "Zarf Agent: Sign the Image" run: cosign sign --key awskms:///${{ secrets.COSIGN_AWS_KMS_KEY }} -a release-engineer=https://github.com/${{ github.actor }} -a version=$GITHUB_REF_NAME ghcr.io/defenseunicorns/zarf/agent:$GITHUB_REF_NAME From 5a227cdf09e2be9ae7e32f9d026c865c13d0fbec Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Wed, 12 Jul 2023 10:51:25 -0500 Subject: [PATCH 5/7] temporarily print the directory contents --- .github/workflows/test-upgrade.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test-upgrade.yml b/.github/workflows/test-upgrade.yml index 76d3f2861c..cd05531714 100644 --- a/.github/workflows/test-upgrade.yml +++ b/.github/workflows/test-upgrade.yml @@ -59,6 +59,11 @@ jobs: with: name: build-artifacts path: build/ + + - name: Temp print dir contents + run: | + ls -la + ls -la build/ - name: Install release version of Zarf uses: defenseunicorns/setup-zarf@main From 28cf2e45ff582647d45af22d989ea6a49e1e07c1 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Wed, 12 Jul 2023 11:04:45 -0500 Subject: [PATCH 6/7] Make zarf executable --- .github/workflows/release.yml | 4 ++++ .github/workflows/test-upgrade.yml | 7 +++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index eb074d9d98..62f920432c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -91,6 +91,10 @@ jobs: name: build-artifacts path: build/ + - name: Make Zarf executable + run: | + chmod +x build/zarf + # Build the example packages and run the tests - name: Build examples and run tests run: | diff --git a/.github/workflows/test-upgrade.yml b/.github/workflows/test-upgrade.yml index cd05531714..bf919c7e39 100644 --- a/.github/workflows/test-upgrade.yml +++ b/.github/workflows/test-upgrade.yml @@ -59,11 +59,10 @@ jobs: with: name: build-artifacts path: build/ - - - name: Temp print dir contents + + - name: Make Zarf executable run: | - ls -la - ls -la build/ + chmod +x build/zarf - name: Install release version of Zarf uses: defenseunicorns/setup-zarf@main From ba637236cb07301cbe68f5bc4f0c0927ac86fdfc Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Wed, 12 Jul 2023 11:33:14 -0500 Subject: [PATCH 7/7] Cleanup the built local versions of the agent image binary --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 7f411e207d..9cf40a491a 100644 --- a/Makefile +++ b/Makefile @@ -108,6 +108,8 @@ build-local-agent-image: ## Build the Zarf agent image to be used in a locally b @ if [ "$(ARCH)" = "arm64" ] && [ ! -s ./build/zarf-arm ]; then $(MAKE) build-cli-linux-arm; fi @ if [ "$(ARCH)" = "arm64" ]; then cp build/zarf-arm build/zarf-linux-arm64; fi docker buildx build --load --platform linux/$(ARCH) --tag ghcr.io/defenseunicorns/zarf/agent:local . + @ if [ "$(ARCH)" = "amd64" ]; then rm build/zarf-linux-amd64; fi + @ if [ "$(ARCH)" = "arm64" ]; then rm build/zarf-linux-arm64; fi init-package: ## Create the zarf init package (must `brew install coreutils` on macOS and have `docker` first) @test -s $(ZARF_BIN) || $(MAKE) build-cli