feat: Add automated test for quick start test (ratify-project#1045)

Signed-off-by: Susan Shi <[email protected]>

.github/workflows/quick-start.yml

@@ -0,0 +1,57 @@
+name: quick-start
+
+on:
+  pull_request_target:
+    types: [labeled]
+  pull_request:
+    branches:
+      - main
+      - 1.0.0*
+  push:
+    branches:
+      - 1.0.0*
+      - main
+  workflow_dispatch:
+
+jobs:
+  test-quick-start:
+    name: "Run quick start test"
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    permissions:
+      contents: read
+    strategy:
+      matrix:
+        KUBERNETES_VERSION: ["1.26.3"]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+      - name: setup go environment
+        uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0
+        with:
+          go-version: "1.20"
+      - name: Run tidy
+        run: go mod tidy
+      - name: Bootstrap e2e
+        run: |
+          mkdir -p $GITHUB_WORKSPACE/bin
+          echo "$GITHUB_WORKSPACE/bin" >> $GITHUB_PATH
+          make e2e-bootstrap KUBERNETES_VERSION=${{ matrix.KUBERNETES_VERSION }}
+          make generate-certs
+      - name: Run e2e with config policy
+        run: |
+          make e2e-helmfile-install
+          make e2e-helmfile-deploy-released-ratify
+          make test-quick-start
+      - name: Save logs
+        if: ${{ always() }}
+        run: |
+          kubectl logs -n gatekeeper-system -l app=ratify --tail=-1 > logs-ratify-preinstall-${{ matrix.KUBERNETES_VERSION }}-config-policy.json
+          kubectl logs -n gatekeeper-system -l app.kubernetes.io/name=ratify --tail=-1 > logs-ratify-${{ matrix.KUBERNETES_VERSION }}-config-policy.json
+      - name: Upload artifacts
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+        if: ${{ always() }}
+        with:
+          name: e2e-logs
+          path: |
+            logs-*.json

Makefile

@@ -21,9 +21,11 @@ NOTATION_VERSION ?= 1.0.0-rc.7
 ORAS_VERSION ?= 1.0.0-rc.2
 
 HELM_VERSION ?= 3.9.2
+HELMFILE_VERSION ?= 0.155.0
 BATS_BASE_TESTS_FILE ?= test/bats/base-test.bats
 BATS_PLUGIN_TESTS_FILE ?= test/bats/plugin-test.bats
 BATS_CLI_TESTS_FILE ?= test/bats/cli-test.bats
+BATS_QUICKSTART_TESTS_FILE ?= test/bats/quickstart-test.bats
 BATS_HA_TESTS_FILE ?= test/bats/high-availability.bats
 BATS_VERSION ?= 1.7.0
 SYFT_VERSION ?= v0.76.0
@@ -143,6 +145,10 @@ test-e2e-cli: e2e-dependencies e2e-create-local-registry e2e-notation-setup e2e-
 	RATIFY_DIR=${INSTALL_DIR} TEST_REGISTRY=${TEST_REGISTRY} ${GITHUB_WORKSPACE}/bin/bats -t ${BATS_CLI_TESTS_FILE}
 	go tool covdata textfmt -i=${GOCOVERDIR} -o test/e2e/coverage.txt
 
+.PHONY: test-quick-start
+test-quick-start:
+	bats -t ${BATS_QUICKSTART_TESTS_FILE}
+
 .PHONY: test-high-availability
 test-high-availability:
 	bats -t ${BATS_HA_TESTS_FILE}
@@ -241,6 +247,12 @@ e2e-helm-install:
 	cd .staging/helm && tar -xvf helmbin.tar.gz
 	./.staging/helm/linux-amd64/helm version --client
 
+e2e-helmfile-install:
+	rm -rf .staging/helmfilebin
+	mkdir -p .staging/helmfilebin
+	curl -L https://github.com/helmfile/helmfile/releases/download/v${HELMFILE_VERSION}/helmfile_${HELMFILE_VERSION}_linux_amd64.tar.gz --output .staging/helmfilebin/helmfilebin.tar.gz
+	cd .staging/helmfilebin && tar -xvf helmfilebin.tar.gz
+
 e2e-docker-credential-store-setup:
 	rm -rf .staging/pass
 	mkdir -p .staging/pass
@@ -460,6 +472,9 @@ e2e-build-local-ratify-image:
 	-t localbuild:test .
 	kind load docker-image --name kind localbuild:test
 
+e2e-helmfile-deploy-released-ratify:
+	curl -L https://raw.githubusercontent.com/deislabs/ratify/main/helmfile.yaml | ./.staging/helmfilebin/helmfile sync -f - 
+
 e2e-helm-deploy-ratify:
 	printf "{\n\t\"auths\": {\n\t\t\"registry:5000\": {\n\t\t\t\"auth\": \"`echo "${TEST_REGISTRY_USERNAME}:${TEST_REGISTRY_PASSWORD}" | tr -d '\n' | base64 -i -w 0`\"\n\t\t}\n\t}\n}" > mount_config.json
 

RELEASES.md

@@ -22,17 +22,18 @@ Example pre-release versions include `v0.1.0-alpha1`, `v0.1.0-beta2`, `v0.1.0-rc
 
 ## Pre Release Activity
 
-Most e2e-scenarios for cli, K8s, and Azure are covered by the Ratify e2e tests. Please refer to this [document](test/validation.md) for the current supported and unsupported tests. 
+1. Most e2e-scenarios for cli, K8s, and Azure are covered by the Ratify e2e tests. Please refer to this [document](test/validation.md) for the current supported and unsupported tests. Please perform manual prerelease validations for the unsupported tests list [here](test/validation.md#unsupported-tests)
 
-Please perform manual prerelease validations for the unsupported tests list [here](test/validation.md#unsupported-tests)
+2. Validate that the format of the data returned for external data calls has not changed. If it has changed update the version in `httpserver/types.go` to reflect a change in the format and document the update.
 
-Validate that the format of the data returned for external data calls has not changed. If it has changed update the version in `httpserver/types.go` to reflect a change in the format and document the update.
+3. Delete all dev images generated since the previous release under the `ratify-dev` and `ratify-crds-dev` packages. Each dev image tag is prefixed with `dev` followed by the date of creation and then the abbreviated 7 character commit SHA (e.g a build generated on March 8, 2023 from main branch with commit SHA `4cf98388ef33c587ef86b82e05cb0f7de2da2ea8` would be tagged `dev.20230308.4cf9838`).
 
-Delete all dev images generated since the previous release under the `ratify-dev` and `ratify-crds-dev` packages. Each dev image tag is prefixed with `dev` followed by the date of creation and then the abbreviated 7 character commit SHA (e.g a build generated on March 8, 2023 from main branch with commit SHA `4cf98388ef33c587ef86b82e05cb0f7de2da2ea8` would be tagged `dev.20230308.4cf9838`).
+4. Copy contents from `dev.helmfile.yaml` to `helmfile.yaml` & `dev.high-availability.helmfile.yaml` to `high-availability.helmfile.yaml`. You MUST update/remove values marked by comments in the files. The `dev` prefixed helmfiles are treated as staging files that are up to date with new changes on main branch. The primary `helmfile.yaml` and `high-availability.helmfile.yaml` MUST stay pinned to the current release since they are used by the quickstarts. Update `dev.helmfile.yaml` & `dev.high-availability.helmfile.yaml` ratify chart version to new release version.
 
-Copy contents from `dev.helmfile.yaml` to `helmfile.yaml` & `dev.high-availability.helmfile.yaml` to `high-availability.helmfile.yaml`. You MUST update/remove values marked by comments in the files. The `dev` prefixed helmfiles are treated as staging files that are up to date with new changes on main branch. The primary `helmfile.yaml` and `high-availability.helmfile.yaml` MUST stay pinned to the current release since they are used by the quickstarts. 
+## Post Release Activity
+
+After a successful release, please manually trigger [quick start action](.github/quick-start.yml) to validate the quick start test is passing. Validate in the run logs that the version of ratify matches the latest released version.
 
-Update `dev.helmfile.yaml` & `dev.high-availability.helmfile.yaml` ratify chart version to new release version
 ## Git Release Flow
 
 This section deals with the practical considerations of versioning in Git, this repo's version control system.  See the semantic versioning specification for the scope of changes allowed for each release type.

test/bats/quickstart-test.bats

@@ -0,0 +1,12 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "validate quick start steps" {
+    run kubectl run demo --image=ghcr.io/deislabs/ratify/notary-image:signed
+    assert_success
+
+    # validate unsigned fails
+    run kubectl run demo1 --image=ghcr.io/deislabs/ratify/notary-image:unsigned
+    assert_failure
+}