From 1a7e79e3bdd62aa59dbd0b6a319d8de4a0000c1a Mon Sep 17 00:00:00 2001 From: "huish@microsoft.com" Date: Wed, 30 Aug 2023 01:18:07 +0000 Subject: [PATCH 01/18] init setup --- .github/workflows/quick-start.yml | 40 +++++++++++++++++++++++++++++++ Makefile | 9 +++++++ 2 files changed, 49 insertions(+) create mode 100644 .github/workflows/quick-start.yml diff --git a/.github/workflows/quick-start.yml b/.github/workflows/quick-start.yml new file mode 100644 index 000000000..0ca37f10c --- /dev/null +++ b/.github/workflows/quick-start.yml @@ -0,0 +1,40 @@ +name: quick-start + +on: + workflow_dispatch: + +jobs: + test-quick-start: + name: "Run quick start test" + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + contents: write + strategy: + matrix: + KUBERNETES_VERSION: ["1.25.8", "1.26.3"] + GATEKEEPER_VERSION: ["3.11.0", "3.12.0"] + steps: + - name: Bootstrap e2e + run: | + mkdir -p $GITHUB_WORKSPACE/bin + echo "$GITHUB_WORKSPACE/bin" >> $GITHUB_PATH + make e2e-bootstrap KUBERNETES_VERSION=${{ matrix.KUBERNETES_VERSION }} + make generate-certs + - name: Run e2e with config policy + run: | + make e2e-deploy-helmfile + make e2e-helmfile-deploy-released-ratify + make test-e2e GATEKEEPER_VERSION=${{ matrix.GATEKEEPER_VERSION }} + - name: Save logs + if: ${{ always() }} + run: | + kubectl logs -n gatekeeper-system -l app=ratify --tail=-1 > logs-ratify-preinstall-${{ matrix.KUBERNETES_VERSION }}-${{ matrix.GATEKEEPER_VERSION }}-config-policy.json + kubectl logs -n gatekeeper-system -l app.kubernetes.io/name=ratify --tail=-1 > logs-ratify-${{ matrix.KUBERNETES_VERSION }}-${{ matrix.GATEKEEPER_VERSION }}-config-policy.json + - name: Upload artifacts + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + if: ${{ always() }} + with: + name: e2e-logs + path: | + logs-*.json diff --git a/Makefile b/Makefile index f7501e787..a98042fcc 100644 --- a/Makefile +++ b/Makefile @@ -241,6 +241,12 @@ e2e-helm-install: cd .staging/helm && tar -xvf helmbin.tar.gz ./.staging/helm/linux-amd64/helm version --client +e2e-helmfile-install: + rm -rf .staging/helm-file + mkdir .staging/helm-file + curl -LO https://github.com/helmfile/helmfile/releases/download/v0.155.0/helmfile_0.155.0_linux_amd64.tar.gz --output .staging/helm-file/helmfilebin.tar.gz + cd .staging/helm-file && tar -xvf helmfile*.tar.gz + e2e-docker-credential-store-setup: rm -rf .staging/pass mkdir -p .staging/pass @@ -458,6 +464,9 @@ e2e-build-local-ratify-image: -t localbuild:test . kind load docker-image --name kind localbuild:test +e2e-helmfile-deploy-released-ratify: + curl -L https://raw.githubusercontent.com/deislabs/ratify/main/helmfile.yaml | ./.staging/helm-file/helmfile sync -f - + e2e-helm-deploy-ratify: printf "{\n\t\"auths\": {\n\t\t\"registry:5000\": {\n\t\t\t\"auth\": \"`echo "${TEST_REGISTRY_USERNAME}:${TEST_REGISTRY_PASSWORD}" | tr -d '\n' | base64 -i -w 0`\"\n\t\t}\n\t}\n}" > mount_config.json From c28b7fa3e7de421e8d07009d381d3a37c58d5009 Mon Sep 17 00:00:00 2001 From: "huish@microsoft.com" Date: Wed, 30 Aug 2023 01:52:28 +0000 Subject: [PATCH 02/18] add quick start test --- .github/workflows/quick-start.yml | 6 +----- Makefile | 5 +++++ test/bats/quickstart-test.bats | 19 +++++++++++++++++++ 3 files changed, 25 insertions(+), 5 deletions(-) create mode 100644 test/bats/quickstart-test.bats diff --git a/.github/workflows/quick-start.yml b/.github/workflows/quick-start.yml index 0ca37f10c..3a8f99949 100644 --- a/.github/workflows/quick-start.yml +++ b/.github/workflows/quick-start.yml @@ -10,10 +10,6 @@ jobs: timeout-minutes: 60 permissions: contents: write - strategy: - matrix: - KUBERNETES_VERSION: ["1.25.8", "1.26.3"] - GATEKEEPER_VERSION: ["3.11.0", "3.12.0"] steps: - name: Bootstrap e2e run: | @@ -25,7 +21,7 @@ jobs: run: | make e2e-deploy-helmfile make e2e-helmfile-deploy-released-ratify - make test-e2e GATEKEEPER_VERSION=${{ matrix.GATEKEEPER_VERSION }} + make test-quick-start - name: Save logs if: ${{ always() }} run: | diff --git a/Makefile b/Makefile index a98042fcc..3d224aa4c 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,7 @@ HELM_VERSION ?= 3.9.2 BATS_BASE_TESTS_FILE ?= test/bats/base-test.bats BATS_PLUGIN_TESTS_FILE ?= test/bats/plugin-test.bats BATS_CLI_TESTS_FILE ?= test/bats/cli-test.bats +BATS_QUICKSTART_TESTS_FILE ?= test/bats/quickstart-test.bats BATS_HA_TESTS_FILE ?= test/bats/high-availability.bats BATS_VERSION ?= 1.7.0 SYFT_VERSION ?= v0.76.0 @@ -143,6 +144,10 @@ test-e2e-cli: e2e-dependencies e2e-create-local-registry e2e-notation-setup e2e- RATIFY_DIR=${INSTALL_DIR} TEST_REGISTRY=${TEST_REGISTRY} ${GITHUB_WORKSPACE}/bin/bats -t ${BATS_CLI_TESTS_FILE} go tool covdata textfmt -i=${GOCOVERDIR} -o test/e2e/coverage.txt +.PHONY: test-quick-start +test-quick-start: + bats -t ${BATS_QUICKSTART_TESTS_FILE} + .PHONY: test-high-availability test-high-availability: bats -t ${BATS_HA_TESTS_FILE} diff --git a/test/bats/quickstart-test.bats b/test/bats/quickstart-test.bats new file mode 100644 index 000000000..1c6c9c4b9 --- /dev/null +++ b/test/bats/quickstart-test.bats @@ -0,0 +1,19 @@ +#!/usr/bin/env bats + +load helpers + +BATS_TESTS_DIR=${BATS_TESTS_DIR:-test/bats/tests} +WAIT_TIME=60 +SLEEP_TIME=1 + +@test "base test without cert rotator" { + teardown() { + echo "cleaning up" + wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete pod demo --namespace default --force --ignore-not-found=true' + } + run kubectl run demo --image=ghcr.io/deislabs/ratify/notary-image:signed + assert_success + + # validate certificate store status property shows success + assert_failure +} From 6f85f080837cb42510bfc0fef67389df90d7b891 Mon Sep 17 00:00:00 2001 From: "huish@microsoft.com" Date: Wed, 30 Aug 2023 03:26:52 +0000 Subject: [PATCH 03/18] add unsigned --- test/bats/quickstart-test.bats | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/bats/quickstart-test.bats b/test/bats/quickstart-test.bats index 1c6c9c4b9..7d6bf47eb 100644 --- a/test/bats/quickstart-test.bats +++ b/test/bats/quickstart-test.bats @@ -14,6 +14,7 @@ SLEEP_TIME=1 run kubectl run demo --image=ghcr.io/deislabs/ratify/notary-image:signed assert_success - # validate certificate store status property shows success + # validate unsigned fails + kubectl run demo1 --image=ghcr.io/deislabs/ratify/notary-image:unsigned assert_failure } From 19ea443978e3627f6a7a7ef712beba56bea17f47 Mon Sep 17 00:00:00 2001 From: "huish@microsoft.com" Date: Fri, 1 Sep 2023 03:26:32 +0000 Subject: [PATCH 04/18] health prob --- charts/ratify/templates/deployment.yaml | 11 +++++++++++ pkg/manager/manager.go | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/charts/ratify/templates/deployment.yaml b/charts/ratify/templates/deployment.yaml index f448d41b5..f8cbc2296 100644 --- a/charts/ratify/templates/deployment.yaml +++ b/charts/ratify/templates/deployment.yaml @@ -38,6 +38,14 @@ spec: - name: {{ .Chart.Name }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} + livenessProbe: + httpGet: + path: /healthz + port: 9090 + readinessProbe: + httpGet: + path: /readyz + port: 9090 securityContext: allowPrivilegeEscalation: false capabilities: @@ -75,6 +83,9 @@ spec: {{- if .Values.instrumentation.metricsEnabled }} - containerPort: {{ required "You must provide .Values.instrumentation.metricsPort" .Values.instrumentation.metricsPort }} {{- end }} + - containerPort: 9090 + name: healthz + protocol: TCP volumeMounts: {{- if .Values.cosign.enabled }} - mountPath: "/usr/local/ratify-certs/cosign" diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 745928464..d01fa5dd1 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -147,7 +147,7 @@ func StartManager(certRotatorReady chan struct{}) { var enableLeaderElection bool var probeAddr string flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") - flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + flag.StringVar(&probeAddr, "health-probe-bind-address", ":9090", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") From e6374d46d5c4e3f3dc0d28f0d4c64ce986b94a68 Mon Sep 17 00:00:00 2001 From: "huish@microsoft.com" Date: Tue, 5 Sep 2023 01:24:02 +0000 Subject: [PATCH 05/18] adding delay --- pkg/manager/manager.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index d01fa5dd1..e6b732c8a 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -266,4 +266,10 @@ func StartManager(certRotatorReady chan struct{}) { setupLog.Error(err, "problem running manager") os.Exit(1) } + + sec, _ := time.ParseDuration("30s") + setupLog.Info("about to sleep for 30sec") + time.Sleep(sec) + setupLog.Info("exiting") + os.Exit(1) } From 6c1b5db6766b889efd784c0198178ad2dc2785fa Mon Sep 17 00:00:00 2001 From: "huish@microsoft.com" Date: Tue, 5 Sep 2023 02:52:38 +0000 Subject: [PATCH 06/18] sleeping --- pkg/manager/manager.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index e6b732c8a..d02d3f8da 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -261,15 +261,16 @@ func StartManager(certRotatorReady chan struct{}) { os.Exit(1) } + sec, _ := time.ParseDuration("30s") + setupLog.Info("about to sleep for 30sec") + time.Sleep(sec) + setupLog.Info("exiting") + os.Exit(1) + setupLog.Info("starting manager") if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { setupLog.Error(err, "problem running manager") os.Exit(1) } - sec, _ := time.ParseDuration("30s") - setupLog.Info("about to sleep for 30sec") - time.Sleep(sec) - setupLog.Info("exiting") - os.Exit(1) } From 05a0c764c001ed42b0316077d9924d64eed0ca9c Mon Sep 17 00:00:00 2001 From: "huish@microsoft.com" Date: Tue, 5 Sep 2023 23:40:05 +0000 Subject: [PATCH 07/18] adding default --- cmd/ratify/cmd/serve.go | 4 +++- httpserver/server.go | 1 + pkg/manager/manager.go | 5 ++--- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cmd/ratify/cmd/serve.go b/cmd/ratify/cmd/serve.go index 4f011cf32..65ebb513c 100644 --- a/cmd/ratify/cmd/serve.go +++ b/cmd/ratify/cmd/serve.go @@ -47,6 +47,7 @@ type serveCmdOptions struct { metricsEnabled bool metricsType string metricsPort int + healthPort string } func NewCmdServe(_ ...string) *cobra.Command { @@ -77,6 +78,7 @@ func NewCmdServe(_ ...string) *cobra.Command { flags.BoolVar(&opts.metricsEnabled, "metrics-enabled", false, "Enable metrics exporter if enabled (default: false)") flags.StringVar(&opts.metricsType, "metrics-type", httpserver.DefaultMetricsType, fmt.Sprintf("Metrics exporter type to use (default: %s)", httpserver.DefaultMetricsType)) flags.IntVar(&opts.metricsPort, "metrics-port", httpserver.DefaultMetricsPort, fmt.Sprintf("Metrics exporter port to use (default: %d)", httpserver.DefaultMetricsPort)) + flags.StringVar(&opts.healthPort, "health-port", httpserver.DefaultHealthPort, fmt.Sprintf("Health port to use (default: %s)", httpserver.DefaultHealthPort)) return cmd } @@ -100,7 +102,7 @@ func serve(opts serveCmdOptions) error { if opts.enableCrdManager { certRotatorReady := make(chan struct{}) logrus.Infof("starting crd manager") - go manager.StartManager(certRotatorReady) + go manager.StartManager(certRotatorReady, opts.healthPort) manager.StartServer(opts.httpServerAddress, opts.configFilePath, opts.certDirectory, opts.caCertFile, opts.cacheTTL, opts.metricsEnabled, opts.metricsType, opts.metricsPort, certRotatorReady) return nil diff --git a/httpserver/server.go b/httpserver/server.go index 54c0d4d51..e95003687 100644 --- a/httpserver/server.go +++ b/httpserver/server.go @@ -46,6 +46,7 @@ const ( DefaultMetricsType = "prometheus" DefaultMetricsPort = 8888 + DefaultHealthPort = ":9090" ) type Server struct { diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index d02d3f8da..6549d848b 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -142,12 +142,11 @@ func StartServer(httpServerAddress, configFilePath, certDirectory, caCertFile st } } -func StartManager(certRotatorReady chan struct{}) { +func StartManager(certRotatorReady chan struct{}, probeAddr string) { var metricsAddr string var enableLeaderElection bool - var probeAddr string + flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") - flag.StringVar(&probeAddr, "health-probe-bind-address", ":9090", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") From 600908f61fcad973c2494db4d9c08b5a9b43c578 Mon Sep 17 00:00:00 2001 From: "huish@microsoft.com" Date: Tue, 5 Sep 2023 23:56:15 +0000 Subject: [PATCH 08/18] adding debug log --- pkg/manager/manager.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 6549d848b..048f1fd35 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -178,6 +178,9 @@ func StartManager(certRotatorReady chan struct{}, probeAddr string) { os.Exit(1) } + setupLog.Debugf("setting up probeAddr at %s", probeAddr) + setupLog.Infof("setting up probeAddr at ", probeAddr) + // Make sure certs are generated and valid if cert rotation is enabled. if featureflag.CertRotation.Enabled { // Make sure TLS cert watcher is already set up. @@ -260,11 +263,13 @@ func StartManager(certRotatorReady chan struct{}, probeAddr string) { os.Exit(1) } - sec, _ := time.ParseDuration("30s") - setupLog.Info("about to sleep for 30sec") - time.Sleep(sec) - setupLog.Info("exiting") - os.Exit(1) + if logrus.GetLevel() == logrus.DebugLevel { + sec, _ := time.ParseDuration("30s") + setupLog.Info("about to sleep for 30sec") + time.Sleep(sec) + setupLog.Info("exiting") + os.Exit(1) + } setupLog.Info("starting manager") if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { From e537ccd2c11d611ed6ed80e171529c942a464ac5 Mon Sep 17 00:00:00 2001 From: "huish@microsoft.com" Date: Wed, 6 Sep 2023 01:05:56 +0000 Subject: [PATCH 09/18] update deployment --- charts/ratify/templates/deployment.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/charts/ratify/templates/deployment.yaml b/charts/ratify/templates/deployment.yaml index f8cbc2296..f61c5d428 100644 --- a/charts/ratify/templates/deployment.yaml +++ b/charts/ratify/templates/deployment.yaml @@ -41,11 +41,11 @@ spec: livenessProbe: httpGet: path: /healthz - port: 9090 + port: {{ .Values.healthPort }} readinessProbe: httpGet: path: /readyz - port: 9090 + port: {{ .Values.healthPort }} securityContext: allowPrivilegeEscalation: false capabilities: @@ -78,14 +78,15 @@ spec: - --metrics-enabled={{ .Values.instrumentation.metricsEnabled }} - --metrics-type={{ .Values.instrumentation.metricsType }} - --metrics-port={{ .Values.instrumentation.metricsPort }} + - --health-port=:{{ .Values.healthPort }} ports: - containerPort: 6001 {{- if .Values.instrumentation.metricsEnabled }} - containerPort: {{ required "You must provide .Values.instrumentation.metricsPort" .Values.instrumentation.metricsPort }} {{- end }} - - containerPort: 9090 - name: healthz - protocol: TCP + - containerPort: {{ .Values.healthPort }} + name: healthz + protocol: TCP volumeMounts: {{- if .Values.cosign.enabled }} - mountPath: "/usr/local/ratify-certs/cosign" From 2b34e88167712a0eb00a4e247406d012c51dd8a4 Mon Sep 17 00:00:00 2001 From: "huish@microsoft.com" Date: Wed, 6 Sep 2023 01:19:35 +0000 Subject: [PATCH 10/18] merge conflict --- .github/workflows/quick-start.yml | 37 +++++++++++++++++++------ Makefile | 17 +++++++----- charts/ratify/templates/deployment.yaml | 2 +- charts/ratify/values.yaml | 1 + test/bats/quickstart-test.bats | 14 ++-------- 5 files changed, 44 insertions(+), 27 deletions(-) diff --git a/.github/workflows/quick-start.yml b/.github/workflows/quick-start.yml index 3a8f99949..e4642c021 100644 --- a/.github/workflows/quick-start.yml +++ b/.github/workflows/quick-start.yml @@ -1,7 +1,17 @@ name: quick-start -on: - workflow_dispatch: +on: + pull_request_target: + types: [labeled] + pull_request: + branches: + - main + - 1.0.0* + push: + branches: + - 1.0.0* + - main + workflow_dispatch: jobs: test-quick-start: @@ -9,8 +19,19 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 60 permissions: - contents: write - steps: + contents: read + strategy: + matrix: + KUBERNETES_VERSION: ["1.26.3"] + steps: + - name: Checkout + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 + - name: setup go environment + uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0 + with: + go-version: "1.20" + - name: Run tidy + run: go mod tidy - name: Bootstrap e2e run: | mkdir -p $GITHUB_WORKSPACE/bin @@ -19,18 +40,18 @@ jobs: make generate-certs - name: Run e2e with config policy run: | - make e2e-deploy-helmfile + make e2e-helmfile-install make e2e-helmfile-deploy-released-ratify make test-quick-start - name: Save logs if: ${{ always() }} run: | - kubectl logs -n gatekeeper-system -l app=ratify --tail=-1 > logs-ratify-preinstall-${{ matrix.KUBERNETES_VERSION }}-${{ matrix.GATEKEEPER_VERSION }}-config-policy.json - kubectl logs -n gatekeeper-system -l app.kubernetes.io/name=ratify --tail=-1 > logs-ratify-${{ matrix.KUBERNETES_VERSION }}-${{ matrix.GATEKEEPER_VERSION }}-config-policy.json + kubectl logs -n gatekeeper-system -l app=ratify --tail=-1 > logs-ratify-preinstall-${{ matrix.KUBERNETES_VERSION }}-config-policy.json + kubectl logs -n gatekeeper-system -l app.kubernetes.io/name=ratify --tail=-1 > logs-ratify-${{ matrix.KUBERNETES_VERSION }}-config-policy.json - name: Upload artifacts uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 if: ${{ always() }} with: name: e2e-logs path: | - logs-*.json + logs-*.json \ No newline at end of file diff --git a/Makefile b/Makefile index a1ac5ddf4..0740c5020 100644 --- a/Makefile +++ b/Makefile @@ -14,13 +14,14 @@ LDFLAGS += -X $(GO_PKG)/internal/version.GitTag=$(GIT_TAG) KIND_VERSION ?= 0.14.0 KUBERNETES_VERSION ?= 1.26.3 -GATEKEEPER_VERSION ?= 3.12.0 +GATEKEEPER_VERSION ?= 3.13.0 DAPR_VERSION ?= 1.11.1 COSIGN_VERSION ?= 1.13.1 NOTATION_VERSION ?= 1.0.0-rc.7 ORAS_VERSION ?= 1.0.0-rc.2 HELM_VERSION ?= 3.9.2 +HELMFILE_VERSION ?= 0.155.0 BATS_BASE_TESTS_FILE ?= test/bats/base-test.bats BATS_PLUGIN_TESTS_FILE ?= test/bats/plugin-test.bats BATS_CLI_TESTS_FILE ?= test/bats/cli-test.bats @@ -247,10 +248,10 @@ e2e-helm-install: ./.staging/helm/linux-amd64/helm version --client e2e-helmfile-install: - rm -rf .staging/helm-file - mkdir .staging/helm-file - curl -LO https://github.com/helmfile/helmfile/releases/download/v0.155.0/helmfile_0.155.0_linux_amd64.tar.gz --output .staging/helm-file/helmfilebin.tar.gz - cd .staging/helm-file && tar -xvf helmfile*.tar.gz + rm -rf .staging/helmfilebin + mkdir -p .staging/helmfilebin + curl -L https://github.com/helmfile/helmfile/releases/download/v${HELMFILE_VERSION}/helmfile_${HELMFILE_VERSION}_linux_amd64.tar.gz --output .staging/helmfilebin/helmfilebin.tar.gz + cd .staging/helmfilebin && tar -xvf helmfilebin.tar.gz e2e-docker-credential-store-setup: rm -rf .staging/pass @@ -425,6 +426,8 @@ e2e-deploy-gatekeeper: e2e-helm-install --set mutatingWebhookTimeoutSeconds=2 \ --set auditInterval=0 + if [ ${GATEKEEPER_VERSION} = "3.13.0" ]; then kubectl -n ${GATEKEEPER_NAMESPACE} patch deployment gatekeeper-controller-manager --type=json -p='[{"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--external-data-provider-response-cache-ttl=1s"}]' && sleep 60; fi + e2e-build-crd-image: docker build --progress=plain --no-cache --build-arg KUBE_VERSION=${KUBERNETES_VERSION} --build-arg TARGETOS="linux" --build-arg TARGETARCH="amd64" -f crd.Dockerfile -t localbuildcrd:test ./charts/ratify/crds kind load docker-image --name kind localbuildcrd:test @@ -470,7 +473,7 @@ e2e-build-local-ratify-image: kind load docker-image --name kind localbuild:test e2e-helmfile-deploy-released-ratify: - curl -L https://raw.githubusercontent.com/deislabs/ratify/main/helmfile.yaml | ./.staging/helm-file/helmfile sync -f - + curl -L https://raw.githubusercontent.com/deislabs/ratify/main/helmfile.yaml | ./.staging/helmfilebin/helmfile sync -f - e2e-helm-deploy-ratify: printf "{\n\t\"auths\": {\n\t\t\"registry:5000\": {\n\t\t\t\"auth\": \"`echo "${TEST_REGISTRY_USERNAME}:${TEST_REGISTRY_PASSWORD}" | tr -d '\n' | base64 -i -w 0`\"\n\t\t}\n\t}\n}" > mount_config.json @@ -641,4 +644,4 @@ $(CONTROLLER_GEN): $(LOCALBIN) .PHONY: conversion-gen conversion-gen: $(CONVERSION_GEN) ## Download conversion-gen locally if necessary. $(CONVERSION_GEN): $(LOCALBIN) - test -s $(LOCALBIN)/conversion-gen || GOBIN=$(LOCALBIN) go install k8s.io/code-generator/cmd/conversion-gen@$(CONVERSION_TOOLS_VERSION) + test -s $(LOCALBIN)/conversion-gen || GOBIN=$(LOCALBIN) go install k8s.io/code-generator/cmd/conversion-gen@$(CONVERSION_TOOLS_VERSION) \ No newline at end of file diff --git a/charts/ratify/templates/deployment.yaml b/charts/ratify/templates/deployment.yaml index f61c5d428..0df4017c4 100644 --- a/charts/ratify/templates/deployment.yaml +++ b/charts/ratify/templates/deployment.yaml @@ -84,7 +84,7 @@ spec: {{- if .Values.instrumentation.metricsEnabled }} - containerPort: {{ required "You must provide .Values.instrumentation.metricsPort" .Values.instrumentation.metricsPort }} {{- end }} - - containerPort: {{ .Values.healthPort }} + - containerPort: {{ required "You must provide .Values.healthPort" .Values.healthPort }} name: healthz protocol: TCP volumeMounts: diff --git a/charts/ratify/values.yaml b/charts/ratify/values.yaml index 706c5dfb7..9a44c3d69 100644 --- a/charts/ratify/values.yaml +++ b/charts/ratify/values.yaml @@ -87,6 +87,7 @@ provider: podAnnotations: {} podLabels: {} enableRuntimeDefaultSeccompProfile: true +healthPort: 9091 rbac: create: true diff --git a/test/bats/quickstart-test.bats b/test/bats/quickstart-test.bats index 7d6bf47eb..1a0a2ca23 100644 --- a/test/bats/quickstart-test.bats +++ b/test/bats/quickstart-test.bats @@ -2,19 +2,11 @@ load helpers -BATS_TESTS_DIR=${BATS_TESTS_DIR:-test/bats/tests} -WAIT_TIME=60 -SLEEP_TIME=1 - -@test "base test without cert rotator" { - teardown() { - echo "cleaning up" - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete pod demo --namespace default --force --ignore-not-found=true' - } +@test "validate quick start steps" { run kubectl run demo --image=ghcr.io/deislabs/ratify/notary-image:signed assert_success # validate unsigned fails - kubectl run demo1 --image=ghcr.io/deislabs/ratify/notary-image:unsigned + run kubectl run demo1 --image=ghcr.io/deislabs/ratify/notary-image:unsigned assert_failure -} +} \ No newline at end of file From 8d1f631cef7906d0fd18651303e86a73d87ec8ed Mon Sep 17 00:00:00 2001 From: "huish@microsoft.com" Date: Wed, 6 Sep 2023 01:38:47 +0000 Subject: [PATCH 11/18] remove debug --- charts/ratify/values.yaml | 2 +- pkg/manager/manager.go | 9 --------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/charts/ratify/values.yaml b/charts/ratify/values.yaml index 9a44c3d69..132b77a21 100644 --- a/charts/ratify/values.yaml +++ b/charts/ratify/values.yaml @@ -87,7 +87,7 @@ provider: podAnnotations: {} podLabels: {} enableRuntimeDefaultSeccompProfile: true -healthPort: 9091 +healthPort: 9090 rbac: create: true diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 048f1fd35..a70260756 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -179,7 +179,6 @@ func StartManager(certRotatorReady chan struct{}, probeAddr string) { } setupLog.Debugf("setting up probeAddr at %s", probeAddr) - setupLog.Infof("setting up probeAddr at ", probeAddr) // Make sure certs are generated and valid if cert rotation is enabled. if featureflag.CertRotation.Enabled { @@ -263,14 +262,6 @@ func StartManager(certRotatorReady chan struct{}, probeAddr string) { os.Exit(1) } - if logrus.GetLevel() == logrus.DebugLevel { - sec, _ := time.ParseDuration("30s") - setupLog.Info("about to sleep for 30sec") - time.Sleep(sec) - setupLog.Info("exiting") - os.Exit(1) - } - setupLog.Info("starting manager") if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { setupLog.Error(err, "problem running manager") From 19c55f8c017ff8f959a81484eb9ece92725d1014 Mon Sep 17 00:00:00 2001 From: "huish@microsoft.com" Date: Wed, 6 Sep 2023 01:49:51 +0000 Subject: [PATCH 12/18] lint --- pkg/manager/manager.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index a70260756..ea045df62 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -267,5 +267,4 @@ func StartManager(certRotatorReady chan struct{}, probeAddr string) { setupLog.Error(err, "problem running manager") os.Exit(1) } - } From 6a556dec574d754fdf70c1e11455af0dc45181eb Mon Sep 17 00:00:00 2001 From: "huish@microsoft.com" Date: Wed, 6 Sep 2023 04:52:36 +0000 Subject: [PATCH 13/18] revert --- .github/workflows/quick-start.yml | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/quick-start.yml b/.github/workflows/quick-start.yml index a09a7f8d5..d5cf54ece 100644 --- a/.github/workflows/quick-start.yml +++ b/.github/workflows/quick-start.yml @@ -25,7 +25,7 @@ jobs: KUBERNETES_VERSION: ["1.26.3"] steps: - name: Checkout - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - name: setup go environment uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0 with: diff --git a/Makefile b/Makefile index 0740c5020..fac7c2a06 100644 --- a/Makefile +++ b/Makefile @@ -644,4 +644,4 @@ $(CONTROLLER_GEN): $(LOCALBIN) .PHONY: conversion-gen conversion-gen: $(CONVERSION_GEN) ## Download conversion-gen locally if necessary. $(CONVERSION_GEN): $(LOCALBIN) - test -s $(LOCALBIN)/conversion-gen || GOBIN=$(LOCALBIN) go install k8s.io/code-generator/cmd/conversion-gen@$(CONVERSION_TOOLS_VERSION) \ No newline at end of file + test -s $(LOCALBIN)/conversion-gen || GOBIN=$(LOCALBIN) go install k8s.io/code-generator/cmd/conversion-gen@$(CONVERSION_TOOLS_VERSION) From 150adf6f998446b8e030807c84909c4f1e450176 Mon Sep 17 00:00:00 2001 From: "huish@microsoft.com" Date: Wed, 6 Sep 2023 05:56:23 +0000 Subject: [PATCH 14/18] revert quickstart --- .github/workflows/quick-start.yml | 2 +- test/bats/quickstart-test.bats | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/quick-start.yml b/.github/workflows/quick-start.yml index d5cf54ece..a09a7f8d5 100644 --- a/.github/workflows/quick-start.yml +++ b/.github/workflows/quick-start.yml @@ -25,7 +25,7 @@ jobs: KUBERNETES_VERSION: ["1.26.3"] steps: - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - name: setup go environment uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0 with: diff --git a/test/bats/quickstart-test.bats b/test/bats/quickstart-test.bats index 1a0a2ca23..fc7c35113 100644 --- a/test/bats/quickstart-test.bats +++ b/test/bats/quickstart-test.bats @@ -9,4 +9,4 @@ load helpers # validate unsigned fails run kubectl run demo1 --image=ghcr.io/deislabs/ratify/notary-image:unsigned assert_failure -} \ No newline at end of file +} From 4d7fbd86abf64642a91a43aa5d364a1dff778fe3 Mon Sep 17 00:00:00 2001 From: "huish@microsoft.com" Date: Wed, 6 Sep 2023 06:20:18 +0000 Subject: [PATCH 15/18] set different port --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index fac7c2a06..c5a2b84f3 100644 --- a/Makefile +++ b/Makefile @@ -559,6 +559,7 @@ e2e-helm-deploy-ratify-replica: e2e-helm-deploy-redis e2e-notation-setup e2e-bui --set featureFlags.RATIFY_EXPERIMENTAL_HIGH_AVAILABILITY=true \ --set resources.requests.memory="64Mi" \ --set resources.requests.cpu="200m" + --set healthPort=9099 rm mount_config.json From 393834fb098a560eec28b0049c0114ba0fc5d16f Mon Sep 17 00:00:00 2001 From: "huish@microsoft.com" Date: Wed, 6 Sep 2023 09:01:35 +0000 Subject: [PATCH 16/18] test port --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c5a2b84f3..baeed220f 100644 --- a/Makefile +++ b/Makefile @@ -558,7 +558,7 @@ e2e-helm-deploy-ratify-replica: e2e-helm-deploy-redis e2e-notation-setup e2e-bui --set provider.cache.name="dapr-redis" \ --set featureFlags.RATIFY_EXPERIMENTAL_HIGH_AVAILABILITY=true \ --set resources.requests.memory="64Mi" \ - --set resources.requests.cpu="200m" + --set resources.requests.cpu="200m" \ --set healthPort=9099 rm mount_config.json From 676c347fadee29a6cf04d2fab45265043f61a633 Mon Sep 17 00:00:00 2001 From: "huish@microsoft.com" Date: Wed, 6 Sep 2023 10:17:28 +0000 Subject: [PATCH 17/18] default port --- Makefile | 1 - charts/ratify/values.yaml | 2 +- httpserver/server.go | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index baeed220f..54263a236 100644 --- a/Makefile +++ b/Makefile @@ -559,7 +559,6 @@ e2e-helm-deploy-ratify-replica: e2e-helm-deploy-redis e2e-notation-setup e2e-bui --set featureFlags.RATIFY_EXPERIMENTAL_HIGH_AVAILABILITY=true \ --set resources.requests.memory="64Mi" \ --set resources.requests.cpu="200m" \ - --set healthPort=9099 rm mount_config.json diff --git a/charts/ratify/values.yaml b/charts/ratify/values.yaml index 82d5900c4..4484ce853 100644 --- a/charts/ratify/values.yaml +++ b/charts/ratify/values.yaml @@ -87,7 +87,7 @@ provider: podAnnotations: {} podLabels: {} enableRuntimeDefaultSeccompProfile: true -healthPort: 9090 +healthPort: 9099 rbac: create: true diff --git a/httpserver/server.go b/httpserver/server.go index 7a9d9032d..c52daf19d 100644 --- a/httpserver/server.go +++ b/httpserver/server.go @@ -46,7 +46,7 @@ const ( DefaultMetricsType = "prometheus" DefaultMetricsPort = 8888 - DefaultHealthPort = ":9090" + DefaultHealthPort = ":9099" ) type Server struct { From f64ffb6bef3cc957191fe7edf5017454fb20a547 Mon Sep 17 00:00:00 2001 From: "huish@microsoft.com" Date: Wed, 6 Sep 2023 10:19:34 +0000 Subject: [PATCH 18/18] update HA --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 54263a236..fac7c2a06 100644 --- a/Makefile +++ b/Makefile @@ -558,7 +558,7 @@ e2e-helm-deploy-ratify-replica: e2e-helm-deploy-redis e2e-notation-setup e2e-bui --set provider.cache.name="dapr-redis" \ --set featureFlags.RATIFY_EXPERIMENTAL_HIGH_AVAILABILITY=true \ --set resources.requests.memory="64Mi" \ - --set resources.requests.cpu="200m" \ + --set resources.requests.cpu="200m" rm mount_config.json