Add kuttl tests for prometheus configuration

This patch adds some required prometheus secret for kuttl jobs to run
and adds validation checks.

Also, it modifies the watcher_deploy make target to create a secret so
that the watcher can be deployed until the secret is created by the
telemetry operator.

Makefile

@@ -385,11 +385,13 @@ watcher: ## Install watcher operator via olm
 
 .PHONY: watcher_deploy
 watcher_deploy: ## Deploy watcher service
+	oc apply -f config/samples/watcher_requirements.yaml
 	oc apply -f config/samples/watcher_v1beta1_watcher.yaml
 
 .PHONY: watcher_deploy_cleanup
 watcher_deploy_cleanup: ## Undeploy watcher service
 	oc delete -f config/samples/watcher_v1beta1_watcher.yaml
+	oc delete -f config/samples/watcher_requirements.yaml
 
 .PHONY: watcher_cleanup
 watcher_cleanup: export CATALOG_IMG=${CATALOG_IMAGE}

config/samples/watcher_requirements.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: metric-storage-prometheus-config
+  namespace: openstack
+stringData:
+  host: metric-storage-prometheus.openstack.svc
+  port: "9090"
+  ca_secret: "combined-ca-bundle"
+  ca_key: "internal-ca-bundle.pem"

tests/kuttl/test-suites/default/common/cleanup-errors.yaml

@@ -96,3 +96,13 @@ apiVersion: keystone.openstack.org/v1beta1
 kind: KeystoneEndpoint
 metadata:
   name: watcher
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: metric-storage-prometheus-config
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: custom-prometheus-config

tests/kuttl/test-suites/default/common/cleanup-watcher.yaml

@@ -4,3 +4,9 @@ delete:
 - apiVersion: watcher.openstack.org/v1beta1
   kind: Watcher
   name: watcher-kuttl
+- apiVersion: v1
+  kind: Secret
+  name: metric-storage-prometheus-config
+- apiVersion: v1
+  kind: Secret
+  name: custom-prometheus-config

tests/kuttl/test-suites/default/common/deploy-with-defaults.yaml

@@ -1,3 +1,14 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: metric-storage-prometheus-config
+  namespace: watcher-kuttl-default
+stringData:
+  host: metric-storage-prometheus.watcher-kuttl-default.svc
+  port: "9090"
+  ca_secret: "combined-ca-bundle"
+  ca_key: "internal-ca-bundle.pem"
+---
 apiVersion: watcher.openstack.org/v1beta1
 kind: Watcher
 metadata:

tests/kuttl/test-suites/default/deps/infra.yaml

@@ -31,4 +31,4 @@ spec:
   cinder:
     enabled: false
   telemetry:
-    enabled: false
+    enabled: true

tests/kuttl/test-suites/default/deps/kustomization.yaml

@@ -5,6 +5,8 @@ namespace: watcher-kuttl-default
 secretGenerator:
 - literals:
   - AdminPassword=password
+  - AodhPassword=password
+  - CeilometerPassword=password
   - DbRootPassword=password
   - DatabasePassword=password
   - WatcherPassword=password
@@ -40,3 +42,4 @@ patches:
       secret: osp-secret
 - path: infra.yaml
 - path: keystone.yaml
+- path: telemetry.yaml

tests/kuttl/test-suites/default/deps/telemetry.yaml

@@ -0,0 +1,35 @@
+apiVersion: core.openstack.org/v1beta1
+kind: OpenStackControlPlane
+metadata:
+  name: openstack
+spec:
+  telemetry:
+    template:
+      metricStorage:
+        enabled: true
+        dashboardsEnabled: true
+        monitoringStack:
+          alertingEnabled: true
+          scrapeInterval: 30s
+          storage:
+            strategy: persistent
+            retention: 24h
+            persistent:
+              pvcStorageRequest: 1G
+      autoscaling:
+        enabled: false
+        aodh:
+          passwordSelectors:
+          databaseAccount: aodh
+          databaseInstance: openstack
+          secret: osp-secret
+        heatInstance: heat
+      ceilometer:
+        enabled: true
+        secret: osp-secret
+        passwordSelector:
+          aodhService: AodhPassword
+          ceilometerService: CeilometerPassword
+        serviceUser: ceilometer
+      logging:
+        enabled: false

tests/kuttl/test-suites/default/watcher-api-service-override/01-deploy-with-defaults.yaml

@@ -1,3 +1,14 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: metric-storage-prometheus-config
+  namespace: watcher-kuttl-default
+stringData:
+  host: metric-storage-prometheus.watcher-kuttl-default.svc
+  port: "9090"
+  ca_secret: "combined-ca-bundle"
+  ca_key: "internal-ca-bundle.pem"
+---
 apiVersion: watcher.openstack.org/v1beta1
 kind: Watcher
 metadata:

tests/kuttl/test-suites/default/watcher/01-assert.yaml

@@ -20,6 +20,7 @@ spec:
   apiServiceTemplate:
     replicas: 1
     resources: {}
+  prometheusSecret: metric-storage-prometheus-config
 status:
   apiServiceReadyCount: 1
   conditions:

tests/kuttl/test-suites/default/watcher/04-assert.yaml

@@ -243,6 +243,7 @@ commands:
       if [ -n "${APIPOD}" ]; then
         [ $(echo $(oc rsh -c watcher-api ${APIPOD} cat /etc/watcher/watcher.conf.d/01-global-custom.conf) |grep -c "^# Global config") == 1 ]
         [ $(echo $(oc rsh -c watcher-api ${APIPOD} cat /etc/watcher/watcher.conf.d/02-service-custom.conf) |grep -c "^# Service config") == 1 ]
+        [ $(echo $(oc rsh -c watcher-api ${APIPOD} cat /etc/watcher/watcher.conf.d/00-default.conf) |grep -czPo '\[prometheus_client\]\s+host\s+=\s+metric-storage-prometheus.watcher-kuttl-default.svc\s+port\s+=\s+9090\s+cafile\s+=\s+/etc/pki/ca-trust/extracted/pem/prometheus/internal-ca-bundle.pem') == 1 ]
       else
         exit 1
       fi

tests/kuttl/test-suites/default/watcher/04-deploy-with-precreated-account.yaml

@@ -1,3 +1,14 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: custom-prometheus-config
+  namespace: watcher-kuttl-default
+stringData:
+  host: metric-storage-prometheus.watcher-kuttl-default.svc
+  port: "9090"
+  ca_secret: "combined-ca-bundle"
+  ca_key: "internal-ca-bundle.pem"
+---
 apiVersion: watcher.openstack.org/v1beta1
 kind: Watcher
 metadata:
@@ -6,6 +17,7 @@ metadata:
 spec:
   databaseInstance: "openstack"
   databaseAccount: watcher-precreated
+  prometheusSecret: custom-prometheus-config
   tls:
     caBundleSecretName: "combined-ca-bundle"
   customServiceConfig: |