Custom grafana dashboard (#317)

* Create a working custom-grafana-dashboard for ArgoCD metrics

This will allow cluster-admins, nerc-org-admins, and nerc-ops teams to
develop new dashboards using the existing multi-cluster observability
metrics.

rh-pre-commit.version: 2.0.3
rh-pre-commit.check-secrets: ENABLED

* Add missing redirect URL for grafana

We're reusing the dex configuration for logging-grafana, but we had not
updated the list of valid redirect urls in Dex, so oauth logins were
failing.

* Replace static service account token

Previously we were storing a service account token in the vault and
retrieving it via an ExternalSecret. This is not necessary; Kubernetes
already has the ability to populate a Secret with the token for a service
account [1].

[1]: https://kubernetes.io/docs/reference/access-authn-authz/service-accounts-admin/#create-token

* Remove live patches

This commit removes the patch-operator [1] based patches and accompanying
support resources such as service accounts, RBAC, etc.

[1]: https://github.com/redhat-cop/patch-operator

* Removed unused ConfigMap

We're not using the grafana-config-overrides ConfigMap.

* Rename secret key containing oauth client secret

Grafana supports setting configuration from environment variables [1]. This
commit renames the secret key in the oauth-client-secret resource to match
the required variable name pattern so that Grafana will get the oauth
client secret from the environment.

[1]: https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#override-configuration-with-environment-variables

* Rename and reconfigure the observability-metrics datasource

Grafana supports environment variable substitution in data sources [1].
This commit modifies the observability-metrics datasource to get the
service account token and CA certificate from the environment, rather than
using a live patch.

[1]: https://grafana.com/docs/grafana/latest/administration/provisioning/#using-environment-variables

* Set static Grafana configuration

The server root_url and the oauth client id are not dynamic nor are they
secret, so we can set them statically rather than patching them.

* Configure grafana to read environment variables from secrets

This commit modifies the Grafana resource to read environment variables
from the oauth-client-secret and grafana-serviceaccount-token Secrets and
the openshift-service-ca.crt ConfigMap.

* Make some minor formatting changes

This commit has some minor formatting changes that didn't
really fit in anywhere else.

* ClusterRoleBinding for grafana-serviceaccount to cluster-monitoring-view

* Point GrafanaDataSource to correctly point to Observability

---------

Co-authored-by: Lars Kellogg-Stedman <[email protected]>
Co-authored-by: Christopher Tate <[email protected]>

cluster-scope/base/core/namespaces/grafana/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+    - namespace.yaml

cluster-scope/base/core/namespaces/grafana/namespace.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: grafana
+spec: {}

cluster-scope/overlays/nerc-ocp-infra/kustomization.yaml

@@ -16,6 +16,7 @@ resources:
 - ../../bundles/multicluster-engine-operator
 - ../../base/core/namespaces/dex
 - ../../base/core/namespaces/nerc-ocp-prod
+- ../../base/core/namespaces/grafana
 - ../../base/operators.coreos.com/subscriptions/openshift-pipelines-operator
 - ../../base/operators.coreos.com/subscriptions/loki-operator
 - clusterversion.yaml

dex/overlays/nerc-ocp-infra/configmaps/files/config.yaml

@@ -26,6 +26,7 @@ staticClients:
     name: Grafana
     redirectURIs:
       - https://logging-grafana.apps.nerc-ocp-infra.rc.fas.harvard.edu/login/generic_oauth
+      - https://grafana.apps.nerc-ocp-infra.rc.fas.harvard.edu/login/generic_oauth
     secretEnv: GRAFANA_SECRET
 
 connectors:

grafana/base/clusterrolebindings/grafana-serviceaccount-cluster-monitoring-view.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: grafana-serviceaccount-cluster-monitoring-view
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-monitoring-view
+subjects:
+- kind: ServiceAccount
+  name: grafana-serviceaccount
+  namespace: grafana

grafana/base/clusterrolebindings/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - grafana-serviceaccount-cluster-monitoring-view.yaml

grafana/base/kustomization.yaml

@@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - operatorgroups
+  - subscriptions
+  - routes
+  - clusterrolebindings
+commonLabels:
+  app.kubernetes.io/name: grafana
+  app.kubernetes.io/component: grafana
+  app.kubernetes.io/part-of: observability

grafana/base/operatorgroups/grafana.yaml

@@ -0,0 +1,8 @@
+apiVersion: operators.coreos.com/v1
+kind: OperatorGroup
+metadata:
+  name: grafana
+  namespace: grafana
+spec:
+  targetNamespaces:
+    - grafana

grafana/base/operatorgroups/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - grafana.yaml

grafana/base/routes/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- route.yaml

grafana/base/routes/route.yaml

@@ -0,0 +1,17 @@
+kind: Route
+apiVersion: route.openshift.io/v1
+metadata:
+  name: grafana
+  namespace: grafana
+spec:
+  host: REPLACE_IN_OVERLAY
+  to:
+    kind: Service
+    name: grafana-service
+    weight: 100
+  port:
+    targetPort: grafana
+  tls:
+    termination: edge
+    insecureEdgeTerminationPolicy: Redirect
+  wildcardPolicy: None

grafana/base/subscriptions/grafana-operator.yaml

@@ -0,0 +1,11 @@
+apiVersion: operators.coreos.com/v1alpha1
+kind: Subscription
+metadata:
+  name: grafana-operator
+  namespace: grafana
+spec:
+  channel: v4
+  installPlanApproval: Automatic
+  name: grafana-operator
+  source: community-operators
+  sourceNamespace: openshift-marketplace

grafana/base/subscriptions/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - grafana-operator.yaml

grafana/overlays/nerc-ocp-infra/externalsecrets/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- oauth-client-secret.yaml

grafana/overlays/nerc-ocp-infra/externalsecrets/oauth-client-secret.yaml

@@ -0,0 +1,16 @@
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: oauth-client-secret
+  namespace: grafana
+spec:
+  secretStoreRef:
+    name: nerc-cluster-secrets
+    kind: ClusterSecretStore
+  target:
+    name: oauth-client-secret
+  data:
+  - secretKey: GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET
+    remoteRef:
+      key: nerc/nerc-ocp-infra/dex/dex-clients
+      property: GRAFANA_SECRET

grafana/overlays/nerc-ocp-infra/grafanadatasources/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - observability-metrics.yaml

grafana/overlays/nerc-ocp-infra/grafanadatasources/observability-metrics.yaml

@@ -0,0 +1,21 @@
+apiVersion: integreatly.org/v1alpha1
+kind: GrafanaDataSource
+metadata:
+  name: observability-metrics
+  namespace: grafana
+spec:
+  name: observability-metrics
+  datasources:
+    - name: observability-metrics
+      access: proxy
+      editable: false
+      isDefault: true
+      jsonData:
+        httpHeaderName1: Authorization
+        timeInterval: 5s
+        tlsAuthWithCACert: true
+      secureJsonData:
+        httpHeaderValue1: "Bearer ${token}"
+        tlsCACert: "${service-ca.crt}"
+      type: prometheus
+      url: 'http://observability-thanos-query.open-cluster-management-observability.svc.cluster.local:9090/'

grafana/overlays/nerc-ocp-infra/grafanas/grafana.yaml

@@ -0,0 +1,31 @@
+apiVersion: integreatly.org/v1alpha1
+kind: Grafana
+metadata:
+  name: grafana
+  namespace: grafana
+spec:
+  deployment:
+    envFrom:
+      - secretRef:
+          name: oauth-client-secret
+      - secretRef:
+          name: grafana-serviceaccount-token
+      - configMapRef:
+          name: openshift-service-ca.crt
+  config:
+    server:
+      root_url: https://grafana.apps.nerc-ocp-infra.rc.fas.harvard.edu
+    auth.generic_oauth:
+      enabled: true
+      scopes: openid email groups profile
+      email_attribute_path: name
+      api_url: https://dex-dex.apps.nerc-ocp-infra.rc.fas.harvard.edu/userinfo
+      auth_url: https://dex-dex.apps.nerc-ocp-infra.rc.fas.harvard.edu/auth
+      token_url: https://dex-dex.apps.nerc-ocp-infra.rc.fas.harvard.edu/token
+      role_attribute_path: >-
+        contains(groups[*], 'cluster-admins') && 'Admin' ||
+        contains(groups[*], 'nerc-org-admins')  && 'Admin' ||
+        contains(groups[*], 'nerc-ops')   && 'Editor' ||
+        'Deny'
+      role_attribute_strict: true
+      client_id: grafana

grafana/overlays/nerc-ocp-infra/grafanas/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - grafana.yaml

grafana/overlays/nerc-ocp-infra/kustomization.yaml

@@ -0,0 +1,20 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: grafana
+
+resources:
+  - ../../base/
+  - externalsecrets/
+  - grafanas
+  - grafanadatasources
+
+patches:
+  - path: patches/grafana-route.yaml
+
+secretGenerator:
+  - name: grafana-serviceaccount-token
+    type: kubernetes.io/service-account-token
+    options:
+      disableNameSuffixHash: true
+      annotations:
+        kubernetes.io/service-account.name: grafana-serviceaccount

grafana/overlays/nerc-ocp-infra/patches/grafana-route.yaml

@@ -0,0 +1,7 @@
+kind: Route
+apiVersion: route.openshift.io/v1
+metadata:
+  name: grafana
+  namespace: grafana
+spec:
+  host: grafana.apps.nerc-ocp-infra.rc.fas.harvard.edu