From 3b9808e3510912e727003f78c30afe5fae70396b Mon Sep 17 00:00:00 2001 From: Camila Macedo Date: Tue, 29 Oct 2024 18:33:05 +0000 Subject: [PATCH] Add Patch to give optional option to enable ServiceMonitor to use cert-manager-managed serving-cert with TLS verification Adds a patch to configure ServiceMonitor with `insecureSkipVerify: false` to ensure TLS verification using cert-manager certificates. Updates documentation and corrects misaligned comments. --- .../testdata/project/cmd/main.go | 7 ++ .../config/certmanager/certificate.yaml | 22 ++++++ .../certmanager_metrics_manager_patch.yaml | 22 ++++++ .../project/config/default/kustomization.yaml | 7 +- .../config/prometheus/kustomization.yaml | 9 +++ .../project/config/prometheus/monitor.yaml | 12 +-- .../config/prometheus/monitor_tls_patch.yaml | 22 ++++++ .../testdata/project/cmd/main.go | 7 ++ .../certmanager_metrics_manager_patch.yaml | 22 ++++++ .../project/config/default/kustomization.yaml | 7 +- .../config/prometheus/kustomization.yaml | 9 +++ .../project/config/prometheus/monitor.yaml | 12 +-- .../config/prometheus/monitor_tls_patch.yaml | 22 ++++++ .../testdata/project/cmd/main.go | 7 ++ .../config/certmanager/certificate.yaml | 22 ++++++ .../certmanager_metrics_manager_patch.yaml | 22 ++++++ .../project/config/default/kustomization.yaml | 7 +- .../config/prometheus/kustomization.yaml | 9 +++ .../project/config/prometheus/monitor.yaml | 12 +-- .../config/prometheus/monitor_tls_patch.yaml | 22 ++++++ .../testdata/project/dist/install.yaml | 21 ++++++ docs/book/src/reference/metrics.md | 50 ++++++++++--- .../common/kustomize/v2/scaffolds/init.go | 2 + .../config/certmanager/certificate.go | 22 ++++++ .../certmanager_metrics_manager_patch.go | 75 +++++++++++++++++++ .../config/kdefault/kustomization.go | 7 +- .../config/prometheus/kustomization.go | 9 +++ .../templates/config/prometheus/monitor.go | 13 ++-- .../config/prometheus/monitor_tls_patch.go | 67 +++++++++++++++++ .../scaffolds/internal/templates/cmd/main.go | 7 ++ test/e2e/v4/generate_test.go | 27 +++++++ testdata/project-v4-multigroup/cmd/main.go | 7 ++ .../config/certmanager/certificate.yaml | 22 ++++++ .../certmanager_metrics_manager_patch.yaml | 22 ++++++ .../config/default/kustomization.yaml | 7 +- .../config/prometheus/kustomization.yaml | 9 +++ .../config/prometheus/monitor.yaml | 12 +-- .../config/prometheus/monitor_tls_patch.yaml | 22 ++++++ testdata/project-v4-with-plugins/cmd/main.go | 7 ++ .../config/certmanager/certificate.yaml | 22 ++++++ .../certmanager_metrics_manager_patch.yaml | 22 ++++++ .../config/default/kustomization.yaml | 7 +- .../config/prometheus/kustomization.yaml | 9 +++ .../config/prometheus/monitor.yaml | 12 +-- .../config/prometheus/monitor_tls_patch.yaml | 22 ++++++ testdata/project-v4/cmd/main.go | 7 ++ .../config/certmanager/certificate.yaml | 22 ++++++ .../certmanager_metrics_manager_patch.yaml | 22 ++++++ .../config/default/kustomization.yaml | 7 +- .../config/prometheus/kustomization.yaml | 9 +++ .../project-v4/config/prometheus/monitor.yaml | 12 +-- .../config/prometheus/monitor_tls_patch.yaml | 22 ++++++ 52 files changed, 811 insertions(+), 73 deletions(-) create mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml create mode 100644 docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/monitor_tls_patch.yaml create mode 100644 docs/book/src/getting-started/testdata/project/config/default/certmanager_metrics_manager_patch.yaml create mode 100644 docs/book/src/getting-started/testdata/project/config/prometheus/monitor_tls_patch.yaml create mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml create mode 100644 docs/book/src/multiversion-tutorial/testdata/project/config/prometheus/monitor_tls_patch.yaml create mode 100644 pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/certmanager_metrics_manager_patch.go create mode 100644 pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/prometheus/monitor_tls_patch.go create mode 100644 testdata/project-v4-multigroup/config/default/certmanager_metrics_manager_patch.yaml create mode 100644 testdata/project-v4-multigroup/config/prometheus/monitor_tls_patch.yaml create mode 100644 testdata/project-v4-with-plugins/config/default/certmanager_metrics_manager_patch.yaml create mode 100644 testdata/project-v4-with-plugins/config/prometheus/monitor_tls_patch.yaml create mode 100644 testdata/project-v4/config/default/certmanager_metrics_manager_patch.yaml create mode 100644 testdata/project-v4/config/prometheus/monitor_tls_patch.yaml diff --git a/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go b/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go index fa0d01fbbde..e4840915796 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go +++ b/docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go @@ -136,6 +136,13 @@ func main() { // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically // generate self-signed certificates for the metrics server. While convenient for development and testing, // this setup is not recommended for production. + + // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, + // you can uncomment the following lines to use the certificate managed by cert-manager. + // metricsServerOptions.CertDir = "/metrics-cert" + // metricsServerOptions.CertName = "tls.crt" + // metricsServerOptions.KeyName = "tls.key" + } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate.yaml index b51082a01e6..64c7776427c 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/certmanager/certificate.yaml @@ -33,3 +33,25 @@ spec: kind: Issuer name: selfsigned-issuer secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: certificate + app.kubernetes.io/instance: serving-cert + app.kubernetes.io/component: certificate + app.kubernetes.io/created-by: project + app.kubernetes.io/part-of: project + app.kubernetes.io/managed-by: kustomize + name: metrics-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + dnsNames: + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml new file mode 100644 index 00000000000..ced953e62da --- /dev/null +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml @@ -0,0 +1,22 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system + labels: + app.kubernetes.io/name: project + app.kubernetes.io/managed-by: kustomize +spec: + template: + spec: + containers: + - name: manager + volumeMounts: + - mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-cert + readOnly: true + volumes: + - name: metrics-cert + secret: + defaultMode: 420 + secretName: metrics-server-cert diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml index 8778c1a5150..b95dd0914a7 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml @@ -33,7 +33,7 @@ resources: # be able to communicate with the Webhook Server. #- ../network-policy -# Uncomment the patches line if you enable Metrics, and/or are using webhooks and cert-manager +# Uncomment the patches line if you enable Metrics patches: # [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. # More info: https://book.kubebuilder.io/reference/metrics @@ -41,6 +41,11 @@ patches: target: kind: Deployment +# Uncomment the patches line if you enable Metrics and CertManager +# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. +# This patch will protect the metrics with certmanager self-signed certs. +# - path: certmanager_metrics_manager_patch.yaml + # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml - path: manager_webhook_patch.yaml diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/kustomization.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/kustomization.yaml index ed137168a1d..dd929ad8db7 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/kustomization.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/kustomization.yaml @@ -1,2 +1,11 @@ resources: - monitor.yaml + +# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus +# to securely reference certificates created and managed by cert-manager. +# Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml +# to mount the "metrics-cert" secret in the Manager Deployment. +#patches: +# - path: monitor_tls_patch.yaml +# target: +# kind: ServiceMonitor diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/monitor.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/monitor.yaml index 1dea5d5fd7b..a535309411e 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/monitor.yaml +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/monitor.yaml @@ -16,14 +16,10 @@ spec: bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token tlsConfig: # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables - # certificate verification. This poses a significant security risk by making the system vulnerable to - # man-in-the-middle attacks, where an attacker could intercept and manipulate the communication between - # Prometheus and the monitored services. This could lead to unauthorized access to sensitive metrics data, - # compromising the integrity and confidentiality of the information. - # Please use the following options for secure configurations: - # caFile: /etc/metrics-certs/ca.crt - # certFile: /etc/metrics-certs/tls.crt - # keyFile: /etc/metrics-certs/tls.key + # certificate verification, exposing the system to potential man-in-the-middle attacks. + # For production environments, it is recommended to use cert-manager for automatic TLS certificate management. + # To apply this configuration, enable cert-manager and use the patch located at config/prometheus/servicemonitor_tls_patch.yaml, + # which securely references the certificate from the 'metrics-cert' secret. insecureSkipVerify: true selector: matchLabels: diff --git a/docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/monitor_tls_patch.yaml b/docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/monitor_tls_patch.yaml new file mode 100644 index 00000000000..8e5768127e9 --- /dev/null +++ b/docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/monitor_tls_patch.yaml @@ -0,0 +1,22 @@ +# Patch for Prometheus ServiceMonitor to enable secure TLS configuration +# using certificates managed by cert-manager +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: controller-manager-metrics-monitor + namespace: system +spec: + endpoints: + - tlsConfig: + insecureSkipVerify: false + ca: + secret: + name: metrics-cert + key: ca.crt + cert: + secret: + name: metrics-cert + key: tls.crt + keySecret: + name: metrics-cert + key: tls.key diff --git a/docs/book/src/getting-started/testdata/project/cmd/main.go b/docs/book/src/getting-started/testdata/project/cmd/main.go index d707ad754bc..9411259e9d2 100644 --- a/docs/book/src/getting-started/testdata/project/cmd/main.go +++ b/docs/book/src/getting-started/testdata/project/cmd/main.go @@ -116,6 +116,13 @@ func main() { // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically // generate self-signed certificates for the metrics server. While convenient for development and testing, // this setup is not recommended for production. + + // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, + // you can uncomment the following lines to use the certificate managed by cert-manager. + // metricsServerOptions.CertDir = "/metrics-cert" + // metricsServerOptions.CertName = "tls.crt" + // metricsServerOptions.KeyName = "tls.key" + } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ diff --git a/docs/book/src/getting-started/testdata/project/config/default/certmanager_metrics_manager_patch.yaml b/docs/book/src/getting-started/testdata/project/config/default/certmanager_metrics_manager_patch.yaml new file mode 100644 index 00000000000..ced953e62da --- /dev/null +++ b/docs/book/src/getting-started/testdata/project/config/default/certmanager_metrics_manager_patch.yaml @@ -0,0 +1,22 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system + labels: + app.kubernetes.io/name: project + app.kubernetes.io/managed-by: kustomize +spec: + template: + spec: + containers: + - name: manager + volumeMounts: + - mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-cert + readOnly: true + volumes: + - name: metrics-cert + secret: + defaultMode: 420 + secretName: metrics-server-cert diff --git a/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml b/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml index 8922567ea88..7b99b9921da 100644 --- a/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml +++ b/docs/book/src/getting-started/testdata/project/config/default/kustomization.yaml @@ -33,7 +33,7 @@ resources: # be able to communicate with the Webhook Server. #- ../network-policy -# Uncomment the patches line if you enable Metrics, and/or are using webhooks and cert-manager +# Uncomment the patches line if you enable Metrics patches: # [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. # More info: https://book.kubebuilder.io/reference/metrics @@ -41,6 +41,11 @@ patches: target: kind: Deployment +# Uncomment the patches line if you enable Metrics and CertManager +# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. +# This patch will protect the metrics with certmanager self-signed certs. +# - path: certmanager_metrics_manager_patch.yaml + # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml #- path: manager_webhook_patch.yaml diff --git a/docs/book/src/getting-started/testdata/project/config/prometheus/kustomization.yaml b/docs/book/src/getting-started/testdata/project/config/prometheus/kustomization.yaml index ed137168a1d..dd929ad8db7 100644 --- a/docs/book/src/getting-started/testdata/project/config/prometheus/kustomization.yaml +++ b/docs/book/src/getting-started/testdata/project/config/prometheus/kustomization.yaml @@ -1,2 +1,11 @@ resources: - monitor.yaml + +# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus +# to securely reference certificates created and managed by cert-manager. +# Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml +# to mount the "metrics-cert" secret in the Manager Deployment. +#patches: +# - path: monitor_tls_patch.yaml +# target: +# kind: ServiceMonitor diff --git a/docs/book/src/getting-started/testdata/project/config/prometheus/monitor.yaml b/docs/book/src/getting-started/testdata/project/config/prometheus/monitor.yaml index 1dea5d5fd7b..a535309411e 100644 --- a/docs/book/src/getting-started/testdata/project/config/prometheus/monitor.yaml +++ b/docs/book/src/getting-started/testdata/project/config/prometheus/monitor.yaml @@ -16,14 +16,10 @@ spec: bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token tlsConfig: # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables - # certificate verification. This poses a significant security risk by making the system vulnerable to - # man-in-the-middle attacks, where an attacker could intercept and manipulate the communication between - # Prometheus and the monitored services. This could lead to unauthorized access to sensitive metrics data, - # compromising the integrity and confidentiality of the information. - # Please use the following options for secure configurations: - # caFile: /etc/metrics-certs/ca.crt - # certFile: /etc/metrics-certs/tls.crt - # keyFile: /etc/metrics-certs/tls.key + # certificate verification, exposing the system to potential man-in-the-middle attacks. + # For production environments, it is recommended to use cert-manager for automatic TLS certificate management. + # To apply this configuration, enable cert-manager and use the patch located at config/prometheus/servicemonitor_tls_patch.yaml, + # which securely references the certificate from the 'metrics-cert' secret. insecureSkipVerify: true selector: matchLabels: diff --git a/docs/book/src/getting-started/testdata/project/config/prometheus/monitor_tls_patch.yaml b/docs/book/src/getting-started/testdata/project/config/prometheus/monitor_tls_patch.yaml new file mode 100644 index 00000000000..8e5768127e9 --- /dev/null +++ b/docs/book/src/getting-started/testdata/project/config/prometheus/monitor_tls_patch.yaml @@ -0,0 +1,22 @@ +# Patch for Prometheus ServiceMonitor to enable secure TLS configuration +# using certificates managed by cert-manager +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: controller-manager-metrics-monitor + namespace: system +spec: + endpoints: + - tlsConfig: + insecureSkipVerify: false + ca: + secret: + name: metrics-cert + key: ca.crt + cert: + secret: + name: metrics-cert + key: tls.crt + keySecret: + name: metrics-cert + key: tls.key diff --git a/docs/book/src/multiversion-tutorial/testdata/project/cmd/main.go b/docs/book/src/multiversion-tutorial/testdata/project/cmd/main.go index 1a002ab638d..afbb080efbb 100644 --- a/docs/book/src/multiversion-tutorial/testdata/project/cmd/main.go +++ b/docs/book/src/multiversion-tutorial/testdata/project/cmd/main.go @@ -135,6 +135,13 @@ func main() { // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically // generate self-signed certificates for the metrics server. While convenient for development and testing, // this setup is not recommended for production. + + // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, + // you can uncomment the following lines to use the certificate managed by cert-manager. + // metricsServerOptions.CertDir = "/metrics-cert" + // metricsServerOptions.CertName = "tls.crt" + // metricsServerOptions.KeyName = "tls.key" + } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate.yaml index b51082a01e6..64c7776427c 100644 --- a/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate.yaml +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/certmanager/certificate.yaml @@ -33,3 +33,25 @@ spec: kind: Issuer name: selfsigned-issuer secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: certificate + app.kubernetes.io/instance: serving-cert + app.kubernetes.io/component: certificate + app.kubernetes.io/created-by: project + app.kubernetes.io/part-of: project + app.kubernetes.io/managed-by: kustomize + name: metrics-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + dnsNames: + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml new file mode 100644 index 00000000000..ced953e62da --- /dev/null +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/default/certmanager_metrics_manager_patch.yaml @@ -0,0 +1,22 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system + labels: + app.kubernetes.io/name: project + app.kubernetes.io/managed-by: kustomize +spec: + template: + spec: + containers: + - name: manager + volumeMounts: + - mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-cert + readOnly: true + volumes: + - name: metrics-cert + secret: + defaultMode: 420 + secretName: metrics-server-cert diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/default/kustomization.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/default/kustomization.yaml index 094f86a8cec..a0a10693101 100644 --- a/docs/book/src/multiversion-tutorial/testdata/project/config/default/kustomization.yaml +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/default/kustomization.yaml @@ -33,7 +33,7 @@ resources: # be able to communicate with the Webhook Server. #- ../network-policy -# Uncomment the patches line if you enable Metrics, and/or are using webhooks and cert-manager +# Uncomment the patches line if you enable Metrics patches: # [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. # More info: https://book.kubebuilder.io/reference/metrics @@ -41,6 +41,11 @@ patches: target: kind: Deployment +# Uncomment the patches line if you enable Metrics and CertManager +# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. +# This patch will protect the metrics with certmanager self-signed certs. +# - path: certmanager_metrics_manager_patch.yaml + # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml - path: manager_webhook_patch.yaml diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/prometheus/kustomization.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/prometheus/kustomization.yaml index ed137168a1d..dd929ad8db7 100644 --- a/docs/book/src/multiversion-tutorial/testdata/project/config/prometheus/kustomization.yaml +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/prometheus/kustomization.yaml @@ -1,2 +1,11 @@ resources: - monitor.yaml + +# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus +# to securely reference certificates created and managed by cert-manager. +# Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml +# to mount the "metrics-cert" secret in the Manager Deployment. +#patches: +# - path: monitor_tls_patch.yaml +# target: +# kind: ServiceMonitor diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/prometheus/monitor.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/prometheus/monitor.yaml index 1dea5d5fd7b..a535309411e 100644 --- a/docs/book/src/multiversion-tutorial/testdata/project/config/prometheus/monitor.yaml +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/prometheus/monitor.yaml @@ -16,14 +16,10 @@ spec: bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token tlsConfig: # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables - # certificate verification. This poses a significant security risk by making the system vulnerable to - # man-in-the-middle attacks, where an attacker could intercept and manipulate the communication between - # Prometheus and the monitored services. This could lead to unauthorized access to sensitive metrics data, - # compromising the integrity and confidentiality of the information. - # Please use the following options for secure configurations: - # caFile: /etc/metrics-certs/ca.crt - # certFile: /etc/metrics-certs/tls.crt - # keyFile: /etc/metrics-certs/tls.key + # certificate verification, exposing the system to potential man-in-the-middle attacks. + # For production environments, it is recommended to use cert-manager for automatic TLS certificate management. + # To apply this configuration, enable cert-manager and use the patch located at config/prometheus/servicemonitor_tls_patch.yaml, + # which securely references the certificate from the 'metrics-cert' secret. insecureSkipVerify: true selector: matchLabels: diff --git a/docs/book/src/multiversion-tutorial/testdata/project/config/prometheus/monitor_tls_patch.yaml b/docs/book/src/multiversion-tutorial/testdata/project/config/prometheus/monitor_tls_patch.yaml new file mode 100644 index 00000000000..8e5768127e9 --- /dev/null +++ b/docs/book/src/multiversion-tutorial/testdata/project/config/prometheus/monitor_tls_patch.yaml @@ -0,0 +1,22 @@ +# Patch for Prometheus ServiceMonitor to enable secure TLS configuration +# using certificates managed by cert-manager +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: controller-manager-metrics-monitor + namespace: system +spec: + endpoints: + - tlsConfig: + insecureSkipVerify: false + ca: + secret: + name: metrics-cert + key: ca.crt + cert: + secret: + name: metrics-cert + key: tls.crt + keySecret: + name: metrics-cert + key: tls.key diff --git a/docs/book/src/multiversion-tutorial/testdata/project/dist/install.yaml b/docs/book/src/multiversion-tutorial/testdata/project/dist/install.yaml index 1345f29120c..e5b02a89bcc 100644 --- a/docs/book/src/multiversion-tutorial/testdata/project/dist/install.yaml +++ b/docs/book/src/multiversion-tutorial/testdata/project/dist/install.yaml @@ -7956,6 +7956,27 @@ spec: --- apiVersion: cert-manager.io/v1 kind: Certificate +metadata: + labels: + app.kubernetes.io/component: certificate + app.kubernetes.io/created-by: project + app.kubernetes.io/instance: serving-cert + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: certificate + app.kubernetes.io/part-of: project + name: project-metrics-cert + namespace: project-system +spec: + dnsNames: + - project-webhook-service.project-system.svc + - project-webhook-service.project-system.svc.cluster.local + issuerRef: + kind: Issuer + name: project-selfsigned-issuer + secretName: metrics-server-cert +--- +apiVersion: cert-manager.io/v1 +kind: Certificate metadata: labels: app.kubernetes.io/component: certificate diff --git a/docs/book/src/reference/metrics.md b/docs/book/src/reference/metrics.md index 763f482c519..dee93057116 100644 --- a/docs/book/src/reference/metrics.md +++ b/docs/book/src/reference/metrics.md @@ -179,6 +179,46 @@ An [issue](https://github.com/kubernetes-sigs/controller-runtime/issues/2781) ha enhance the controller-runtime and address these considerations. +### By exposing the metrics endpoint using HTTPS and Cert-Manager + +Integrating `cert-manager` with your metrics service enables secure +HTTPS access via TLS encryption. Follow the steps below to configure +your project to expose the metrics endpoint using HTTPS with cert-manager. + +1. **Enable Cert-Manager in `config/default/kustomization.yaml`:** + - Uncomment the cert-manager resource to include it in your project: + + ```yaml + - ../certmanager + ``` + +2. **Enable the Patch for the `ServiceMonitor` to Use the Cert-Manager-Managed Secret `config/prometheus/kustomization.yaml`:** + - Add or uncomment the `ServiceMonitor` patch to securely reference the cert-manager-managed secret, replacing insecure configurations with secure certificate verification: + + ```yaml + - path: monitor_tls_patch.yaml + target: + kind: ServiceMonitor + ``` + +3. **Enable the Patch to Mount the Cert-Manager-Managed Secret in the Controller Deployment in `config/default/kustomization.yaml`:** + - Use the `manager_webhook_patch.yaml` (or create a custom metrics patch) to mount the `serving-cert` secret in the Manager Deployment. + + ```yaml + - path: manager_webhook_patch.yaml + ``` + +4. **Update `cmd/main.go` to Use the Certificate Managed by Cert-Manager:** + - Modify `cmd/main.go` to configure the metrics server to use the cert-manager-managed certificates. + Uncomment the lines for `CertDir`, `CertName`, and `KeyName`: + + ```go + if secureMetrics { + metricsServerOptions.CertDir = "/var/run/secrets/kubernetes.io/certs" + metricsServerOptions.CertName = "tls.crt" + metricsServerOptions.KeyName = "tls.key" + } + ``` ### By using Network Policy (You can optionally enable) @@ -194,16 +234,6 @@ Uncomment the following line in the `config/default/kustomization.yaml`: #- ../network-policy ``` -### By exposing the metrics endpoint using HTTPS and CertManager - -Integrating `cert-manager` with your metrics service can secure the endpoint via TLS encryption. - -To modify your project setup to expose metrics using HTTPS with -the help of cert-manager, you'll need to change the configuration of both -the `Service` under `config/default/metrics_service.yaml` and -the `ServiceMonitor` under `config/prometheus/monitor.yaml` to use a secure HTTPS port -and ensure the necessary certificate is applied. - ## Exporting Metrics for Prometheus Follow the steps below to export the metrics using the Prometheus Operator: diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/init.go b/pkg/plugins/common/kustomize/v2/scaffolds/init.go index ccddd7a09de..227f7a681cd 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/init.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/init.go @@ -78,12 +78,14 @@ func (s *initScaffolder) Scaffold() error { &rbac.ServiceAccount{}, &manager.Kustomization{}, &kdefault.ManagerMetricsPatch{}, + &kdefault.CertManagerMetricsPatch{}, &manager.Config{Image: imageName}, &kdefault.Kustomization{}, &network_policy.Kustomization{}, &network_policy.NetworkPolicyAllowMetrics{}, &prometheus.Kustomization{}, &prometheus.Monitor{}, + &prometheus.ServiceMonitorPatch{}, } return scaffold.Execute(templates...) diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate.go index 04bf0cdd120..98c4e769ea6 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/certmanager/certificate.go @@ -79,4 +79,26 @@ spec: kind: Issuer name: selfsigned-issuer secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: certificate + app.kubernetes.io/instance: serving-cert + app.kubernetes.io/component: certificate + app.kubernetes.io/created-by: {{ .ProjectName }} + app.kubernetes.io/part-of: {{ .ProjectName }} + app.kubernetes.io/managed-by: kustomize + name: metrics-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + dnsNames: + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize ` diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/certmanager_metrics_manager_patch.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/certmanager_metrics_manager_patch.go new file mode 100644 index 00000000000..e11aae01ee9 --- /dev/null +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/certmanager_metrics_manager_patch.go @@ -0,0 +1,75 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kdefault + +import ( + "path/filepath" + + "sigs.k8s.io/kubebuilder/v4/pkg/machinery" +) + +var _ machinery.Template = &CertManagerMetricsPatch{} + +// CertManagerMetricsPatch scaffolds a file that defines the patch that enables webhooks on the manager +type CertManagerMetricsPatch struct { + machinery.TemplateMixin + machinery.ProjectNameMixin + + Force bool +} + +// SetTemplateDefaults implements file.Template +func (f *CertManagerMetricsPatch) SetTemplateDefaults() error { + if f.Path == "" { + f.Path = filepath.Join("config", "default", "certmanager_metrics_manager_patch.yaml") + } + + f.TemplateBody = metricsManagerPatchTemplate + + if f.Force { + f.IfExistsAction = machinery.OverwriteFile + } else { + // If file exists (ex. because a webhook was already created), skip creation. + f.IfExistsAction = machinery.SkipFile + } + + return nil +} + +const metricsManagerPatchTemplate = `apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system + labels: + app.kubernetes.io/name: {{ .ProjectName }} + app.kubernetes.io/managed-by: kustomize +spec: + template: + spec: + containers: + - name: manager + volumeMounts: + - mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-cert + readOnly: true + volumes: + - name: metrics-cert + secret: + defaultMode: 420 + secretName: metrics-server-cert +` diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go index bc82de3ffcf..3b65753382a 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/kdefault/kustomization.go @@ -78,7 +78,7 @@ resources: # be able to communicate with the Webhook Server. #- ../network-policy -# Uncomment the patches line if you enable Metrics, and/or are using webhooks and cert-manager +# Uncomment the patches line if you enable Metrics patches: # [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. # More info: https://book.kubebuilder.io/reference/metrics @@ -86,6 +86,11 @@ patches: target: kind: Deployment +# Uncomment the patches line if you enable Metrics and CertManager +# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. +# This patch will protect the metrics with certmanager self-signed certs. +# - path: certmanager_metrics_manager_patch.yaml + # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml #- path: manager_webhook_patch.yaml diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/prometheus/kustomization.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/prometheus/kustomization.go index 65547bdf41a..5a64447fc4b 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/prometheus/kustomization.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/prometheus/kustomization.go @@ -42,4 +42,13 @@ func (f *Kustomization) SetTemplateDefaults() error { const kustomizationTemplate = `resources: - monitor.yaml + +# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus +# to securely reference certificates created and managed by cert-manager. +# Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml +# to mount the "metrics-cert" secret in the Manager Deployment. +#patches: +# - path: monitor_tls_patch.yaml +# target: +# kind: ServiceMonitor ` diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/prometheus/monitor.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/prometheus/monitor.go index 73ce389aa01..597c0021f12 100644 --- a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/prometheus/monitor.go +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/prometheus/monitor.go @@ -41,6 +41,7 @@ func (f *Monitor) SetTemplateDefaults() error { return nil } +// nolint:lll const serviceMonitorTemplate = `# Prometheus Monitor Service (Metrics) apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor @@ -59,14 +60,10 @@ spec: bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token tlsConfig: # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables - # certificate verification. This poses a significant security risk by making the system vulnerable to - # man-in-the-middle attacks, where an attacker could intercept and manipulate the communication between - # Prometheus and the monitored services. This could lead to unauthorized access to sensitive metrics data, - # compromising the integrity and confidentiality of the information. - # Please use the following options for secure configurations: - # caFile: /etc/metrics-certs/ca.crt - # certFile: /etc/metrics-certs/tls.crt - # keyFile: /etc/metrics-certs/tls.key + # certificate verification, exposing the system to potential man-in-the-middle attacks. + # For production environments, it is recommended to use cert-manager for automatic TLS certificate management. + # To apply this configuration, enable cert-manager and use the patch located at config/prometheus/servicemonitor_tls_patch.yaml, + # which securely references the certificate from the 'metrics-cert' secret. insecureSkipVerify: true selector: matchLabels: diff --git a/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/prometheus/monitor_tls_patch.go b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/prometheus/monitor_tls_patch.go new file mode 100644 index 00000000000..7671ddb859d --- /dev/null +++ b/pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/prometheus/monitor_tls_patch.go @@ -0,0 +1,67 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package prometheus + +import ( + "path/filepath" + + "sigs.k8s.io/kubebuilder/v4/pkg/machinery" +) + +var _ machinery.Template = &ServiceMonitorPatch{} + +// ServiceMonitorPatch scaffolds a file that defines the patch for the ServiceMonitor +// to use cert-manager managed certificates for secure TLS configuration. +type ServiceMonitorPatch struct { + machinery.TemplateMixin + machinery.ProjectNameMixin +} + +// SetTemplateDefaults implements file.Template +func (f *ServiceMonitorPatch) SetTemplateDefaults() error { + if f.Path == "" { + f.Path = filepath.Join("config", "prometheus", "monitor_tls_patch.yaml") + } + + f.TemplateBody = serviceMonitorPatchTemplate + + return nil +} + +const serviceMonitorPatchTemplate = `# Patch for Prometheus ServiceMonitor to enable secure TLS configuration +# using certificates managed by cert-manager +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: controller-manager-metrics-monitor + namespace: system +spec: + endpoints: + - tlsConfig: + insecureSkipVerify: false + ca: + secret: + name: metrics-cert + key: ca.crt + cert: + secret: + name: metrics-cert + key: tls.crt + keySecret: + name: metrics-cert + key: tls.key +` diff --git a/pkg/plugins/golang/v4/scaffolds/internal/templates/cmd/main.go b/pkg/plugins/golang/v4/scaffolds/internal/templates/cmd/main.go index 127581c10d0..9dda0c182f3 100644 --- a/pkg/plugins/golang/v4/scaffolds/internal/templates/cmd/main.go +++ b/pkg/plugins/golang/v4/scaffolds/internal/templates/cmd/main.go @@ -318,6 +318,13 @@ func main() { // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically // generate self-signed certificates for the metrics server. While convenient for development and testing, // this setup is not recommended for production. + + // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, + // you can uncomment the following lines to use the certificate managed by cert-manager. + // metricsServerOptions.CertDir = "/metrics-cert" + // metricsServerOptions.CertName = "tls.crt" + // metricsServerOptions.KeyName = "tls.key" + } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ diff --git a/test/e2e/v4/generate_test.go b/test/e2e/v4/generate_test.go index 0a22b102eb8..00a68f96b92 100644 --- a/test/e2e/v4/generate_test.go +++ b/test/e2e/v4/generate_test.go @@ -63,6 +63,15 @@ func GenerateV4(kbc *utils.TestContext) { "#- ../prometheus", "#")).To(Succeed()) ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), certManagerTarget, "#")).To(Succeed()) + ExpectWithOffset(1, pluginutil.UncommentCode( + filepath.Join(kbc.Dir, "config", "prometheus", "kustomization.yaml"), + monitorTlsPatch, "#")).To(Succeed()) + ExpectWithOffset(1, pluginutil.UncommentCode( + filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), + `# - path: certmanager_metrics_manager_patch.yaml`, "#")).To(Succeed()) + ExpectWithOffset(1, pluginutil.UncommentCode( + filepath.Join(kbc.Dir, "cmd", "main.go"), + tlsConfigManager, "// ")).To(Succeed()) if kbc.IsRestricted { By("uncomment kustomize files to ensure that pods are restricted") @@ -162,6 +171,15 @@ func GenerateV4WithNetworkPolicies(kbc *utils.TestContext) { ExpectWithOffset(1, pluginutil.UncommentCode( filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), metricsTarget, "#")).To(Succeed()) + ExpectWithOffset(1, pluginutil.UncommentCode( + filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), + `# - path: certmanager_metrics_manager_patch.yaml`, "#")).To(Succeed()) + ExpectWithOffset(1, pluginutil.UncommentCode( + filepath.Join(kbc.Dir, "config", "prometheus", "kustomization.yaml"), + monitorTlsPatch, "#")).To(Succeed()) + ExpectWithOffset(1, pluginutil.UncommentCode( + filepath.Join(kbc.Dir, "cmd", "main.go"), + tlsConfigManager, "// ")).To(Succeed()) By("uncomment kustomization.yaml to enable network policy") ExpectWithOffset(1, pluginutil.UncommentCode( filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"), @@ -368,3 +386,12 @@ func uncommentPodStandards(kbc *utils.TestContext) { ExpectWithOffset(1, err).NotTo(HaveOccurred()) } } + +const monitorTlsPatch = `#patches: +# - path: monitor_tls_patch.yaml +# target: +# kind: ServiceMonitor` + +const tlsConfigManager = `// metricsServerOptions.CertDir = "/metrics-cert" + // metricsServerOptions.CertName = "tls.crt" + // metricsServerOptions.KeyName = "tls.key"` diff --git a/testdata/project-v4-multigroup/cmd/main.go b/testdata/project-v4-multigroup/cmd/main.go index 9444e6e0db6..6fdd9774958 100644 --- a/testdata/project-v4-multigroup/cmd/main.go +++ b/testdata/project-v4-multigroup/cmd/main.go @@ -156,6 +156,13 @@ func main() { // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically // generate self-signed certificates for the metrics server. While convenient for development and testing, // this setup is not recommended for production. + + // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, + // you can uncomment the following lines to use the certificate managed by cert-manager. + // metricsServerOptions.CertDir = "/metrics-cert" + // metricsServerOptions.CertName = "tls.crt" + // metricsServerOptions.KeyName = "tls.key" + } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ diff --git a/testdata/project-v4-multigroup/config/certmanager/certificate.yaml b/testdata/project-v4-multigroup/config/certmanager/certificate.yaml index d6bd556f1b4..e0b85f52967 100644 --- a/testdata/project-v4-multigroup/config/certmanager/certificate.yaml +++ b/testdata/project-v4-multigroup/config/certmanager/certificate.yaml @@ -33,3 +33,25 @@ spec: kind: Issuer name: selfsigned-issuer secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: certificate + app.kubernetes.io/instance: serving-cert + app.kubernetes.io/component: certificate + app.kubernetes.io/created-by: project-v4-multigroup + app.kubernetes.io/part-of: project-v4-multigroup + app.kubernetes.io/managed-by: kustomize + name: metrics-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + dnsNames: + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/testdata/project-v4-multigroup/config/default/certmanager_metrics_manager_patch.yaml b/testdata/project-v4-multigroup/config/default/certmanager_metrics_manager_patch.yaml new file mode 100644 index 00000000000..a2a7babf6ac --- /dev/null +++ b/testdata/project-v4-multigroup/config/default/certmanager_metrics_manager_patch.yaml @@ -0,0 +1,22 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system + labels: + app.kubernetes.io/name: project-v4-multigroup + app.kubernetes.io/managed-by: kustomize +spec: + template: + spec: + containers: + - name: manager + volumeMounts: + - mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-cert + readOnly: true + volumes: + - name: metrics-cert + secret: + defaultMode: 420 + secretName: metrics-server-cert diff --git a/testdata/project-v4-multigroup/config/default/kustomization.yaml b/testdata/project-v4-multigroup/config/default/kustomization.yaml index 42805e76152..96b3a4d9688 100644 --- a/testdata/project-v4-multigroup/config/default/kustomization.yaml +++ b/testdata/project-v4-multigroup/config/default/kustomization.yaml @@ -33,7 +33,7 @@ resources: # be able to communicate with the Webhook Server. #- ../network-policy -# Uncomment the patches line if you enable Metrics, and/or are using webhooks and cert-manager +# Uncomment the patches line if you enable Metrics patches: # [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. # More info: https://book.kubebuilder.io/reference/metrics @@ -41,6 +41,11 @@ patches: target: kind: Deployment +# Uncomment the patches line if you enable Metrics and CertManager +# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. +# This patch will protect the metrics with certmanager self-signed certs. +# - path: certmanager_metrics_manager_patch.yaml + # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml - path: manager_webhook_patch.yaml diff --git a/testdata/project-v4-multigroup/config/prometheus/kustomization.yaml b/testdata/project-v4-multigroup/config/prometheus/kustomization.yaml index ed137168a1d..dd929ad8db7 100644 --- a/testdata/project-v4-multigroup/config/prometheus/kustomization.yaml +++ b/testdata/project-v4-multigroup/config/prometheus/kustomization.yaml @@ -1,2 +1,11 @@ resources: - monitor.yaml + +# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus +# to securely reference certificates created and managed by cert-manager. +# Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml +# to mount the "metrics-cert" secret in the Manager Deployment. +#patches: +# - path: monitor_tls_patch.yaml +# target: +# kind: ServiceMonitor diff --git a/testdata/project-v4-multigroup/config/prometheus/monitor.yaml b/testdata/project-v4-multigroup/config/prometheus/monitor.yaml index 89d2f351f5b..24dca0117aa 100644 --- a/testdata/project-v4-multigroup/config/prometheus/monitor.yaml +++ b/testdata/project-v4-multigroup/config/prometheus/monitor.yaml @@ -16,14 +16,10 @@ spec: bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token tlsConfig: # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables - # certificate verification. This poses a significant security risk by making the system vulnerable to - # man-in-the-middle attacks, where an attacker could intercept and manipulate the communication between - # Prometheus and the monitored services. This could lead to unauthorized access to sensitive metrics data, - # compromising the integrity and confidentiality of the information. - # Please use the following options for secure configurations: - # caFile: /etc/metrics-certs/ca.crt - # certFile: /etc/metrics-certs/tls.crt - # keyFile: /etc/metrics-certs/tls.key + # certificate verification, exposing the system to potential man-in-the-middle attacks. + # For production environments, it is recommended to use cert-manager for automatic TLS certificate management. + # To apply this configuration, enable cert-manager and use the patch located at config/prometheus/servicemonitor_tls_patch.yaml, + # which securely references the certificate from the 'metrics-cert' secret. insecureSkipVerify: true selector: matchLabels: diff --git a/testdata/project-v4-multigroup/config/prometheus/monitor_tls_patch.yaml b/testdata/project-v4-multigroup/config/prometheus/monitor_tls_patch.yaml new file mode 100644 index 00000000000..8e5768127e9 --- /dev/null +++ b/testdata/project-v4-multigroup/config/prometheus/monitor_tls_patch.yaml @@ -0,0 +1,22 @@ +# Patch for Prometheus ServiceMonitor to enable secure TLS configuration +# using certificates managed by cert-manager +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: controller-manager-metrics-monitor + namespace: system +spec: + endpoints: + - tlsConfig: + insecureSkipVerify: false + ca: + secret: + name: metrics-cert + key: ca.crt + cert: + secret: + name: metrics-cert + key: tls.crt + keySecret: + name: metrics-cert + key: tls.key diff --git a/testdata/project-v4-with-plugins/cmd/main.go b/testdata/project-v4-with-plugins/cmd/main.go index 11f1914fb79..8707ad0c8a6 100644 --- a/testdata/project-v4-with-plugins/cmd/main.go +++ b/testdata/project-v4-with-plugins/cmd/main.go @@ -122,6 +122,13 @@ func main() { // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically // generate self-signed certificates for the metrics server. While convenient for development and testing, // this setup is not recommended for production. + + // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, + // you can uncomment the following lines to use the certificate managed by cert-manager. + // metricsServerOptions.CertDir = "/metrics-cert" + // metricsServerOptions.CertName = "tls.crt" + // metricsServerOptions.KeyName = "tls.key" + } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ diff --git a/testdata/project-v4-with-plugins/config/certmanager/certificate.yaml b/testdata/project-v4-with-plugins/config/certmanager/certificate.yaml index 68214a62d39..7a00ef5e693 100644 --- a/testdata/project-v4-with-plugins/config/certmanager/certificate.yaml +++ b/testdata/project-v4-with-plugins/config/certmanager/certificate.yaml @@ -33,3 +33,25 @@ spec: kind: Issuer name: selfsigned-issuer secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: certificate + app.kubernetes.io/instance: serving-cert + app.kubernetes.io/component: certificate + app.kubernetes.io/created-by: project-v4-with-plugins + app.kubernetes.io/part-of: project-v4-with-plugins + app.kubernetes.io/managed-by: kustomize + name: metrics-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + dnsNames: + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/testdata/project-v4-with-plugins/config/default/certmanager_metrics_manager_patch.yaml b/testdata/project-v4-with-plugins/config/default/certmanager_metrics_manager_patch.yaml new file mode 100644 index 00000000000..258360938a2 --- /dev/null +++ b/testdata/project-v4-with-plugins/config/default/certmanager_metrics_manager_patch.yaml @@ -0,0 +1,22 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system + labels: + app.kubernetes.io/name: project-v4-with-plugins + app.kubernetes.io/managed-by: kustomize +spec: + template: + spec: + containers: + - name: manager + volumeMounts: + - mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-cert + readOnly: true + volumes: + - name: metrics-cert + secret: + defaultMode: 420 + secretName: metrics-server-cert diff --git a/testdata/project-v4-with-plugins/config/default/kustomization.yaml b/testdata/project-v4-with-plugins/config/default/kustomization.yaml index 3ae7a46eb9d..76289836199 100644 --- a/testdata/project-v4-with-plugins/config/default/kustomization.yaml +++ b/testdata/project-v4-with-plugins/config/default/kustomization.yaml @@ -33,7 +33,7 @@ resources: # be able to communicate with the Webhook Server. #- ../network-policy -# Uncomment the patches line if you enable Metrics, and/or are using webhooks and cert-manager +# Uncomment the patches line if you enable Metrics patches: # [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. # More info: https://book.kubebuilder.io/reference/metrics @@ -41,6 +41,11 @@ patches: target: kind: Deployment +# Uncomment the patches line if you enable Metrics and CertManager +# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. +# This patch will protect the metrics with certmanager self-signed certs. +# - path: certmanager_metrics_manager_patch.yaml + # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml - path: manager_webhook_patch.yaml diff --git a/testdata/project-v4-with-plugins/config/prometheus/kustomization.yaml b/testdata/project-v4-with-plugins/config/prometheus/kustomization.yaml index ed137168a1d..dd929ad8db7 100644 --- a/testdata/project-v4-with-plugins/config/prometheus/kustomization.yaml +++ b/testdata/project-v4-with-plugins/config/prometheus/kustomization.yaml @@ -1,2 +1,11 @@ resources: - monitor.yaml + +# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus +# to securely reference certificates created and managed by cert-manager. +# Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml +# to mount the "metrics-cert" secret in the Manager Deployment. +#patches: +# - path: monitor_tls_patch.yaml +# target: +# kind: ServiceMonitor diff --git a/testdata/project-v4-with-plugins/config/prometheus/monitor.yaml b/testdata/project-v4-with-plugins/config/prometheus/monitor.yaml index 58e9d5440eb..9b1f188adb9 100644 --- a/testdata/project-v4-with-plugins/config/prometheus/monitor.yaml +++ b/testdata/project-v4-with-plugins/config/prometheus/monitor.yaml @@ -16,14 +16,10 @@ spec: bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token tlsConfig: # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables - # certificate verification. This poses a significant security risk by making the system vulnerable to - # man-in-the-middle attacks, where an attacker could intercept and manipulate the communication between - # Prometheus and the monitored services. This could lead to unauthorized access to sensitive metrics data, - # compromising the integrity and confidentiality of the information. - # Please use the following options for secure configurations: - # caFile: /etc/metrics-certs/ca.crt - # certFile: /etc/metrics-certs/tls.crt - # keyFile: /etc/metrics-certs/tls.key + # certificate verification, exposing the system to potential man-in-the-middle attacks. + # For production environments, it is recommended to use cert-manager for automatic TLS certificate management. + # To apply this configuration, enable cert-manager and use the patch located at config/prometheus/servicemonitor_tls_patch.yaml, + # which securely references the certificate from the 'metrics-cert' secret. insecureSkipVerify: true selector: matchLabels: diff --git a/testdata/project-v4-with-plugins/config/prometheus/monitor_tls_patch.yaml b/testdata/project-v4-with-plugins/config/prometheus/monitor_tls_patch.yaml new file mode 100644 index 00000000000..8e5768127e9 --- /dev/null +++ b/testdata/project-v4-with-plugins/config/prometheus/monitor_tls_patch.yaml @@ -0,0 +1,22 @@ +# Patch for Prometheus ServiceMonitor to enable secure TLS configuration +# using certificates managed by cert-manager +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: controller-manager-metrics-monitor + namespace: system +spec: + endpoints: + - tlsConfig: + insecureSkipVerify: false + ca: + secret: + name: metrics-cert + key: ca.crt + cert: + secret: + name: metrics-cert + key: tls.crt + keySecret: + name: metrics-cert + key: tls.key diff --git a/testdata/project-v4/cmd/main.go b/testdata/project-v4/cmd/main.go index e7b186138dd..bc65eca5b7e 100644 --- a/testdata/project-v4/cmd/main.go +++ b/testdata/project-v4/cmd/main.go @@ -124,6 +124,13 @@ func main() { // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically // generate self-signed certificates for the metrics server. While convenient for development and testing, // this setup is not recommended for production. + + // TODO(user): If cert-manager is enabled in config/default/kustomization.yaml, + // you can uncomment the following lines to use the certificate managed by cert-manager. + // metricsServerOptions.CertDir = "/metrics-cert" + // metricsServerOptions.CertName = "tls.crt" + // metricsServerOptions.KeyName = "tls.key" + } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ diff --git a/testdata/project-v4/config/certmanager/certificate.yaml b/testdata/project-v4/config/certmanager/certificate.yaml index c7e34e79ce3..cbfa888b8c1 100644 --- a/testdata/project-v4/config/certmanager/certificate.yaml +++ b/testdata/project-v4/config/certmanager/certificate.yaml @@ -33,3 +33,25 @@ spec: kind: Issuer name: selfsigned-issuer secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: certificate + app.kubernetes.io/instance: serving-cert + app.kubernetes.io/component: certificate + app.kubernetes.io/created-by: project-v4 + app.kubernetes.io/part-of: project-v4 + app.kubernetes.io/managed-by: kustomize + name: metrics-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + dnsNames: + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: metrics-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/testdata/project-v4/config/default/certmanager_metrics_manager_patch.yaml b/testdata/project-v4/config/default/certmanager_metrics_manager_patch.yaml new file mode 100644 index 00000000000..c7e659c8f92 --- /dev/null +++ b/testdata/project-v4/config/default/certmanager_metrics_manager_patch.yaml @@ -0,0 +1,22 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system + labels: + app.kubernetes.io/name: project-v4 + app.kubernetes.io/managed-by: kustomize +spec: + template: + spec: + containers: + - name: manager + volumeMounts: + - mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-cert + readOnly: true + volumes: + - name: metrics-cert + secret: + defaultMode: 420 + secretName: metrics-server-cert diff --git a/testdata/project-v4/config/default/kustomization.yaml b/testdata/project-v4/config/default/kustomization.yaml index abb243b2308..c3385b7411b 100644 --- a/testdata/project-v4/config/default/kustomization.yaml +++ b/testdata/project-v4/config/default/kustomization.yaml @@ -33,7 +33,7 @@ resources: # be able to communicate with the Webhook Server. #- ../network-policy -# Uncomment the patches line if you enable Metrics, and/or are using webhooks and cert-manager +# Uncomment the patches line if you enable Metrics patches: # [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. # More info: https://book.kubebuilder.io/reference/metrics @@ -41,6 +41,11 @@ patches: target: kind: Deployment +# Uncomment the patches line if you enable Metrics and CertManager +# [METRICS WITH CERTMANGER] To enable metrics protected with certmanager, uncomment the following line. +# This patch will protect the metrics with certmanager self-signed certs. +# - path: certmanager_metrics_manager_patch.yaml + # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml - path: manager_webhook_patch.yaml diff --git a/testdata/project-v4/config/prometheus/kustomization.yaml b/testdata/project-v4/config/prometheus/kustomization.yaml index ed137168a1d..dd929ad8db7 100644 --- a/testdata/project-v4/config/prometheus/kustomization.yaml +++ b/testdata/project-v4/config/prometheus/kustomization.yaml @@ -1,2 +1,11 @@ resources: - monitor.yaml + +# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus +# to securely reference certificates created and managed by cert-manager. +# Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml +# to mount the "metrics-cert" secret in the Manager Deployment. +#patches: +# - path: monitor_tls_patch.yaml +# target: +# kind: ServiceMonitor diff --git a/testdata/project-v4/config/prometheus/monitor.yaml b/testdata/project-v4/config/prometheus/monitor.yaml index 1e3f1aec14c..26c2f1cb75f 100644 --- a/testdata/project-v4/config/prometheus/monitor.yaml +++ b/testdata/project-v4/config/prometheus/monitor.yaml @@ -16,14 +16,10 @@ spec: bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token tlsConfig: # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables - # certificate verification. This poses a significant security risk by making the system vulnerable to - # man-in-the-middle attacks, where an attacker could intercept and manipulate the communication between - # Prometheus and the monitored services. This could lead to unauthorized access to sensitive metrics data, - # compromising the integrity and confidentiality of the information. - # Please use the following options for secure configurations: - # caFile: /etc/metrics-certs/ca.crt - # certFile: /etc/metrics-certs/tls.crt - # keyFile: /etc/metrics-certs/tls.key + # certificate verification, exposing the system to potential man-in-the-middle attacks. + # For production environments, it is recommended to use cert-manager for automatic TLS certificate management. + # To apply this configuration, enable cert-manager and use the patch located at config/prometheus/servicemonitor_tls_patch.yaml, + # which securely references the certificate from the 'metrics-cert' secret. insecureSkipVerify: true selector: matchLabels: diff --git a/testdata/project-v4/config/prometheus/monitor_tls_patch.yaml b/testdata/project-v4/config/prometheus/monitor_tls_patch.yaml new file mode 100644 index 00000000000..8e5768127e9 --- /dev/null +++ b/testdata/project-v4/config/prometheus/monitor_tls_patch.yaml @@ -0,0 +1,22 @@ +# Patch for Prometheus ServiceMonitor to enable secure TLS configuration +# using certificates managed by cert-manager +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: controller-manager-metrics-monitor + namespace: system +spec: + endpoints: + - tlsConfig: + insecureSkipVerify: false + ca: + secret: + name: metrics-cert + key: ca.crt + cert: + secret: + name: metrics-cert + key: tls.crt + keySecret: + name: metrics-cert + key: tls.key