From e01147b0b26cfc3bdb5923a73046f10e1d93af50 Mon Sep 17 00:00:00 2001 From: Thibault Richard Date: Thu, 22 Jul 2021 12:01:48 +0200 Subject: [PATCH 01/26] Init doc --- docs/advanced-topics/advanced-topics.asciidoc | 2 ++ docs/advanced-topics/stack-monitoring.asciidoc | 11 +++++++++++ 2 files changed, 13 insertions(+) create mode 100644 docs/advanced-topics/stack-monitoring.asciidoc diff --git a/docs/advanced-topics/advanced-topics.asciidoc b/docs/advanced-topics/advanced-topics.asciidoc index 10a69500a7..76088361c6 100644 --- a/docs/advanced-topics/advanced-topics.asciidoc +++ b/docs/advanced-topics/advanced-topics.asciidoc @@ -15,6 +15,7 @@ endif::[] - <<{p}-traffic-splitting>> - <<{p}-network-policies>> - <<{p}-webhook-namespace-selectors>> +- <<{p}-stack-monitoring>> -- include::openshift.asciidoc[leveloffset=+1] @@ -23,3 +24,4 @@ include::service-meshes.asciidoc[leveloffset=+1] include::traffic-splitting.asciidoc[leveloffset=+1] include::network-policies.asciidoc[leveloffset=+1] include::webhook-namespace-selectors.asciidoc[leveloffset=+1] +include::stack-monitoring.asciidoc[leveloffset=+1] diff --git a/docs/advanced-topics/stack-monitoring.asciidoc b/docs/advanced-topics/stack-monitoring.asciidoc new file mode 100644 index 0000000000..82ef71ba13 --- /dev/null +++ b/docs/advanced-topics/stack-monitoring.asciidoc @@ -0,0 +1,11 @@ +:page_id: stack-monitoring +ifdef::env-github[] +**** +link:https://www.elastic.co/guide/en/cloud-on-k8s/master/k8s-{page_id}.html[View this document on the Elastic website] +**** +endif::[] + +[id="{p}-{page_id}"] += Stack Monitoring + +TODO From 7ce4aaca931b4ad8afd983125eaa0af29016459c Mon Sep 17 00:00:00 2001 From: Thibault Richard Date: Thu, 22 Jul 2021 12:23:13 +0200 Subject: [PATCH 02/26] draft --- .../advanced-topics/stack-monitoring.asciidoc | 141 +++++++++++++++++- 1 file changed, 140 insertions(+), 1 deletion(-) diff --git a/docs/advanced-topics/stack-monitoring.asciidoc b/docs/advanced-topics/stack-monitoring.asciidoc index 82ef71ba13..d03eb2ffa3 100644 --- a/docs/advanced-topics/stack-monitoring.asciidoc +++ b/docs/advanced-topics/stack-monitoring.asciidoc @@ -8,4 +8,143 @@ endif::[] [id="{p}-{page_id}"] = Stack Monitoring -TODO +You can enable link:https://www.elastic.co/guide/en/elasticsearch/reference/current/monitor-elasticsearch-cluster.html[Stack Monitoring] +on your Elasticsearch and Kibana to collect and ship their metrics and logs. + +You just have to reference the separate monitoring Elasticsearch cluster in the `spec.monitoring` section. + +[source,yaml,subs="attributes,callouts"] +---- +cat $$<<$$EOF | kubectl apply -f - +apiVersion: elasticsearch.k8s.elastic.co/{eck_crd_version} +kind: Elasticsearch +metadata: + name: monitored-sample + namespace: production +spec: + version: {version} + monitoring: + metrics: + elasticsearchRefs: + - name: monitoring + namespace: observability + logs: + elasticsearchRefs: + - name: monitoring + namespace: observability + nodeSets: + - name: default + count: 1 + config: + node.store.allow_mmap: false +--- +apiVersion: kibana.k8s.elastic.co/{eck_crd_version} +kind: Kibana +metadata: + name: monitored-sample + namespace: production +spec: + version: {version} + elasticsearchRef: + name: monitored-sample + namespace: production + monitoring: + metrics: + elasticsearchRefs: + - name: monitoring + namespace: observability + logs: + elasticsearchRefs: + - name: monitoring + namespace: observability + count: 1 +EOF +---- + +The use of `namespace` is optional if the Elasticsearch cluster and the Kibana are running in the same namespace. + +The monitoring cluster must be managed by ECK in the same Kubernetes cluster as the monitored one. + +You can send metrics and logs to two different Elasticsearch monitoring clusters. + +You can enable Stack Monitoring only on Elasticsearch or only on Kibana. In the latter case, Kibana will not be available on the Stack Monitoring Kibana page (see link:https://www.elastic.co/guide/en/kibana/current/monitoring-data.html#monitoring-data(View monitoring data in Kibana)). + +== How it works + +In the background, Metricbeat and Filebeat are deployed as sidecar containers in the same Pod than Elasticsearch and Kibana. + +Metricbeat is used to collect monitoring metrics and Filebeat to monitor the Elasticsearch log files, collect log events. The two beats are configured to ship data directly to the monitoring cluster(s) using dedicated Elastic users managed by ECK. + +Elasticsearch and Kibana containers are configured to share the + +== Audit logging + +Audit logs are collected and ship to the monitoring cluster referenced in the `monitoring.logs` section when audit logging is enabled (disabled by default). + +[source,yaml,subs="attributes,callouts"] +---- +apiVersion: elasticsearch.k8s.elastic.co/v1 +kind: Elasticsearch +spec: + monitoring: + metrics: + elasticsearchRefs: + - name: monitoring + namespace: observability + logs: + elasticsearchRefs: + - name: monitoring + namespace: observability + nodeSets: + - name: default + config: + # https://www.elastic.co/guide/en/elasticsearch/reference/current/enable-audit-logging.html + xpack.security.audit.enabled: true +--- +apiVersion: kibana.k8s.elastic.co/v1 +kind: Kibana +spec: + monitoring: + metrics: + elasticsearchRefs: + - name: monitoring + namespace: observability + logs: + elasticsearchRefs: + - name: monitoring + namespace: observability + config: + # https://www.elastic.co/guide/en/kibana/current/xpack-security-audit-logging.html + xpack.security.audit.enabled: true +---- + +== Override Beats Pod Template + +You can customize the Filebeat and Metricbeat containers through the Pod template. Your config is merged with our defaults. + +[source,yaml,subs="attributes,callouts"] +---- +apiVersion: elasticsearch.k8s.elastic.co/v1 +kind: Elasticsearch +spec: + nodeSets: + - name: default + monitoring: + metrics: + elasticsearchRef: + name: monitoring + namespace: observability + logs: + elasticsearchRef: + name: monitoring + namespace: observability + podTemplate: + spec: + containers: + - name: metricbeat + env: + - foo: bar + - name: filebeat + env: + - foo: bar +---- From 9ff7c289bd657baaa7910d9cf103c926d12e5915 Mon Sep 17 00:00:00 2001 From: Thibault Richard Date: Tue, 27 Jul 2021 11:59:59 +0200 Subject: [PATCH 03/26] Update docs/advanced-topics/stack-monitoring.asciidoc Co-authored-by: Peter Brachwitz --- docs/advanced-topics/stack-monitoring.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/advanced-topics/stack-monitoring.asciidoc b/docs/advanced-topics/stack-monitoring.asciidoc index d03eb2ffa3..e76b343099 100644 --- a/docs/advanced-topics/stack-monitoring.asciidoc +++ b/docs/advanced-topics/stack-monitoring.asciidoc @@ -67,7 +67,7 @@ The monitoring cluster must be managed by ECK in the same Kubernetes cluster as You can send metrics and logs to two different Elasticsearch monitoring clusters. -You can enable Stack Monitoring only on Elasticsearch or only on Kibana. In the latter case, Kibana will not be available on the Stack Monitoring Kibana page (see link:https://www.elastic.co/guide/en/kibana/current/monitoring-data.html#monitoring-data(View monitoring data in Kibana)). +You can also enable Stack Monitoring only on Elasticsearch or only on Kibana. In the latter case, Kibana will not be available on the Stack Monitoring Kibana page (see link:https://www.elastic.co/guide/en/kibana/current/monitoring-data.html#monitoring-data(View monitoring data in Kibana)). == How it works From 4b7c8a336bfc0f1dc149d1f6f6c3df6aa8d0cabd Mon Sep 17 00:00:00 2001 From: Thibault Richard Date: Tue, 27 Jul 2021 12:00:07 +0200 Subject: [PATCH 04/26] Update docs/advanced-topics/stack-monitoring.asciidoc Co-authored-by: Peter Brachwitz --- docs/advanced-topics/stack-monitoring.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/advanced-topics/stack-monitoring.asciidoc b/docs/advanced-topics/stack-monitoring.asciidoc index e76b343099..3870acab61 100644 --- a/docs/advanced-topics/stack-monitoring.asciidoc +++ b/docs/advanced-topics/stack-monitoring.asciidoc @@ -11,7 +11,7 @@ endif::[] You can enable link:https://www.elastic.co/guide/en/elasticsearch/reference/current/monitor-elasticsearch-cluster.html[Stack Monitoring] on your Elasticsearch and Kibana to collect and ship their metrics and logs. -You just have to reference the separate monitoring Elasticsearch cluster in the `spec.monitoring` section. +You just have to reference a separate monitoring Elasticsearch cluster in the `spec.monitoring` section. [source,yaml,subs="attributes,callouts"] ---- From 2cb786b843ff8ff296c1a1f5284539be5ea103ba Mon Sep 17 00:00:00 2001 From: Thibault Richard Date: Tue, 27 Jul 2021 12:00:33 +0200 Subject: [PATCH 05/26] Update docs/advanced-topics/stack-monitoring.asciidoc Co-authored-by: Peter Brachwitz --- docs/advanced-topics/stack-monitoring.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/advanced-topics/stack-monitoring.asciidoc b/docs/advanced-topics/stack-monitoring.asciidoc index 3870acab61..9f124e7b8a 100644 --- a/docs/advanced-topics/stack-monitoring.asciidoc +++ b/docs/advanced-topics/stack-monitoring.asciidoc @@ -118,7 +118,7 @@ spec: xpack.security.audit.enabled: true ---- -== Override Beats Pod Template +== Override the Beats Pod Template You can customize the Filebeat and Metricbeat containers through the Pod template. Your config is merged with our defaults. From e0a504d349984eb6dab732e64b5f9fc9286cf393 Mon Sep 17 00:00:00 2001 From: Thibault Richard Date: Tue, 27 Jul 2021 12:00:56 +0200 Subject: [PATCH 06/26] Update docs/advanced-topics/stack-monitoring.asciidoc Co-authored-by: Peter Brachwitz --- docs/advanced-topics/stack-monitoring.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/advanced-topics/stack-monitoring.asciidoc b/docs/advanced-topics/stack-monitoring.asciidoc index 9f124e7b8a..2194bafeb2 100644 --- a/docs/advanced-topics/stack-monitoring.asciidoc +++ b/docs/advanced-topics/stack-monitoring.asciidoc @@ -120,7 +120,7 @@ spec: == Override the Beats Pod Template -You can customize the Filebeat and Metricbeat containers through the Pod template. Your config is merged with our defaults. +You can customize the Filebeat and Metricbeat containers through the Pod template. Your config is merged with the values of the default Pod template ECK uses. [source,yaml,subs="attributes,callouts"] ---- From 3808407df336ad608220835417623da91b663f21 Mon Sep 17 00:00:00 2001 From: Thibault Richard Date: Tue, 27 Jul 2021 12:01:05 +0200 Subject: [PATCH 07/26] Update docs/advanced-topics/stack-monitoring.asciidoc Co-authored-by: Peter Brachwitz --- docs/advanced-topics/stack-monitoring.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/advanced-topics/stack-monitoring.asciidoc b/docs/advanced-topics/stack-monitoring.asciidoc index 2194bafeb2..18a318b79b 100644 --- a/docs/advanced-topics/stack-monitoring.asciidoc +++ b/docs/advanced-topics/stack-monitoring.asciidoc @@ -73,7 +73,7 @@ You can also enable Stack Monitoring only on Elasticsearch or only on Kibana. In In the background, Metricbeat and Filebeat are deployed as sidecar containers in the same Pod than Elasticsearch and Kibana. -Metricbeat is used to collect monitoring metrics and Filebeat to monitor the Elasticsearch log files, collect log events. The two beats are configured to ship data directly to the monitoring cluster(s) using dedicated Elastic users managed by ECK. +Metricbeat is used to collect monitoring metrics and Filebeat to monitor the Elasticsearch log files, collect log events. The two Beats are configured to ship data directly to the monitoring cluster(s) using dedicated Elastic users managed by ECK. Elasticsearch and Kibana containers are configured to share the From 9b8db69f6009fade545a39b1b267823784cb7230 Mon Sep 17 00:00:00 2001 From: Thibault Richard Date: Tue, 27 Jul 2021 12:01:17 +0200 Subject: [PATCH 08/26] Update docs/advanced-topics/stack-monitoring.asciidoc Co-authored-by: Peter Brachwitz --- docs/advanced-topics/stack-monitoring.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/advanced-topics/stack-monitoring.asciidoc b/docs/advanced-topics/stack-monitoring.asciidoc index 18a318b79b..cd88987c0b 100644 --- a/docs/advanced-topics/stack-monitoring.asciidoc +++ b/docs/advanced-topics/stack-monitoring.asciidoc @@ -71,7 +71,7 @@ You can also enable Stack Monitoring only on Elasticsearch or only on Kibana. In == How it works -In the background, Metricbeat and Filebeat are deployed as sidecar containers in the same Pod than Elasticsearch and Kibana. +In the background, Metricbeat and Filebeat are deployed as sidecar containers in the same Pod as Elasticsearch and Kibana. Metricbeat is used to collect monitoring metrics and Filebeat to monitor the Elasticsearch log files, collect log events. The two Beats are configured to ship data directly to the monitoring cluster(s) using dedicated Elastic users managed by ECK. From 8d8cf00536b3777de978463c3724c38a31b3cb49 Mon Sep 17 00:00:00 2001 From: Thibault Richard Date: Tue, 27 Jul 2021 16:03:11 +0200 Subject: [PATCH 09/26] Use callouts --- docs/advanced-topics/stack-monitoring.asciidoc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/advanced-topics/stack-monitoring.asciidoc b/docs/advanced-topics/stack-monitoring.asciidoc index cd88987c0b..f803f62ea9 100644 --- a/docs/advanced-topics/stack-monitoring.asciidoc +++ b/docs/advanced-topics/stack-monitoring.asciidoc @@ -47,21 +47,22 @@ spec: version: {version} elasticsearchRef: name: monitored-sample - namespace: production + namespace: production <1> monitoring: metrics: elasticsearchRefs: - name: monitoring - namespace: observability + namespace: observability <2> logs: elasticsearchRefs: - name: monitoring - namespace: observability + namespace: observability <2> count: 1 EOF ---- -The use of `namespace` is optional if the Elasticsearch cluster and the Kibana are running in the same namespace. +<1> The use of `namespace` is optional if the Elasticsearch cluster and the Kibana are running in the same namespace. +<2> The use of `namespace` is optional if the monitoring Elasticsearch cluster and the monitored Elasticsearch cluster are running in the same namespace. The monitoring cluster must be managed by ECK in the same Kubernetes cluster as the monitored one. From 2b04361804cfcad322d8bbab84d78ec0f070376e Mon Sep 17 00:00:00 2001 From: Thibault Richard Date: Tue, 27 Jul 2021 16:04:47 +0200 Subject: [PATCH 10/26] Fix link syntax --- docs/advanced-topics/stack-monitoring.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/advanced-topics/stack-monitoring.asciidoc b/docs/advanced-topics/stack-monitoring.asciidoc index f803f62ea9..9641e05a88 100644 --- a/docs/advanced-topics/stack-monitoring.asciidoc +++ b/docs/advanced-topics/stack-monitoring.asciidoc @@ -68,7 +68,7 @@ The monitoring cluster must be managed by ECK in the same Kubernetes cluster as You can send metrics and logs to two different Elasticsearch monitoring clusters. -You can also enable Stack Monitoring only on Elasticsearch or only on Kibana. In the latter case, Kibana will not be available on the Stack Monitoring Kibana page (see link:https://www.elastic.co/guide/en/kibana/current/monitoring-data.html#monitoring-data(View monitoring data in Kibana)). +You can also enable Stack Monitoring only on Elasticsearch or only on Kibana. In the latter case, Kibana will not be available on the Stack Monitoring Kibana page (see link:https://www.elastic.co/guide/en/kibana/current/monitoring-data.html#monitoring-data[View monitoring data in Kibana]). == How it works From 8d99e30e9a25b75bf0376367c11f5983acbf793b Mon Sep 17 00:00:00 2001 From: Thibault Richard Date: Tue, 27 Jul 2021 16:15:43 +0200 Subject: [PATCH 11/26] Remove incomplete sentence --- docs/advanced-topics/stack-monitoring.asciidoc | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/advanced-topics/stack-monitoring.asciidoc b/docs/advanced-topics/stack-monitoring.asciidoc index 9641e05a88..be7e52c134 100644 --- a/docs/advanced-topics/stack-monitoring.asciidoc +++ b/docs/advanced-topics/stack-monitoring.asciidoc @@ -76,8 +76,6 @@ In the background, Metricbeat and Filebeat are deployed as sidecar containers in Metricbeat is used to collect monitoring metrics and Filebeat to monitor the Elasticsearch log files, collect log events. The two Beats are configured to ship data directly to the monitoring cluster(s) using dedicated Elastic users managed by ECK. -Elasticsearch and Kibana containers are configured to share the - == Audit logging Audit logs are collected and ship to the monitoring cluster referenced in the `monitoring.logs` section when audit logging is enabled (disabled by default). From bd9c04c6bdcc67910749841df7e57a7a73086b16 Mon Sep 17 00:00:00 2001 From: Thibault Richard Date: Tue, 27 Jul 2021 16:45:11 +0200 Subject: [PATCH 12/26] Add 'when to use it' section --- docs/advanced-topics/stack-monitoring.asciidoc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/advanced-topics/stack-monitoring.asciidoc b/docs/advanced-topics/stack-monitoring.asciidoc index be7e52c134..e759798013 100644 --- a/docs/advanced-topics/stack-monitoring.asciidoc +++ b/docs/advanced-topics/stack-monitoring.asciidoc @@ -70,6 +70,14 @@ You can send metrics and logs to two different Elasticsearch monitoring clusters You can also enable Stack Monitoring only on Elasticsearch or only on Kibana. In the latter case, Kibana will not be available on the Stack Monitoring Kibana page (see link:https://www.elastic.co/guide/en/kibana/current/monitoring-data.html#monitoring-data[View monitoring data in Kibana]). +== When to use it + +This feature is a good solution if you need to monitor your Elastic applications in restricted Kubernetes environments where you cannot grant advanced permissions: +- to Metricbeat to allow queriying the k8s API +- to Filebeat to deploy a privileged DaemonSet + +However, for maximum efficiency and minimising resource consumption, or advanced use cases that require specific Beats configurations, you can deploy a standalone Metricbeat Deployment and a Filebeat Daemonset. See the <<{p}-beat-configuration-examples,Beats configuration Examples>> for more information. + == How it works In the background, Metricbeat and Filebeat are deployed as sidecar containers in the same Pod as Elasticsearch and Kibana. From afe653b5c8788beb5ea92bbaa28b3fa58e98c7ef Mon Sep 17 00:00:00 2001 From: Thibault Richard Date: Tue, 27 Jul 2021 22:53:29 +0200 Subject: [PATCH 13/26] Add empty new lines --- .../advanced-topics/stack-monitoring.asciidoc | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/docs/advanced-topics/stack-monitoring.asciidoc b/docs/advanced-topics/stack-monitoring.asciidoc index e759798013..a6e763a54b 100644 --- a/docs/advanced-topics/stack-monitoring.asciidoc +++ b/docs/advanced-topics/stack-monitoring.asciidoc @@ -27,11 +27,11 @@ spec: metrics: elasticsearchRefs: - name: monitoring - namespace: observability + namespace: observability <1> logs: elasticsearchRefs: - name: monitoring - namespace: observability + namespace: observability <1> nodeSets: - name: default count: 1 @@ -47,22 +47,22 @@ spec: version: {version} elasticsearchRef: name: monitored-sample - namespace: production <1> + namespace: production <2> monitoring: metrics: elasticsearchRefs: - name: monitoring - namespace: observability <2> + namespace: observability <1> logs: elasticsearchRefs: - name: monitoring - namespace: observability <2> + namespace: observability <1> count: 1 EOF ---- -<1> The use of `namespace` is optional if the Elasticsearch cluster and the Kibana are running in the same namespace. -<2> The use of `namespace` is optional if the monitoring Elasticsearch cluster and the monitored Elasticsearch cluster are running in the same namespace. +<1> The use of `namespace` is optional if the monitoring Elasticsearch cluster and the monitored Elasticsearch cluster are running in the same namespace. +<2> The use of `namespace` is optional if the Elasticsearch cluster and the Kibana are running in the same namespace. The monitoring cluster must be managed by ECK in the same Kubernetes cluster as the monitored one. @@ -73,6 +73,7 @@ You can also enable Stack Monitoring only on Elasticsearch or only on Kibana. In == When to use it This feature is a good solution if you need to monitor your Elastic applications in restricted Kubernetes environments where you cannot grant advanced permissions: + - to Metricbeat to allow queriying the k8s API - to Filebeat to deploy a privileged DaemonSet @@ -82,7 +83,9 @@ However, for maximum efficiency and minimising resource consumption, or advanced In the background, Metricbeat and Filebeat are deployed as sidecar containers in the same Pod as Elasticsearch and Kibana. -Metricbeat is used to collect monitoring metrics and Filebeat to monitor the Elasticsearch log files, collect log events. The two Beats are configured to ship data directly to the monitoring cluster(s) using dedicated Elastic users managed by ECK. +Metricbeat is used to collect monitoring metrics and Filebeat to monitor the Elasticsearch log files and collect log events. + +The two Beats are configured to ship data directly to the monitoring cluster(s) using dedicated Elastic users managed by ECK. == Audit logging From 57e6abedfab7e9974c2d14deab29654d88386084 Mon Sep 17 00:00:00 2001 From: Thibault Richard Date: Wed, 28 Jul 2021 14:02:40 +0200 Subject: [PATCH 14/26] YAML indentation --- .../advanced-topics/stack-monitoring.asciidoc | 46 +++++++++---------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/docs/advanced-topics/stack-monitoring.asciidoc b/docs/advanced-topics/stack-monitoring.asciidoc index a6e763a54b..1defb52875 100644 --- a/docs/advanced-topics/stack-monitoring.asciidoc +++ b/docs/advanced-topics/stack-monitoring.asciidoc @@ -15,7 +15,6 @@ You just have to reference a separate monitoring Elasticsearch cluster in the `s [source,yaml,subs="attributes,callouts"] ---- -cat $$<<$$EOF | kubectl apply -f - apiVersion: elasticsearch.k8s.elastic.co/{eck_crd_version} kind: Elasticsearch metadata: @@ -26,12 +25,12 @@ spec: monitoring: metrics: elasticsearchRefs: - - name: monitoring - namespace: observability <1> + - name: monitoring + namespace: observability <1> logs: elasticsearchRefs: - - name: monitoring - namespace: observability <1> + - name: monitoring + namespace: observability <1> nodeSets: - name: default count: 1 @@ -51,14 +50,13 @@ spec: monitoring: metrics: elasticsearchRefs: - - name: monitoring - namespace: observability <1> + - name: monitoring + namespace: observability <1> logs: elasticsearchRefs: - - name: monitoring - namespace: observability <1> + - name: monitoring + namespace: observability <1> count: 1 -EOF ---- <1> The use of `namespace` is optional if the monitoring Elasticsearch cluster and the monitored Elasticsearch cluster are running in the same namespace. @@ -99,12 +97,12 @@ spec: monitoring: metrics: elasticsearchRefs: - - name: monitoring - namespace: observability + - name: monitoring + namespace: observability logs: elasticsearchRefs: - - name: monitoring - namespace: observability + - name: monitoring + namespace: observability nodeSets: - name: default config: @@ -117,12 +115,12 @@ spec: monitoring: metrics: elasticsearchRefs: - - name: monitoring - namespace: observability + - name: monitoring + namespace: observability logs: elasticsearchRefs: - - name: monitoring - namespace: observability + - name: monitoring + namespace: observability config: # https://www.elastic.co/guide/en/kibana/current/xpack-security-audit-logging.html xpack.security.audit.enabled: true @@ -151,10 +149,10 @@ spec: podTemplate: spec: containers: - - name: metricbeat - env: - - foo: bar - - name: filebeat - env: - - foo: bar + - name: metricbeat + env: + - foo: bar + - name: filebeat + env: + - foo: bar ---- From 63861ad21c31b6e49aca26d0fe514c15a530f504 Mon Sep 17 00:00:00 2001 From: Thibault Richard Date: Wed, 28 Jul 2021 14:11:52 +0200 Subject: [PATCH 15/26] Add a caution note about self monitoring --- docs/advanced-topics/stack-monitoring.asciidoc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/advanced-topics/stack-monitoring.asciidoc b/docs/advanced-topics/stack-monitoring.asciidoc index 1defb52875..8b8eb2b7eb 100644 --- a/docs/advanced-topics/stack-monitoring.asciidoc +++ b/docs/advanced-topics/stack-monitoring.asciidoc @@ -62,7 +62,9 @@ spec: <1> The use of `namespace` is optional if the monitoring Elasticsearch cluster and the monitored Elasticsearch cluster are running in the same namespace. <2> The use of `namespace` is optional if the Elasticsearch cluster and the Kibana are running in the same namespace. -The monitoring cluster must be managed by ECK in the same Kubernetes cluster as the monitored one. +CAUTION: You cannot configure an Elasticsearch cluster to monitor itself, the monitoring cluster has to be a separate cluster. + +IMPORTANT: The monitoring cluster must be managed by ECK in the same Kubernetes cluster as the monitored one. You can send metrics and logs to two different Elasticsearch monitoring clusters. From b56003872cd8ebec17fd7c1b6ee7fc9a80a8776e Mon Sep 17 00:00:00 2001 From: Thibault Richard Date: Wed, 28 Jul 2021 14:12:31 +0200 Subject: [PATCH 16/26] Precise via HTTPS --- docs/advanced-topics/stack-monitoring.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/advanced-topics/stack-monitoring.asciidoc b/docs/advanced-topics/stack-monitoring.asciidoc index 8b8eb2b7eb..8f05adc1e6 100644 --- a/docs/advanced-topics/stack-monitoring.asciidoc +++ b/docs/advanced-topics/stack-monitoring.asciidoc @@ -85,7 +85,7 @@ In the background, Metricbeat and Filebeat are deployed as sidecar containers in Metricbeat is used to collect monitoring metrics and Filebeat to monitor the Elasticsearch log files and collect log events. -The two Beats are configured to ship data directly to the monitoring cluster(s) using dedicated Elastic users managed by ECK. +The two Beats are configured to ship data directly to the monitoring cluster(s) using HTTPS and dedicated Elastic users managed by ECK. == Audit logging From ca167cf9be550dde6f216744ae3d4fb64a39f3d5 Mon Sep 17 00:00:00 2001 From: Thibault Richard Date: Wed, 28 Jul 2021 14:20:18 +0200 Subject: [PATCH 17/26] Link documentation in the highlights --- docs/release-notes/highlights-1.7.0.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/release-notes/highlights-1.7.0.asciidoc b/docs/release-notes/highlights-1.7.0.asciidoc index cd86fa7143..d538cb45b0 100644 --- a/docs/release-notes/highlights-1.7.0.asciidoc +++ b/docs/release-notes/highlights-1.7.0.asciidoc @@ -19,7 +19,7 @@ Starting with this release, the `CustomResourceDefinitions` (CRD) and the `Valid In this release, the Elasticsearch and Kibana resources have been enhanced to let you specify a reference to a monitoring cluster. When specified, sidecar containers are automatically setup by ECK to ship logs and metrics to the referenced Elasticsearch cluster. -*Add a link to documentation* +<<{p}-stack-monitoring,Stack Monitoring documentation>> [float] [id="{p}-170-autoscaling"] From b90b1c5bec0f5969fca7d51064e3f5d6e2354700 Mon Sep 17 00:00:00 2001 From: Thibault Richard Date: Wed, 28 Jul 2021 15:58:38 +0200 Subject: [PATCH 18/26] Update docs/advanced-topics/stack-monitoring.asciidoc Co-authored-by: David Kilfoyle <41695641+kilfoyle@users.noreply.github.com> --- docs/advanced-topics/stack-monitoring.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/advanced-topics/stack-monitoring.asciidoc b/docs/advanced-topics/stack-monitoring.asciidoc index 8f05adc1e6..bee6b5f1a1 100644 --- a/docs/advanced-topics/stack-monitoring.asciidoc +++ b/docs/advanced-topics/stack-monitoring.asciidoc @@ -60,7 +60,7 @@ spec: ---- <1> The use of `namespace` is optional if the monitoring Elasticsearch cluster and the monitored Elasticsearch cluster are running in the same namespace. -<2> The use of `namespace` is optional if the Elasticsearch cluster and the Kibana are running in the same namespace. +<2> The use of `namespace` is optional if the Elasticsearch cluster and the Kibana instance are running in the same namespace. CAUTION: You cannot configure an Elasticsearch cluster to monitor itself, the monitoring cluster has to be a separate cluster. From 2e03f55a377e8bbbf92e246f701e170f0620c4a3 Mon Sep 17 00:00:00 2001 From: Thibault Richard Date: Wed, 28 Jul 2021 15:58:52 +0200 Subject: [PATCH 19/26] Update docs/advanced-topics/stack-monitoring.asciidoc Co-authored-by: David Kilfoyle <41695641+kilfoyle@users.noreply.github.com> --- docs/advanced-topics/stack-monitoring.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/advanced-topics/stack-monitoring.asciidoc b/docs/advanced-topics/stack-monitoring.asciidoc index bee6b5f1a1..12108a6595 100644 --- a/docs/advanced-topics/stack-monitoring.asciidoc +++ b/docs/advanced-topics/stack-monitoring.asciidoc @@ -68,7 +68,7 @@ IMPORTANT: The monitoring cluster must be managed by ECK in the same Kubernetes You can send metrics and logs to two different Elasticsearch monitoring clusters. -You can also enable Stack Monitoring only on Elasticsearch or only on Kibana. In the latter case, Kibana will not be available on the Stack Monitoring Kibana page (see link:https://www.elastic.co/guide/en/kibana/current/monitoring-data.html#monitoring-data[View monitoring data in Kibana]). +You can also enable Stack Monitoring on Elasticsearch only or on Kibana only. In the latter case, Kibana will not be available on the Stack Monitoring Kibana page (see link:https://www.elastic.co/guide/en/kibana/current/monitoring-data.html#monitoring-data[View monitoring data in Kibana]). == When to use it From 0d68923ea1c0296ea85f5bcace5d2d02f9730966 Mon Sep 17 00:00:00 2001 From: Thibault Richard Date: Wed, 28 Jul 2021 15:59:16 +0200 Subject: [PATCH 20/26] Update docs/advanced-topics/stack-monitoring.asciidoc Co-authored-by: David Kilfoyle <41695641+kilfoyle@users.noreply.github.com> --- docs/advanced-topics/stack-monitoring.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/advanced-topics/stack-monitoring.asciidoc b/docs/advanced-topics/stack-monitoring.asciidoc index 12108a6595..b4a748d8fc 100644 --- a/docs/advanced-topics/stack-monitoring.asciidoc +++ b/docs/advanced-topics/stack-monitoring.asciidoc @@ -89,7 +89,7 @@ The two Beats are configured to ship data directly to the monitoring cluster(s) == Audit logging -Audit logs are collected and ship to the monitoring cluster referenced in the `monitoring.logs` section when audit logging is enabled (disabled by default). +Audit logs are collected and shipped to the monitoring cluster referenced in the `monitoring.logs` section when audit logging is enabled (it is disabled by default). [source,yaml,subs="attributes,callouts"] ---- From 8baa1ebbf2a658ca0e3f9122b765e00ef9973590 Mon Sep 17 00:00:00 2001 From: Thibault Richard Date: Wed, 28 Jul 2021 15:59:26 +0200 Subject: [PATCH 21/26] Update docs/advanced-topics/stack-monitoring.asciidoc Co-authored-by: David Kilfoyle <41695641+kilfoyle@users.noreply.github.com> --- docs/advanced-topics/stack-monitoring.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/advanced-topics/stack-monitoring.asciidoc b/docs/advanced-topics/stack-monitoring.asciidoc index b4a748d8fc..ad7903a53b 100644 --- a/docs/advanced-topics/stack-monitoring.asciidoc +++ b/docs/advanced-topics/stack-monitoring.asciidoc @@ -130,7 +130,7 @@ spec: == Override the Beats Pod Template -You can customize the Filebeat and Metricbeat containers through the Pod template. Your config is merged with the values of the default Pod template ECK uses. +You can customize the Filebeat and Metricbeat containers through the Pod template. Your configuration is merged with the values of the default Pod template that ECK uses. [source,yaml,subs="attributes,callouts"] ---- From 92c6cf07ec7770046d24d3055c7c3462a455b4c8 Mon Sep 17 00:00:00 2001 From: Thibault Richard Date: Wed, 28 Jul 2021 16:02:24 +0200 Subject: [PATCH 22/26] Update docs/advanced-topics/stack-monitoring.asciidoc Co-authored-by: David Kilfoyle <41695641+kilfoyle@users.noreply.github.com> --- docs/advanced-topics/stack-monitoring.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/advanced-topics/stack-monitoring.asciidoc b/docs/advanced-topics/stack-monitoring.asciidoc index ad7903a53b..8a5c1ca28f 100644 --- a/docs/advanced-topics/stack-monitoring.asciidoc +++ b/docs/advanced-topics/stack-monitoring.asciidoc @@ -9,7 +9,7 @@ endif::[] = Stack Monitoring You can enable link:https://www.elastic.co/guide/en/elasticsearch/reference/current/monitor-elasticsearch-cluster.html[Stack Monitoring] -on your Elasticsearch and Kibana to collect and ship their metrics and logs. +on Elasticsearch and Kibana to collect and ship their metrics and logs to a dedicated monitoring cluster. You just have to reference a separate monitoring Elasticsearch cluster in the `spec.monitoring` section. From b9ffb29655bc1c2033673726ca7e31dc4928daf2 Mon Sep 17 00:00:00 2001 From: Thibault Richard Date: Wed, 28 Jul 2021 16:15:52 +0200 Subject: [PATCH 23/26] Apply review input --- docs/advanced-topics/stack-monitoring.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/advanced-topics/stack-monitoring.asciidoc b/docs/advanced-topics/stack-monitoring.asciidoc index 8a5c1ca28f..3fa396f684 100644 --- a/docs/advanced-topics/stack-monitoring.asciidoc +++ b/docs/advanced-topics/stack-monitoring.asciidoc @@ -11,7 +11,7 @@ endif::[] You can enable link:https://www.elastic.co/guide/en/elasticsearch/reference/current/monitor-elasticsearch-cluster.html[Stack Monitoring] on Elasticsearch and Kibana to collect and ship their metrics and logs to a dedicated monitoring cluster. -You just have to reference a separate monitoring Elasticsearch cluster in the `spec.monitoring` section. +To enable stack monitoring, simply reference the monitoring Elasticsearch cluster in the `spec.monitoring` section of their specification. [source,yaml,subs="attributes,callouts"] ---- From 7e23538b68bdc1bae99a5a995ebc0f2b002916bc Mon Sep 17 00:00:00 2001 From: Thibault Richard Date: Thu, 29 Jul 2021 10:04:16 +0200 Subject: [PATCH 24/26] Update docs/release-notes/highlights-1.7.0.asciidoc Co-authored-by: Michael Morello --- docs/release-notes/highlights-1.7.0.asciidoc | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/release-notes/highlights-1.7.0.asciidoc b/docs/release-notes/highlights-1.7.0.asciidoc index d538cb45b0..09fa7b8c57 100644 --- a/docs/release-notes/highlights-1.7.0.asciidoc +++ b/docs/release-notes/highlights-1.7.0.asciidoc @@ -19,7 +19,6 @@ Starting with this release, the `CustomResourceDefinitions` (CRD) and the `Valid In this release, the Elasticsearch and Kibana resources have been enhanced to let you specify a reference to a monitoring cluster. When specified, sidecar containers are automatically setup by ECK to ship logs and metrics to the referenced Elasticsearch cluster. -<<{p}-stack-monitoring,Stack Monitoring documentation>> [float] [id="{p}-170-autoscaling"] From c6834b857f39dbd1b77608a615403724b6f2692e Mon Sep 17 00:00:00 2001 From: Thibault Richard Date: Thu, 29 Jul 2021 10:04:27 +0200 Subject: [PATCH 25/26] Update docs/release-notes/highlights-1.7.0.asciidoc Co-authored-by: Michael Morello --- docs/release-notes/highlights-1.7.0.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/release-notes/highlights-1.7.0.asciidoc b/docs/release-notes/highlights-1.7.0.asciidoc index 09fa7b8c57..6940012f92 100644 --- a/docs/release-notes/highlights-1.7.0.asciidoc +++ b/docs/release-notes/highlights-1.7.0.asciidoc @@ -17,7 +17,7 @@ Starting with this release, the `CustomResourceDefinitions` (CRD) and the `Valid [id="{p}-170-stack-monitoring"] ==== Stack Monitoring -In this release, the Elasticsearch and Kibana resources have been enhanced to let you specify a reference to a monitoring cluster. When specified, sidecar containers are automatically setup by ECK to ship logs and metrics to the referenced Elasticsearch cluster. +In this release, the Elasticsearch and Kibana resources have been enhanced to let you specify a reference to a monitoring cluster. When specified, sidecar containers are automatically setup by ECK to ship logs and metrics to the referenced Elasticsearch cluster. Refer to the <<{p}-stack-monitoring,stack monitoring documentation>> for more details. [float] From e9a6d6975166b8b82a2b8b95f7b4bada86132914 Mon Sep 17 00:00:00 2001 From: Thibault Richard Date: Thu, 29 Jul 2021 10:05:54 +0200 Subject: [PATCH 26/26] Case and empty lines --- docs/release-notes/highlights-1.7.0.asciidoc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/release-notes/highlights-1.7.0.asciidoc b/docs/release-notes/highlights-1.7.0.asciidoc index 6940012f92..63616df097 100644 --- a/docs/release-notes/highlights-1.7.0.asciidoc +++ b/docs/release-notes/highlights-1.7.0.asciidoc @@ -17,8 +17,7 @@ Starting with this release, the `CustomResourceDefinitions` (CRD) and the `Valid [id="{p}-170-stack-monitoring"] ==== Stack Monitoring -In this release, the Elasticsearch and Kibana resources have been enhanced to let you specify a reference to a monitoring cluster. When specified, sidecar containers are automatically setup by ECK to ship logs and metrics to the referenced Elasticsearch cluster. Refer to the <<{p}-stack-monitoring,stack monitoring documentation>> for more details. - +In this release, the Elasticsearch and Kibana resources have been enhanced to let you specify a reference to a monitoring cluster. When specified, sidecar containers are automatically setup by ECK to ship logs and metrics to the referenced Elasticsearch cluster. Refer to the <<{p}-stack-monitoring,Stack Monitoring documentation>> for more details. [float] [id="{p}-170-autoscaling"]