From eaf6d87f58934a67417ee1638a676f57741aa147 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 21 Feb 2023 20:30:27 -0500 Subject: [PATCH 1/3] Add condition and leader election to elasticsearch integration --- packages/elasticsearch/changelog.yml | 8 ++++++ .../audit/agent/stream/log.yml.hbs | 3 +++ .../ccr/agent/stream/stream.yml.hbs | 11 ++++++++ .../cluster_stats/agent/stream/stream.yml.hbs | 11 ++++++++ .../deprecation/agent/stream/log.yml.hbs | 3 +++ .../enrich/agent/stream/stream.yml.hbs | 11 ++++++++ .../data_stream/gc/agent/stream/log.yml.hbs | 3 +++ .../index/agent/stream/stream.yml.hbs | 11 ++++++++ .../agent/stream/stream.yml.hbs | 11 ++++++++ .../index_summary/agent/stream/stream.yml.hbs | 11 ++++++++ .../ml_job/agent/stream/stream.yml.hbs | 11 ++++++++ .../node/agent/stream/stream.yml.hbs | 11 ++++++++ .../node_stats/agent/stream/stream.yml.hbs | 11 ++++++++ .../pending_tasks/agent/stream/stream.yml.hbs | 11 ++++++++ .../server/agent/stream/log.yml.hbs | 3 +++ .../shard/agent/stream/stream.yml.hbs | 11 ++++++++ .../slowlog/agent/stream/log.yml.hbs | 3 +++ packages/elasticsearch/manifest.yml | 25 ++++++++++++++++++- 18 files changed, 168 insertions(+), 1 deletion(-) diff --git a/packages/elasticsearch/changelog.yml b/packages/elasticsearch/changelog.yml index 2d32b1f3a6c..abc0a7bfe8c 100644 --- a/packages/elasticsearch/changelog.yml +++ b/packages/elasticsearch/changelog.yml @@ -1,4 +1,12 @@ # newer versions go on top +- version: "1.3.0" + changes: + - description: Add conditional support to Elasticsearch log and metrics inputs + type: enhancement + link: https://github.com/elastic/integrations/issues/5141 + - description: Add leader election support to Elasticsearch inputs metrics inputs + type: enhancement + link: https://github.com/elastic/integrations/issues/5141 - version: "1.2.0" changes: - description: Add period variable to define polling frequency diff --git a/packages/elasticsearch/data_stream/audit/agent/stream/log.yml.hbs b/packages/elasticsearch/data_stream/audit/agent/stream/log.yml.hbs index 9df020413f3..dbb3d8ec36b 100644 --- a/packages/elasticsearch/data_stream/audit/agent/stream/log.yml.hbs +++ b/packages/elasticsearch/data_stream/audit/agent/stream/log.yml.hbs @@ -2,6 +2,9 @@ paths: {{#each paths as |path i|}} - {{path}} {{/each}} +{{#if condition}} +condition: {{ condition }} +{{/if}} exclude_files: [".gz$"] processors: - add_locale: ~ diff --git a/packages/elasticsearch/data_stream/ccr/agent/stream/stream.yml.hbs b/packages/elasticsearch/data_stream/ccr/agent/stream/stream.yml.hbs index 5107bb8568d..7e63bdf8ae7 100644 --- a/packages/elasticsearch/data_stream/ccr/agent/stream/stream.yml.hbs +++ b/packages/elasticsearch/data_stream/ccr/agent/stream/stream.yml.hbs @@ -13,4 +13,15 @@ password: {{password}} period: {{period}} {{#if ssl}} ssl: {{ssl}} +{{/if}} +{{#if leaderelection }} +{{#if condition }} +condition: ${kubernetes_leaderelection.leader} == true and {{ condition }} +{{ else }} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} +{{ else }} +{{#if condition }} +condition: {{ condition }} +{{/if}} {{/if}} \ No newline at end of file diff --git a/packages/elasticsearch/data_stream/cluster_stats/agent/stream/stream.yml.hbs b/packages/elasticsearch/data_stream/cluster_stats/agent/stream/stream.yml.hbs index 7a92ad83179..c9fcaa8732b 100644 --- a/packages/elasticsearch/data_stream/cluster_stats/agent/stream/stream.yml.hbs +++ b/packages/elasticsearch/data_stream/cluster_stats/agent/stream/stream.yml.hbs @@ -13,4 +13,15 @@ password: {{password}} period: {{period}} {{#if ssl}} ssl: {{ssl}} +{{/if}} +{{#if leaderelection }} +{{#if condition }} +condition: ${kubernetes_leaderelection.leader} == true and {{ condition }} +{{ else }} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} +{{ else }} +{{#if condition }} +condition: {{ condition }} +{{/if}} {{/if}} \ No newline at end of file diff --git a/packages/elasticsearch/data_stream/deprecation/agent/stream/log.yml.hbs b/packages/elasticsearch/data_stream/deprecation/agent/stream/log.yml.hbs index f71e806435a..c10163dea79 100644 --- a/packages/elasticsearch/data_stream/deprecation/agent/stream/log.yml.hbs +++ b/packages/elasticsearch/data_stream/deprecation/agent/stream/log.yml.hbs @@ -2,4 +2,7 @@ paths: {{#each paths as |path i|}} - {{path}} {{/each}} +{{#if condition}} +condition: {{ condition }} +{{/if}} exclude_files: [".gz$","_slowlog.log$","_access.log$"] \ No newline at end of file diff --git a/packages/elasticsearch/data_stream/enrich/agent/stream/stream.yml.hbs b/packages/elasticsearch/data_stream/enrich/agent/stream/stream.yml.hbs index 2409e06a1e4..516dcd1ced1 100644 --- a/packages/elasticsearch/data_stream/enrich/agent/stream/stream.yml.hbs +++ b/packages/elasticsearch/data_stream/enrich/agent/stream/stream.yml.hbs @@ -13,4 +13,15 @@ password: {{password}} period: {{period}} {{#if ssl}} ssl: {{ssl}} +{{/if}} +{{#if leaderelection }} +{{#if condition }} +condition: ${kubernetes_leaderelection.leader} == true and {{ condition }} +{{ else }} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} +{{ else }} +{{#if condition }} +condition: {{ condition }} +{{/if}} {{/if}} \ No newline at end of file diff --git a/packages/elasticsearch/data_stream/gc/agent/stream/log.yml.hbs b/packages/elasticsearch/data_stream/gc/agent/stream/log.yml.hbs index 76304b3314b..f1a9a299bf2 100644 --- a/packages/elasticsearch/data_stream/gc/agent/stream/log.yml.hbs +++ b/packages/elasticsearch/data_stream/gc/agent/stream/log.yml.hbs @@ -2,6 +2,9 @@ paths: {{#each paths as |path i|}} - {{path}} {{/each}} +{{#if condition}} +condition: {{ condition }} +{{/if}} exclude_files: [".gz$"] exclude_lines: ["^(OpenJDK|Java HotSpot).* Server VM ", "^CommandLine flags: ", "^Memory: ", "^{"] # exclude JVM8 banner and JSON multiline: diff --git a/packages/elasticsearch/data_stream/index/agent/stream/stream.yml.hbs b/packages/elasticsearch/data_stream/index/agent/stream/stream.yml.hbs index 4363a43d625..d31882fa7ca 100644 --- a/packages/elasticsearch/data_stream/index/agent/stream/stream.yml.hbs +++ b/packages/elasticsearch/data_stream/index/agent/stream/stream.yml.hbs @@ -13,4 +13,15 @@ password: {{password}} period: {{period}} {{#if ssl}} ssl: {{ssl}} +{{/if}} +{{#if leaderelection }} +{{#if condition }} +condition: ${kubernetes_leaderelection.leader} == true and {{ condition }} +{{ else }} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} +{{ else }} +{{#if condition }} +condition: {{ condition }} +{{/if}} {{/if}} \ No newline at end of file diff --git a/packages/elasticsearch/data_stream/index_recovery/agent/stream/stream.yml.hbs b/packages/elasticsearch/data_stream/index_recovery/agent/stream/stream.yml.hbs index a315f0c013a..de36761d7d1 100644 --- a/packages/elasticsearch/data_stream/index_recovery/agent/stream/stream.yml.hbs +++ b/packages/elasticsearch/data_stream/index_recovery/agent/stream/stream.yml.hbs @@ -13,4 +13,15 @@ password: {{password}} period: {{period}} {{#if ssl}} ssl: {{ssl}} +{{/if}} +{{#if leaderelection }} +{{#if condition }} +condition: ${kubernetes_leaderelection.leader} == true and {{ condition }} +{{ else }} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} +{{ else }} +{{#if condition }} +condition: {{ condition }} +{{/if}} {{/if}} \ No newline at end of file diff --git a/packages/elasticsearch/data_stream/index_summary/agent/stream/stream.yml.hbs b/packages/elasticsearch/data_stream/index_summary/agent/stream/stream.yml.hbs index 8fe8a04792f..e98e753be10 100644 --- a/packages/elasticsearch/data_stream/index_summary/agent/stream/stream.yml.hbs +++ b/packages/elasticsearch/data_stream/index_summary/agent/stream/stream.yml.hbs @@ -13,4 +13,15 @@ password: {{password}} period: {{period}} {{#if ssl}} ssl: {{ssl}} +{{/if}} +{{#if leaderelection }} +{{#if condition }} +condition: ${kubernetes_leaderelection.leader} == true and {{ condition }} +{{ else }} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} +{{ else }} +{{#if condition }} +condition: {{ condition }} +{{/if}} {{/if}} \ No newline at end of file diff --git a/packages/elasticsearch/data_stream/ml_job/agent/stream/stream.yml.hbs b/packages/elasticsearch/data_stream/ml_job/agent/stream/stream.yml.hbs index d86b2a76e8b..6e9de5fca1e 100644 --- a/packages/elasticsearch/data_stream/ml_job/agent/stream/stream.yml.hbs +++ b/packages/elasticsearch/data_stream/ml_job/agent/stream/stream.yml.hbs @@ -13,4 +13,15 @@ password: {{password}} period: {{period}} {{#if ssl}} ssl: {{ssl}} +{{/if}} +{{#if leaderelection }} +{{#if condition }} +condition: ${kubernetes_leaderelection.leader} == true and {{ condition }} +{{ else }} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} +{{ else }} +{{#if condition }} +condition: {{ condition }} +{{/if}} {{/if}} \ No newline at end of file diff --git a/packages/elasticsearch/data_stream/node/agent/stream/stream.yml.hbs b/packages/elasticsearch/data_stream/node/agent/stream/stream.yml.hbs index ebb22c7f32b..463a0625b87 100644 --- a/packages/elasticsearch/data_stream/node/agent/stream/stream.yml.hbs +++ b/packages/elasticsearch/data_stream/node/agent/stream/stream.yml.hbs @@ -13,4 +13,15 @@ password: {{password}} period: {{period}} {{#if ssl}} ssl: {{ssl}} +{{/if}} +{{#if leaderelection }} +{{#if condition }} +condition: ${kubernetes_leaderelection.leader} == true and {{ condition }} +{{ else }} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} +{{ else }} +{{#if condition }} +condition: {{ condition }} +{{/if}} {{/if}} \ No newline at end of file diff --git a/packages/elasticsearch/data_stream/node_stats/agent/stream/stream.yml.hbs b/packages/elasticsearch/data_stream/node_stats/agent/stream/stream.yml.hbs index 1286f69c293..b94fe55803c 100644 --- a/packages/elasticsearch/data_stream/node_stats/agent/stream/stream.yml.hbs +++ b/packages/elasticsearch/data_stream/node_stats/agent/stream/stream.yml.hbs @@ -13,4 +13,15 @@ password: {{password}} period: {{period}} {{#if ssl}} ssl: {{ssl}} +{{/if}} +{{#if leaderelection }} +{{#if condition }} +condition: ${kubernetes_leaderelection.leader} == true and {{ condition }} +{{ else }} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} +{{ else }} +{{#if condition }} +condition: {{ condition }} +{{/if}} {{/if}} \ No newline at end of file diff --git a/packages/elasticsearch/data_stream/pending_tasks/agent/stream/stream.yml.hbs b/packages/elasticsearch/data_stream/pending_tasks/agent/stream/stream.yml.hbs index f45c90ac403..41afa52e3ed 100644 --- a/packages/elasticsearch/data_stream/pending_tasks/agent/stream/stream.yml.hbs +++ b/packages/elasticsearch/data_stream/pending_tasks/agent/stream/stream.yml.hbs @@ -13,4 +13,15 @@ password: {{password}} period: {{period}} {{#if ssl}} ssl: {{ssl}} +{{/if}} +{{#if leaderelection }} +{{#if condition }} +condition: ${kubernetes_leaderelection.leader} == true and {{ condition }} +{{ else }} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} +{{ else }} +{{#if condition }} +condition: {{ condition }} +{{/if}} {{/if}} \ No newline at end of file diff --git a/packages/elasticsearch/data_stream/server/agent/stream/log.yml.hbs b/packages/elasticsearch/data_stream/server/agent/stream/log.yml.hbs index e0e5d1ffa98..698ed7ea9c8 100644 --- a/packages/elasticsearch/data_stream/server/agent/stream/log.yml.hbs +++ b/packages/elasticsearch/data_stream/server/agent/stream/log.yml.hbs @@ -2,4 +2,7 @@ paths: {{#each paths as |path i|}} - {{path}} {{/each}} +{{#if condition}} +condition: {{ condition }} +{{/if}} exclude_files: [".gz$","_slowlog.log$","_access.log$","_deprecation.log$"] \ No newline at end of file diff --git a/packages/elasticsearch/data_stream/shard/agent/stream/stream.yml.hbs b/packages/elasticsearch/data_stream/shard/agent/stream/stream.yml.hbs index b0475a8c354..e5b626ba0e8 100644 --- a/packages/elasticsearch/data_stream/shard/agent/stream/stream.yml.hbs +++ b/packages/elasticsearch/data_stream/shard/agent/stream/stream.yml.hbs @@ -13,4 +13,15 @@ password: {{password}} period: {{period}} {{#if ssl}} ssl: {{ssl}} +{{/if}} +{{#if leaderelection }} +{{#if condition }} +condition: ${kubernetes_leaderelection.leader} == true and {{ condition }} +{{ else }} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} +{{ else }} +{{#if condition }} +condition: {{ condition }} +{{/if}} {{/if}} \ No newline at end of file diff --git a/packages/elasticsearch/data_stream/slowlog/agent/stream/log.yml.hbs b/packages/elasticsearch/data_stream/slowlog/agent/stream/log.yml.hbs index 367360d901a..809534c4e1e 100644 --- a/packages/elasticsearch/data_stream/slowlog/agent/stream/log.yml.hbs +++ b/packages/elasticsearch/data_stream/slowlog/agent/stream/log.yml.hbs @@ -2,4 +2,7 @@ paths: {{#each paths as |path i|}} - {{path}} {{/each}} +{{#if condition}} +condition: {{ condition }} +{{/if}} exclude_files: [".gz$"] \ No newline at end of file diff --git a/packages/elasticsearch/manifest.yml b/packages/elasticsearch/manifest.yml index 553f0b3cddb..e0ad42d6819 100644 --- a/packages/elasticsearch/manifest.yml +++ b/packages/elasticsearch/manifest.yml @@ -1,6 +1,6 @@ name: elasticsearch title: Elasticsearch -version: 1.2.0 +version: 1.3.0 description: Elasticsearch Integration type: integration icons: @@ -21,6 +21,14 @@ policy_templates: - type: logfile title: Collect Elasticsearch logs description: "Collecting audit, deprecation, gc, server and slowlog logs from Elasticsearch instances (input: logfile)" + vars: + - name: condition + title: Condition + description: Condition to filter when to apply this datastream + type: text + multi: false + required: false + show_user: false - type: elasticsearch/metrics title: Collect Elasticsearch metrics description: Collect Elasticsearch metrics about indices, CCR, cluster stats, machine learning or node statistics @@ -67,5 +75,20 @@ policy_templates: #certificate_authorities: ["/etc/ca.crt"] #certificate: "/etc/client.crt" #key: "/etc/client.key" + - name: leaderelection + type: bool + title: Leader Election + description: Enable leaderelection between a set of Elastic Agents running on Kubernetes. Useful for when scope is `cluster`. + multi: false + required: true + show_user: false + default: false + - name: condition + title: Condition + description: Condition to filter when to apply this datastream + type: text + multi: false + required: false + show_user: false owner: github: elastic/infra-monitoring-ui From 66903ac13de7e40391402c95ad559e08c1917824 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 21 Feb 2023 20:43:21 -0500 Subject: [PATCH 2/3] Update changelog link --- packages/elasticsearch/changelog.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/elasticsearch/changelog.yml b/packages/elasticsearch/changelog.yml index abc0a7bfe8c..57e6a6ccc23 100644 --- a/packages/elasticsearch/changelog.yml +++ b/packages/elasticsearch/changelog.yml @@ -3,10 +3,10 @@ changes: - description: Add conditional support to Elasticsearch log and metrics inputs type: enhancement - link: https://github.com/elastic/integrations/issues/5141 + link: https://github.com/elastic/integrations/issues/5353 - description: Add leader election support to Elasticsearch inputs metrics inputs type: enhancement - link: https://github.com/elastic/integrations/issues/5141 + link: https://github.com/elastic/integrations/issues/5353 - version: "1.2.0" changes: - description: Add period variable to define polling frequency From 4c306122dad791c8fac3092608b1f083a6831652 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 23 Feb 2023 10:21:00 -0500 Subject: [PATCH 3/3] Update condition description wording --- packages/elasticsearch/manifest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/elasticsearch/manifest.yml b/packages/elasticsearch/manifest.yml index e0ad42d6819..d02fa3750da 100644 --- a/packages/elasticsearch/manifest.yml +++ b/packages/elasticsearch/manifest.yml @@ -24,7 +24,7 @@ policy_templates: vars: - name: condition title: Condition - description: Condition to filter when to apply this datastream + description: Condition to filter when to collect this input type: text multi: false required: false @@ -85,7 +85,7 @@ policy_templates: default: false - name: condition title: Condition - description: Condition to filter when to apply this datastream + description: Condition to filter when to collect this input type: text multi: false required: false