diff --git a/packages/elasticsearch/changelog.yml b/packages/elasticsearch/changelog.yml index 2d32b1f3a6c..57e6a6ccc23 100644 --- a/packages/elasticsearch/changelog.yml +++ b/packages/elasticsearch/changelog.yml @@ -1,4 +1,12 @@ # newer versions go on top +- version: "1.3.0" + changes: + - description: Add conditional support to Elasticsearch log and metrics inputs + type: enhancement + link: https://github.com/elastic/integrations/issues/5353 + - description: Add leader election support to Elasticsearch inputs metrics inputs + type: enhancement + link: https://github.com/elastic/integrations/issues/5353 - version: "1.2.0" changes: - description: Add period variable to define polling frequency diff --git a/packages/elasticsearch/data_stream/audit/agent/stream/log.yml.hbs b/packages/elasticsearch/data_stream/audit/agent/stream/log.yml.hbs index 9df020413f3..dbb3d8ec36b 100644 --- a/packages/elasticsearch/data_stream/audit/agent/stream/log.yml.hbs +++ b/packages/elasticsearch/data_stream/audit/agent/stream/log.yml.hbs @@ -2,6 +2,9 @@ paths: {{#each paths as |path i|}} - {{path}} {{/each}} +{{#if condition}} +condition: {{ condition }} +{{/if}} exclude_files: [".gz$"] processors: - add_locale: ~ diff --git a/packages/elasticsearch/data_stream/ccr/agent/stream/stream.yml.hbs b/packages/elasticsearch/data_stream/ccr/agent/stream/stream.yml.hbs index 5107bb8568d..7e63bdf8ae7 100644 --- a/packages/elasticsearch/data_stream/ccr/agent/stream/stream.yml.hbs +++ b/packages/elasticsearch/data_stream/ccr/agent/stream/stream.yml.hbs @@ -13,4 +13,15 @@ password: {{password}} period: {{period}} {{#if ssl}} ssl: {{ssl}} +{{/if}} +{{#if leaderelection }} +{{#if condition }} +condition: ${kubernetes_leaderelection.leader} == true and {{ condition }} +{{ else }} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} +{{ else }} +{{#if condition }} +condition: {{ condition }} +{{/if}} {{/if}} \ No newline at end of file diff --git a/packages/elasticsearch/data_stream/cluster_stats/agent/stream/stream.yml.hbs b/packages/elasticsearch/data_stream/cluster_stats/agent/stream/stream.yml.hbs index 7a92ad83179..c9fcaa8732b 100644 --- a/packages/elasticsearch/data_stream/cluster_stats/agent/stream/stream.yml.hbs +++ b/packages/elasticsearch/data_stream/cluster_stats/agent/stream/stream.yml.hbs @@ -13,4 +13,15 @@ password: {{password}} period: {{period}} {{#if ssl}} ssl: {{ssl}} +{{/if}} +{{#if leaderelection }} +{{#if condition }} +condition: ${kubernetes_leaderelection.leader} == true and {{ condition }} +{{ else }} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} +{{ else }} +{{#if condition }} +condition: {{ condition }} +{{/if}} {{/if}} \ No newline at end of file diff --git a/packages/elasticsearch/data_stream/deprecation/agent/stream/log.yml.hbs b/packages/elasticsearch/data_stream/deprecation/agent/stream/log.yml.hbs index f71e806435a..c10163dea79 100644 --- a/packages/elasticsearch/data_stream/deprecation/agent/stream/log.yml.hbs +++ b/packages/elasticsearch/data_stream/deprecation/agent/stream/log.yml.hbs @@ -2,4 +2,7 @@ paths: {{#each paths as |path i|}} - {{path}} {{/each}} +{{#if condition}} +condition: {{ condition }} +{{/if}} exclude_files: [".gz$","_slowlog.log$","_access.log$"] \ No newline at end of file diff --git a/packages/elasticsearch/data_stream/enrich/agent/stream/stream.yml.hbs b/packages/elasticsearch/data_stream/enrich/agent/stream/stream.yml.hbs index 2409e06a1e4..516dcd1ced1 100644 --- a/packages/elasticsearch/data_stream/enrich/agent/stream/stream.yml.hbs +++ b/packages/elasticsearch/data_stream/enrich/agent/stream/stream.yml.hbs @@ -13,4 +13,15 @@ password: {{password}} period: {{period}} {{#if ssl}} ssl: {{ssl}} +{{/if}} +{{#if leaderelection }} +{{#if condition }} +condition: ${kubernetes_leaderelection.leader} == true and {{ condition }} +{{ else }} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} +{{ else }} +{{#if condition }} +condition: {{ condition }} +{{/if}} {{/if}} \ No newline at end of file diff --git a/packages/elasticsearch/data_stream/gc/agent/stream/log.yml.hbs b/packages/elasticsearch/data_stream/gc/agent/stream/log.yml.hbs index 76304b3314b..f1a9a299bf2 100644 --- a/packages/elasticsearch/data_stream/gc/agent/stream/log.yml.hbs +++ b/packages/elasticsearch/data_stream/gc/agent/stream/log.yml.hbs @@ -2,6 +2,9 @@ paths: {{#each paths as |path i|}} - {{path}} {{/each}} +{{#if condition}} +condition: {{ condition }} +{{/if}} exclude_files: [".gz$"] exclude_lines: ["^(OpenJDK|Java HotSpot).* Server VM ", "^CommandLine flags: ", "^Memory: ", "^{"] # exclude JVM8 banner and JSON multiline: diff --git a/packages/elasticsearch/data_stream/index/agent/stream/stream.yml.hbs b/packages/elasticsearch/data_stream/index/agent/stream/stream.yml.hbs index 4363a43d625..d31882fa7ca 100644 --- a/packages/elasticsearch/data_stream/index/agent/stream/stream.yml.hbs +++ b/packages/elasticsearch/data_stream/index/agent/stream/stream.yml.hbs @@ -13,4 +13,15 @@ password: {{password}} period: {{period}} {{#if ssl}} ssl: {{ssl}} +{{/if}} +{{#if leaderelection }} +{{#if condition }} +condition: ${kubernetes_leaderelection.leader} == true and {{ condition }} +{{ else }} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} +{{ else }} +{{#if condition }} +condition: {{ condition }} +{{/if}} {{/if}} \ No newline at end of file diff --git a/packages/elasticsearch/data_stream/index_recovery/agent/stream/stream.yml.hbs b/packages/elasticsearch/data_stream/index_recovery/agent/stream/stream.yml.hbs index a315f0c013a..de36761d7d1 100644 --- a/packages/elasticsearch/data_stream/index_recovery/agent/stream/stream.yml.hbs +++ b/packages/elasticsearch/data_stream/index_recovery/agent/stream/stream.yml.hbs @@ -13,4 +13,15 @@ password: {{password}} period: {{period}} {{#if ssl}} ssl: {{ssl}} +{{/if}} +{{#if leaderelection }} +{{#if condition }} +condition: ${kubernetes_leaderelection.leader} == true and {{ condition }} +{{ else }} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} +{{ else }} +{{#if condition }} +condition: {{ condition }} +{{/if}} {{/if}} \ No newline at end of file diff --git a/packages/elasticsearch/data_stream/index_summary/agent/stream/stream.yml.hbs b/packages/elasticsearch/data_stream/index_summary/agent/stream/stream.yml.hbs index 8fe8a04792f..e98e753be10 100644 --- a/packages/elasticsearch/data_stream/index_summary/agent/stream/stream.yml.hbs +++ b/packages/elasticsearch/data_stream/index_summary/agent/stream/stream.yml.hbs @@ -13,4 +13,15 @@ password: {{password}} period: {{period}} {{#if ssl}} ssl: {{ssl}} +{{/if}} +{{#if leaderelection }} +{{#if condition }} +condition: ${kubernetes_leaderelection.leader} == true and {{ condition }} +{{ else }} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} +{{ else }} +{{#if condition }} +condition: {{ condition }} +{{/if}} {{/if}} \ No newline at end of file diff --git a/packages/elasticsearch/data_stream/ml_job/agent/stream/stream.yml.hbs b/packages/elasticsearch/data_stream/ml_job/agent/stream/stream.yml.hbs index d86b2a76e8b..6e9de5fca1e 100644 --- a/packages/elasticsearch/data_stream/ml_job/agent/stream/stream.yml.hbs +++ b/packages/elasticsearch/data_stream/ml_job/agent/stream/stream.yml.hbs @@ -13,4 +13,15 @@ password: {{password}} period: {{period}} {{#if ssl}} ssl: {{ssl}} +{{/if}} +{{#if leaderelection }} +{{#if condition }} +condition: ${kubernetes_leaderelection.leader} == true and {{ condition }} +{{ else }} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} +{{ else }} +{{#if condition }} +condition: {{ condition }} +{{/if}} {{/if}} \ No newline at end of file diff --git a/packages/elasticsearch/data_stream/node/agent/stream/stream.yml.hbs b/packages/elasticsearch/data_stream/node/agent/stream/stream.yml.hbs index ebb22c7f32b..463a0625b87 100644 --- a/packages/elasticsearch/data_stream/node/agent/stream/stream.yml.hbs +++ b/packages/elasticsearch/data_stream/node/agent/stream/stream.yml.hbs @@ -13,4 +13,15 @@ password: {{password}} period: {{period}} {{#if ssl}} ssl: {{ssl}} +{{/if}} +{{#if leaderelection }} +{{#if condition }} +condition: ${kubernetes_leaderelection.leader} == true and {{ condition }} +{{ else }} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} +{{ else }} +{{#if condition }} +condition: {{ condition }} +{{/if}} {{/if}} \ No newline at end of file diff --git a/packages/elasticsearch/data_stream/node_stats/agent/stream/stream.yml.hbs b/packages/elasticsearch/data_stream/node_stats/agent/stream/stream.yml.hbs index 1286f69c293..b94fe55803c 100644 --- a/packages/elasticsearch/data_stream/node_stats/agent/stream/stream.yml.hbs +++ b/packages/elasticsearch/data_stream/node_stats/agent/stream/stream.yml.hbs @@ -13,4 +13,15 @@ password: {{password}} period: {{period}} {{#if ssl}} ssl: {{ssl}} +{{/if}} +{{#if leaderelection }} +{{#if condition }} +condition: ${kubernetes_leaderelection.leader} == true and {{ condition }} +{{ else }} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} +{{ else }} +{{#if condition }} +condition: {{ condition }} +{{/if}} {{/if}} \ No newline at end of file diff --git a/packages/elasticsearch/data_stream/pending_tasks/agent/stream/stream.yml.hbs b/packages/elasticsearch/data_stream/pending_tasks/agent/stream/stream.yml.hbs index f45c90ac403..41afa52e3ed 100644 --- a/packages/elasticsearch/data_stream/pending_tasks/agent/stream/stream.yml.hbs +++ b/packages/elasticsearch/data_stream/pending_tasks/agent/stream/stream.yml.hbs @@ -13,4 +13,15 @@ password: {{password}} period: {{period}} {{#if ssl}} ssl: {{ssl}} +{{/if}} +{{#if leaderelection }} +{{#if condition }} +condition: ${kubernetes_leaderelection.leader} == true and {{ condition }} +{{ else }} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} +{{ else }} +{{#if condition }} +condition: {{ condition }} +{{/if}} {{/if}} \ No newline at end of file diff --git a/packages/elasticsearch/data_stream/server/agent/stream/log.yml.hbs b/packages/elasticsearch/data_stream/server/agent/stream/log.yml.hbs index e0e5d1ffa98..698ed7ea9c8 100644 --- a/packages/elasticsearch/data_stream/server/agent/stream/log.yml.hbs +++ b/packages/elasticsearch/data_stream/server/agent/stream/log.yml.hbs @@ -2,4 +2,7 @@ paths: {{#each paths as |path i|}} - {{path}} {{/each}} +{{#if condition}} +condition: {{ condition }} +{{/if}} exclude_files: [".gz$","_slowlog.log$","_access.log$","_deprecation.log$"] \ No newline at end of file diff --git a/packages/elasticsearch/data_stream/shard/agent/stream/stream.yml.hbs b/packages/elasticsearch/data_stream/shard/agent/stream/stream.yml.hbs index b0475a8c354..e5b626ba0e8 100644 --- a/packages/elasticsearch/data_stream/shard/agent/stream/stream.yml.hbs +++ b/packages/elasticsearch/data_stream/shard/agent/stream/stream.yml.hbs @@ -13,4 +13,15 @@ password: {{password}} period: {{period}} {{#if ssl}} ssl: {{ssl}} +{{/if}} +{{#if leaderelection }} +{{#if condition }} +condition: ${kubernetes_leaderelection.leader} == true and {{ condition }} +{{ else }} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} +{{ else }} +{{#if condition }} +condition: {{ condition }} +{{/if}} {{/if}} \ No newline at end of file diff --git a/packages/elasticsearch/data_stream/slowlog/agent/stream/log.yml.hbs b/packages/elasticsearch/data_stream/slowlog/agent/stream/log.yml.hbs index 367360d901a..809534c4e1e 100644 --- a/packages/elasticsearch/data_stream/slowlog/agent/stream/log.yml.hbs +++ b/packages/elasticsearch/data_stream/slowlog/agent/stream/log.yml.hbs @@ -2,4 +2,7 @@ paths: {{#each paths as |path i|}} - {{path}} {{/each}} +{{#if condition}} +condition: {{ condition }} +{{/if}} exclude_files: [".gz$"] \ No newline at end of file diff --git a/packages/elasticsearch/manifest.yml b/packages/elasticsearch/manifest.yml index 553f0b3cddb..d02fa3750da 100644 --- a/packages/elasticsearch/manifest.yml +++ b/packages/elasticsearch/manifest.yml @@ -1,6 +1,6 @@ name: elasticsearch title: Elasticsearch -version: 1.2.0 +version: 1.3.0 description: Elasticsearch Integration type: integration icons: @@ -21,6 +21,14 @@ policy_templates: - type: logfile title: Collect Elasticsearch logs description: "Collecting audit, deprecation, gc, server and slowlog logs from Elasticsearch instances (input: logfile)" + vars: + - name: condition + title: Condition + description: Condition to filter when to collect this input + type: text + multi: false + required: false + show_user: false - type: elasticsearch/metrics title: Collect Elasticsearch metrics description: Collect Elasticsearch metrics about indices, CCR, cluster stats, machine learning or node statistics @@ -67,5 +75,20 @@ policy_templates: #certificate_authorities: ["/etc/ca.crt"] #certificate: "/etc/client.crt" #key: "/etc/client.key" + - name: leaderelection + type: bool + title: Leader Election + description: Enable leaderelection between a set of Elastic Agents running on Kubernetes. Useful for when scope is `cluster`. + multi: false + required: true + show_user: false + default: false + - name: condition + title: Condition + description: Condition to filter when to collect this input + type: text + multi: false + required: false + show_user: false owner: github: elastic/infra-monitoring-ui