[Elasticsearch] Add condition and leader election to Elasticsearch in…

…tegration (elastic#5353) * Add condition and leader election to elasticsearch integration * Update changelog link * Update condition description wording
agithomas · Mar 20, 2023 · c775da6 · c775da6
1 parent a7e9153
commit c775da6
Show file tree

Hide file tree

Showing 18 changed files with 168 additions and 1 deletion.
diff --git a/packages/elasticsearch/changelog.yml b/packages/elasticsearch/changelog.yml
@@ -1,4 +1,12 @@
 # newer versions go on top
+- version: "1.3.0"
+  changes:
+    - description: Add conditional support to Elasticsearch log and metrics inputs
+      type: enhancement
+      link: https://github.com/elastic/integrations/issues/5353
+    - description: Add leader election support to Elasticsearch inputs metrics inputs
+      type: enhancement
+      link: https://github.com/elastic/integrations/issues/5353
 - version: "1.2.0"
   changes:
     - description: Add period variable to define polling frequency

diff --git a/packages/elasticsearch/data_stream/audit/agent/stream/log.yml.hbs b/packages/elasticsearch/data_stream/audit/agent/stream/log.yml.hbs
@@ -2,6 +2,9 @@ paths:
 {{#each paths as |path i|}}
  - {{path}}
 {{/each}}
+{{#if condition}}
+condition: {{ condition }}
+{{/if}}
 exclude_files: [".gz$"]
 processors:
   - add_locale: ~

diff --git a/packages/elasticsearch/data_stream/ccr/agent/stream/stream.yml.hbs b/packages/elasticsearch/data_stream/ccr/agent/stream/stream.yml.hbs
@@ -13,4 +13,15 @@ password: {{password}}
 period: {{period}}
 {{#if ssl}}
 ssl: {{ssl}}
+{{/if}}
+{{#if leaderelection }}
+{{#if condition }}
+condition: ${kubernetes_leaderelection.leader} == true and {{ condition }}
+{{ else }}
+condition: ${kubernetes_leaderelection.leader} == true
+{{/if}}
+{{ else }}
+{{#if condition }}
+condition: {{ condition }}
+{{/if}}
 {{/if}}
diff --git a/packages/elasticsearch/data_stream/cluster_stats/agent/stream/stream.yml.hbs b/packages/elasticsearch/data_stream/cluster_stats/agent/stream/stream.yml.hbs
@@ -13,4 +13,15 @@ password: {{password}}
 period: {{period}}
 {{#if ssl}}
 ssl: {{ssl}}
+{{/if}}
+{{#if leaderelection }}
+{{#if condition }}
+condition: ${kubernetes_leaderelection.leader} == true and {{ condition }}
+{{ else }}
+condition: ${kubernetes_leaderelection.leader} == true
+{{/if}}
+{{ else }}
+{{#if condition }}
+condition: {{ condition }}
+{{/if}}
 {{/if}}
diff --git a/packages/elasticsearch/data_stream/deprecation/agent/stream/log.yml.hbs b/packages/elasticsearch/data_stream/deprecation/agent/stream/log.yml.hbs
@@ -2,4 +2,7 @@ paths:
 {{#each paths as |path i|}}
  - {{path}}
 {{/each}}
+{{#if condition}}
+condition: {{ condition }}
+{{/if}}
 exclude_files: [".gz$","_slowlog.log$","_access.log$"]
diff --git a/packages/elasticsearch/data_stream/enrich/agent/stream/stream.yml.hbs b/packages/elasticsearch/data_stream/enrich/agent/stream/stream.yml.hbs
@@ -13,4 +13,15 @@ password: {{password}}
 period: {{period}}
 {{#if ssl}}
 ssl: {{ssl}}
+{{/if}}
+{{#if leaderelection }}
+{{#if condition }}
+condition: ${kubernetes_leaderelection.leader} == true and {{ condition }}
+{{ else }}
+condition: ${kubernetes_leaderelection.leader} == true
+{{/if}}
+{{ else }}
+{{#if condition }}
+condition: {{ condition }}
+{{/if}}
 {{/if}}
diff --git a/packages/elasticsearch/data_stream/gc/agent/stream/log.yml.hbs b/packages/elasticsearch/data_stream/gc/agent/stream/log.yml.hbs
@@ -2,6 +2,9 @@ paths:
 {{#each paths as |path i|}}
  - {{path}}
 {{/each}}
+{{#if condition}}
+condition: {{ condition }}
+{{/if}}
 exclude_files: [".gz$"]
 exclude_lines: ["^(OpenJDK|Java HotSpot).* Server VM ", "^CommandLine flags: ", "^Memory: ", "^{"] # exclude JVM8 banner and JSON
 multiline:

diff --git a/packages/elasticsearch/data_stream/index/agent/stream/stream.yml.hbs b/packages/elasticsearch/data_stream/index/agent/stream/stream.yml.hbs
@@ -13,4 +13,15 @@ password: {{password}}
 period: {{period}}
 {{#if ssl}}
 ssl: {{ssl}}
+{{/if}}
+{{#if leaderelection }}
+{{#if condition }}
+condition: ${kubernetes_leaderelection.leader} == true and {{ condition }}
+{{ else }}
+condition: ${kubernetes_leaderelection.leader} == true
+{{/if}}
+{{ else }}
+{{#if condition }}
+condition: {{ condition }}
+{{/if}}
 {{/if}}
diff --git a/packages/elasticsearch/data_stream/index_recovery/agent/stream/stream.yml.hbs b/packages/elasticsearch/data_stream/index_recovery/agent/stream/stream.yml.hbs
@@ -13,4 +13,15 @@ password: {{password}}
 period: {{period}}
 {{#if ssl}}
 ssl: {{ssl}}
+{{/if}}
+{{#if leaderelection }}
+{{#if condition }}
+condition: ${kubernetes_leaderelection.leader} == true and {{ condition }}
+{{ else }}
+condition: ${kubernetes_leaderelection.leader} == true
+{{/if}}
+{{ else }}
+{{#if condition }}
+condition: {{ condition }}
+{{/if}}
 {{/if}}
diff --git a/packages/elasticsearch/data_stream/index_summary/agent/stream/stream.yml.hbs b/packages/elasticsearch/data_stream/index_summary/agent/stream/stream.yml.hbs
@@ -13,4 +13,15 @@ password: {{password}}
 period: {{period}}
 {{#if ssl}}
 ssl: {{ssl}}
+{{/if}}
+{{#if leaderelection }}
+{{#if condition }}
+condition: ${kubernetes_leaderelection.leader} == true and {{ condition }}
+{{ else }}
+condition: ${kubernetes_leaderelection.leader} == true
+{{/if}}
+{{ else }}
+{{#if condition }}
+condition: {{ condition }}
+{{/if}}
 {{/if}}
diff --git a/packages/elasticsearch/data_stream/ml_job/agent/stream/stream.yml.hbs b/packages/elasticsearch/data_stream/ml_job/agent/stream/stream.yml.hbs
@@ -13,4 +13,15 @@ password: {{password}}
 period: {{period}}
 {{#if ssl}}
 ssl: {{ssl}}
+{{/if}}
+{{#if leaderelection }}
+{{#if condition }}
+condition: ${kubernetes_leaderelection.leader} == true and {{ condition }}
+{{ else }}
+condition: ${kubernetes_leaderelection.leader} == true
+{{/if}}
+{{ else }}
+{{#if condition }}
+condition: {{ condition }}
+{{/if}}
 {{/if}}
diff --git a/packages/elasticsearch/data_stream/node/agent/stream/stream.yml.hbs b/packages/elasticsearch/data_stream/node/agent/stream/stream.yml.hbs
@@ -13,4 +13,15 @@ password: {{password}}
 period: {{period}}
 {{#if ssl}}
 ssl: {{ssl}}
+{{/if}}
+{{#if leaderelection }}
+{{#if condition }}
+condition: ${kubernetes_leaderelection.leader} == true and {{ condition }}
+{{ else }}
+condition: ${kubernetes_leaderelection.leader} == true
+{{/if}}
+{{ else }}
+{{#if condition }}
+condition: {{ condition }}
+{{/if}}
 {{/if}}
diff --git a/packages/elasticsearch/data_stream/node_stats/agent/stream/stream.yml.hbs b/packages/elasticsearch/data_stream/node_stats/agent/stream/stream.yml.hbs
@@ -13,4 +13,15 @@ password: {{password}}
 period: {{period}}
 {{#if ssl}}
 ssl: {{ssl}}
+{{/if}}
+{{#if leaderelection }}
+{{#if condition }}
+condition: ${kubernetes_leaderelection.leader} == true and {{ condition }}
+{{ else }}
+condition: ${kubernetes_leaderelection.leader} == true
+{{/if}}
+{{ else }}
+{{#if condition }}
+condition: {{ condition }}
+{{/if}}
 {{/if}}
diff --git a/packages/elasticsearch/data_stream/pending_tasks/agent/stream/stream.yml.hbs b/packages/elasticsearch/data_stream/pending_tasks/agent/stream/stream.yml.hbs
@@ -13,4 +13,15 @@ password: {{password}}
 period: {{period}}
 {{#if ssl}}
 ssl: {{ssl}}
+{{/if}}
+{{#if leaderelection }}
+{{#if condition }}
+condition: ${kubernetes_leaderelection.leader} == true and {{ condition }}
+{{ else }}
+condition: ${kubernetes_leaderelection.leader} == true
+{{/if}}
+{{ else }}
+{{#if condition }}
+condition: {{ condition }}
+{{/if}}
 {{/if}}
diff --git a/packages/elasticsearch/data_stream/server/agent/stream/log.yml.hbs b/packages/elasticsearch/data_stream/server/agent/stream/log.yml.hbs
@@ -2,4 +2,7 @@ paths:
 {{#each paths as |path i|}}
  - {{path}}
 {{/each}}
+{{#if condition}}
+condition: {{ condition }}
+{{/if}}
 exclude_files: [".gz$","_slowlog.log$","_access.log$","_deprecation.log$"]
diff --git a/packages/elasticsearch/data_stream/shard/agent/stream/stream.yml.hbs b/packages/elasticsearch/data_stream/shard/agent/stream/stream.yml.hbs
@@ -13,4 +13,15 @@ password: {{password}}
 period: {{period}}
 {{#if ssl}}
 ssl: {{ssl}}
+{{/if}}
+{{#if leaderelection }}
+{{#if condition }}
+condition: ${kubernetes_leaderelection.leader} == true and {{ condition }}
+{{ else }}
+condition: ${kubernetes_leaderelection.leader} == true
+{{/if}}
+{{ else }}
+{{#if condition }}
+condition: {{ condition }}
+{{/if}}
 {{/if}}
diff --git a/packages/elasticsearch/data_stream/slowlog/agent/stream/log.yml.hbs b/packages/elasticsearch/data_stream/slowlog/agent/stream/log.yml.hbs
@@ -2,4 +2,7 @@ paths:
 {{#each paths as |path i|}}
  - {{path}}
 {{/each}}
+{{#if condition}}
+condition: {{ condition }}
+{{/if}}
 exclude_files: [".gz$"]
diff --git a/packages/elasticsearch/manifest.yml b/packages/elasticsearch/manifest.yml
@@ -1,6 +1,6 @@
 name: elasticsearch
 title: Elasticsearch
-version: 1.2.0
+version: 1.3.0
 description: Elasticsearch Integration
 type: integration
 icons:
@@ -21,6 +21,14 @@ policy_templates:
       - type: logfile
         title: Collect Elasticsearch logs
         description: "Collecting audit, deprecation, gc, server and slowlog logs from Elasticsearch instances (input: logfile)"
+        vars:
+          - name: condition
+            title: Condition
+            description: Condition to filter when to collect this input
+            type: text
+            multi: false
+            required: false
+            show_user: false
       - type: elasticsearch/metrics
         title: Collect Elasticsearch metrics
         description: Collect Elasticsearch metrics about indices, CCR, cluster stats, machine learning or node statistics
@@ -67,5 +75,20 @@ policy_templates:
               #certificate_authorities: ["/etc/ca.crt"]
               #certificate: "/etc/client.crt"
               #key: "/etc/client.key"
+          - name: leaderelection
+            type: bool
+            title: Leader Election
+            description: Enable leaderelection between a set of Elastic Agents running on Kubernetes. Useful for when scope is `cluster`.
+            multi: false
+            required: true
+            show_user: false
+            default: false
+          - name: condition
+            title: Condition
+            description: Condition to filter when to collect this input
+            type: text
+            multi: false
+            required: false
+            show_user: false
 owner:
   github: elastic/infra-monitoring-ui