Update with more cluster and node fields

elastic · Dec 8, 2022 · 264bc06 · 264bc06
1 parent 7c5ce88
commit 264bc06
Show file tree

Hide file tree

Showing 5 changed files with 2,976 additions and 85 deletions.
diff --git a/packages/elasticsearch/data_stream/ingest/agent/stream/stream.yml.hbs b/packages/elasticsearch/data_stream/ingest/agent/stream/stream.yml.hbs
@@ -9,29 +9,31 @@ auth.basic:
 
 request:
   method: "GET"
-  url: {{hosts.[0]}}/_nodes/stats/ingest?filter_path=nodes.*.ingest
+  url: {{hosts.[0]}}/_nodes/stats/ingest
   # timeout:
 
 response:
   split:
     type: map
     target: body.nodes
     key_field: node_id
+    keep_parent: true
     split:
       type: map
-      target: body.ingest.pipelines
+      target: body.nodes.ingest.pipelines
       keep_parent: true
       key_field: pipeline_id
 
 processors:
-- decode_json_fields:
-    fields: ["message"]
-- drop_event:
-    when:
-      and:
-        - equals:
-            message.ingest.pipelines.count: 0
-        - equals:
-            message.ingest.pipelines.failed: 0
-        - equals:
-            message.ingest.pipelines.time_in_millis: 0
+  - decode_json_fields:
+      fields:
+        - message
+  - drop_event:
+      when:
+        and:
+          - equals:
+              message.nodes.ingest.pipelines.count: 0
+          - equals:
+              message.nodes.ingest.pipelines.failed: 0
+          - equals:
+              message.nodes.ingest.pipelines.time_in_millis: 0
diff --git a/packages/elasticsearch/data_stream/ingest/elasticsearch/ingest_pipeline/default.yml b/packages/elasticsearch/data_stream/ingest/elasticsearch/ingest_pipeline/default.yml
@@ -2,28 +2,33 @@
 description: Pipeline for parsing pipeline ingest stats
 processors:
   - set:
-      ignore_empty_value: true
-      field: "event.original"
-      copy_from: "message"
-  - set:
-      field: "elasticsearch.ingest.pipeline.name"
-      copy_from: "message.ingest.pipelines.pipeline_id"
+      field: "elasticsearch.cluster.name"
+      copy_from: "message.cluster_name"
   - set:
       field: "elasticsearch.node.id"
-      copy_from: "message.node_id"
+      copy_from: "message.nodes.node_id"
+  - set:
+      field: "elasticsearch.node.name"
+      copy_from: "message.nodes.name"
+  - set:
+      field: "elasticsearch.node.roles"
+      copy_from: "message.nodes.roles"
+  - set:
+      field: "elasticsearch.ingest.pipeline.name"
+      copy_from: "message.nodes.ingest.pipelines.pipeline_id"
   - set:
       field: "elasticsearch.ingest.pipeline.total.count"
-      copy_from: "message.ingest.pipelines.count"
+      copy_from: "message.nodes.ingest.pipelines.count"
   - set:
       field: "elasticsearch.ingest.pipeline.total.failed"
-      copy_from: "message.ingest.pipelines.failed"
+      copy_from: "message.nodes.ingest.pipelines.failed"
   - set:
       field: "elasticsearch.ingest.pipeline.total.total_cpu_time"
-      copy_from: "message.ingest.pipelines.time_in_millis"
+      copy_from: "message.nodes.ingest.pipelines.time_in_millis"
   - script:
       lang: painless
       source: |
-        Map pipeline = ctx['message']['ingest']['pipelines'];
+        Map pipeline = ctx['message']['nodes']['ingest']['pipelines'];
         long self_cpu_time = pipeline['time_in_millis'];
         long count_processors = 0;
         def processors = new ArrayList();

diff --git a/packages/elasticsearch/data_stream/ingest/fields/fields.yml b/packages/elasticsearch/data_stream/ingest/fields/fields.yml
@@ -1,9 +1,3 @@
-- name: elasticsearch.node
-  type: group
-  fields:
-    - name: id
-      type: keyword
-
 - name: elasticsearch.ingest.pipeline
   type: group
   fields:

diff --git a/packages/elasticsearch/data_stream/ingest/fields/package-fields.yml b/packages/elasticsearch/data_stream/ingest/fields/package-fields.yml
@@ -0,0 +1,22 @@
+- name: elasticsearch
+  type: group
+  fields:
+    - name: cluster.name
+      type: keyword
+      description: |
+        Elasticsearch cluster name
+    - name: node
+      type: group
+      fields:
+        - name: id
+          type: keyword
+          description: |
+            Node ID
+        - name: name
+          type: keyword
+          description: |
+            Node name
+        - name: roles
+          type: keyword
+          description: |
+            Node roles