Add ingest_pipeline data streams and dashboards to Elasticsearch package

packages/elasticsearch/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.3.0-preview1"
+  changes:
+    - description: Add ingest pipeline monitoring metricset and dashboard
+      type: enhancement
+      link: https://github.com/elastic/integrations/issues/5141
 - version: "1.2.0"
   changes:
     - description: Add period variable to define polling frequency

packages/elasticsearch/data_stream/ingest_pipeline/_dev/test/system/test-default-config.yml

@@ -0,0 +1,8 @@
+type: httpjson
+dataset: elasticsearch.stack_monitoring.ingest
+vars:
+  hosts:
+    - "https://{{Hostname}}:9200"
+  username: elastic
+  password: changeme
+data_stream: ~

packages/elasticsearch/data_stream/ingest_pipeline/agent/stream/stream.yml.hbs

@@ -0,0 +1,15 @@
+metricsets: ["ingest_pipeline"]
+hosts:
+{{#each hosts}}
+  - {{this}}
+{{/each}}
+scope: {{scope}}
+{{#if username}}
+username: {{username}}
+{{/if}}
+{{#if password}}
+password: {{password}}
+{{/if}}
+period: {{period}}
+
+ingest_pipeline.processor_sample_rate: {{ingest_pipeline_processor_sampling_rate}}

packages/elasticsearch/data_stream/ingest_pipeline/fields/base-fields.yml

@@ -0,0 +1,9 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.

packages/elasticsearch/data_stream/ingest_pipeline/fields/ecs.yml

@@ -0,0 +1,21 @@
+- name: '@timestamp'
+  external: ecs
+- name: ecs.version
+  external: ecs
+- name: event.dataset
+  external: ecs
+- name: event.duration
+  external: ecs
+- name: event.module
+  external: ecs
+- name: host.name
+  external: ecs
+- name: service.address
+  type: keyword
+  description: Service address
+- name: service.type
+  external: ecs
+- name: service.name
+  external: ecs
+- name: error.message
+  external: ecs

packages/elasticsearch/data_stream/ingest_pipeline/fields/fields.yml

@@ -0,0 +1,45 @@
+- name: elasticsearch.ingest_pipeline
+  type: group
+  release: beta
+  description: Runtime metrics on ingest pipeline execution
+  fields:
+    - name: name
+      type: wildcard
+      description: Name / id of the ingest pipeline
+    - name: total
+      type: group
+      description: Metrics on the total ingest pipeline execution, including all processors.
+      fields:
+        - name: count
+          type: long
+          description: Number of documents processed by this pipeline
+        - name: failed
+          type: long
+          description: Number of documented failed to process by this pipeline
+        - name: time.total.ms
+          type: long
+          description: Total time spent processing documents through this pipeline, inclusive of other pipelines called
+        - name: time.self.ms
+          type: long
+          description: Time spent processing documents through this pipeline, exclusive of other pipelines called
+    - name: processor
+      type: group
+      fields:
+        - name: type
+          type: keyword
+          description: The type of ingest processor
+        - name: type_tag
+          type: keyword
+          description: The type and the tag for this processor in the format "<type>:<tag>"
+        - name: order_index
+          type: long
+          description: The order this processor appears in the pipeline definition
+        - name: count
+          type: long
+          description: Number of documents processed by this processor
+        - name: failed
+          type: long
+          description: Number of documented failed to process by this processor
+        - name: time.total.ms
+          type: long
+          description: Total time spent processing documents through this processor

packages/elasticsearch/data_stream/ingest_pipeline/fields/package-fields.yml

@@ -0,0 +1,26 @@
+- name: elasticsearch
+  type: group
+  fields:
+    - name: cluster.name
+      type: keyword
+      description: |
+        Elasticsearch cluster name
+    - name: cluster.id
+      type: keyword
+      description: |
+        Elasticsearch cluster id
+    - name: node
+      type: group
+      fields:
+        - name: id
+          type: keyword
+          description: |
+            Node ID
+        - name: name
+          type: keyword
+          description: |
+            Node name
+        - name: roles
+          type: keyword
+          description: |
+            Node roles

packages/elasticsearch/data_stream/ingest_pipeline/manifest.yml

@@ -0,0 +1,20 @@
+type: metrics
+title: Elasticsearch ingest metrics
+release: experimental
+dataset: elasticsearch.ingest_pipeline
+elasticsearch:
+  index_template:
+    mappings:
+      dynamic: false
+streams:
+  - input: elasticsearch/metrics
+    title: Ingest Pipeline metrics
+    description: Collect metrics on Ingest Pipelines
+    vars:
+      - name: ingest_pipeline_processor_sampling_rate
+        type: text
+        title: Processor metrics sampling rate
+        description: How often to collect the processor-level metrics. Number between 0 and 1.
+        required: true
+        show_user: true
+        default: "0.25"