Add ingest_pipeline data streams and dashboards to Elasticsearch pack…

…age (#4597)

packages/elasticsearch/_dev/deploy/docker/scripts/generate-logs.sh

@@ -111,7 +111,43 @@ curl --request POST \
   --header 'X-Opaque-ID: myApp1' \
   --header 'traceparent: 00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01'
 
+## Ingest pipeline monitoring
 
+## Create a pipeline
+curl --request PUT \
+  --url $ES_SERVICE_HOST/_ingest/pipeline/test-pipeline \
+  --header "Authorization: Basic $auth" \
+  --header 'Content-Type: application/json' \
+  --header 'X-Opaque-ID: myApp1' \
+  --header 'traceparent: 00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01' \
+  --data '{
+  "processors" : [
+    {
+      "set" : {
+        "field": "my-keyword-field",
+        "value": "foo"
+      }
+    }
+  ]
+}'
+
+## Create an index that uses the ingest pipeline
+curl --request PUT \
+  --url $ES_SERVICE_HOST/test_ip \
+  --header "Authorization: Basic $auth" \
+  --header 'Content-Type: application/json' \
+  --header 'X-Opaque-ID: myApp1' \
+  --header 'traceparent: 00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01'
+
+curl --request PUT \
+  --url $ES_SERVICE_HOST/test_ip/_settings \
+  --header "Authorization: Basic $auth" \
+  --header 'Content-Type: application/json' \
+  --header 'X-Opaque-ID: myApp1' \
+  --header 'traceparent: 00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01' \
+  --data '{"settings": {
+	"index.default_pipeline": "test-pipeline"
+}}'
 
 while true
 do
@@ -208,5 +244,22 @@ do
 
   copy_log_files
 
+  echo Generating ingest pipeline load
+  curl --request POST \
+    --url $ES_SERVICE_HOST/test_ip/_bulk \
+    --header "Authorization: Basic $auth" \
+    --header 'Content-Type: application/json' \
+    --header 'X-Opaque-ID: myApp1' \
+    --header 'traceparent: 00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01' \
+    --data '{ "create": {} }
+{ "a":1 }
+{ "create": {} }
+{ "a":2 }
+{ "create": {} }
+{ "a":3 }
+{ "create": {} }
+{ "a":4 }
+'
+
   sleep 10
 done

packages/elasticsearch/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.4.0-beta1"
+  changes:
+    - description: Add ingest pipeline monitoring dataset and dashboard (experimental)
+      type: enhancement
+      link: https://github.com/elastic/integrations/issues/5141
 - version: "1.3.0"
   changes:
     - description: Add conditional support to Elasticsearch log and metrics inputs

packages/elasticsearch/data_stream/ingest_pipeline/_dev/test/system/test-default-config.yml

@@ -0,0 +1,8 @@
+type: elasticsearch/metrics
+dataset: elasticsearch.ingest_pipeline
+vars:
+  hosts:
+    - "http://{{Hostname}}:9200"
+  username: elastic
+  password: changeme
+data_stream: ~

packages/elasticsearch/data_stream/ingest_pipeline/agent/stream/stream.yml.hbs

@@ -0,0 +1,15 @@
+metricsets: ["ingest_pipeline"]
+hosts:
+{{#each hosts}}
+  - {{this}}
+{{/each}}
+scope: {{scope}}
+{{#if username}}
+username: {{username}}
+{{/if}}
+{{#if password}}
+password: {{password}}
+{{/if}}
+period: {{period}}
+
+ingest_pipeline.processor_sample_rate: {{ingest_pipeline_processor_sampling_rate}}

packages/elasticsearch/data_stream/ingest_pipeline/fields/base-fields.yml

@@ -0,0 +1,9 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.

packages/elasticsearch/data_stream/ingest_pipeline/fields/ecs.yml

@@ -0,0 +1,21 @@
+- name: '@timestamp'
+  external: ecs
+- name: ecs.version
+  external: ecs
+- name: event.dataset
+  external: ecs
+- name: event.duration
+  external: ecs
+- name: event.module
+  external: ecs
+- name: host.name
+  external: ecs
+- name: service.address
+  type: keyword
+  description: Service address
+- name: service.type
+  external: ecs
+- name: service.name
+  external: ecs
+- name: error.message
+  external: ecs

packages/elasticsearch/data_stream/ingest_pipeline/fields/fields.yml

@@ -0,0 +1,45 @@
+- name: elasticsearch.ingest_pipeline
+  type: group
+  release: beta
+  description: Runtime metrics on ingest pipeline execution
+  fields:
+    - name: name
+      type: wildcard
+      description: Name / id of the ingest pipeline
+    - name: total
+      type: group
+      description: Metrics on the total ingest pipeline execution, including all processors.
+      fields:
+        - name: count
+          type: long
+          description: Number of documents processed by this pipeline
+        - name: failed
+          type: long
+          description: Number of documented failed to process by this pipeline
+        - name: time.total.ms
+          type: long
+          description: Total time spent processing documents through this pipeline, inclusive of other pipelines called
+        - name: time.self.ms
+          type: long
+          description: Time spent processing documents through this pipeline, exclusive of other pipelines called
+    - name: processor
+      type: group
+      fields:
+        - name: type
+          type: keyword
+          description: The type of ingest processor
+        - name: type_tag
+          type: keyword
+          description: The type and the tag for this processor in the format "<type>:<tag>"
+        - name: order_index
+          type: long
+          description: The order this processor appears in the pipeline definition
+        - name: count
+          type: long
+          description: Number of documents processed by this processor
+        - name: failed
+          type: long
+          description: Number of documented failed to process by this processor
+        - name: time.total.ms
+          type: long
+          description: Total time spent processing documents through this processor

packages/elasticsearch/data_stream/ingest_pipeline/fields/package-fields.yml

@@ -0,0 +1,26 @@
+- name: elasticsearch
+  type: group
+  fields:
+    - name: cluster.name
+      type: keyword
+      description: |
+        Elasticsearch cluster name
+    - name: cluster.id
+      type: keyword
+      description: |
+        Elasticsearch cluster id
+    - name: node
+      type: group
+      fields:
+        - name: id
+          type: keyword
+          description: |
+            Node ID
+        - name: name
+          type: keyword
+          description: |
+            Node name
+        - name: roles
+          type: keyword
+          description: |
+            Node roles

packages/elasticsearch/data_stream/ingest_pipeline/manifest.yml

@@ -0,0 +1,21 @@
+type: metrics
+title: Elasticsearch ingest metrics
+# These metrics should stay at least beta until the metricset in Metricbeat is GA'd
+release: experimental
+dataset: elasticsearch.ingest_pipeline
+elasticsearch:
+  index_template:
+    mappings:
+      dynamic: false
+streams:
+  - input: elasticsearch/metrics
+    title: Ingest Pipeline metrics
+    description: Collect metrics on Ingest Pipelines
+    vars:
+      - name: ingest_pipeline_processor_sampling_rate
+        type: text
+        title: Processor metrics sampling rate
+        description: How often to collect the processor-level metrics. Number between 0 and 1.
+        required: true
+        show_user: true
+        default: "0.25"