From efac2f2bbbf915715d2a3791461b263e13c31a6b Mon Sep 17 00:00:00 2001 From: Josh Dover <1813008+joshdover@users.noreply.github.com> Date: Fri, 24 Feb 2023 11:31:02 +0000 Subject: [PATCH] Add ingest_pipeline data streams and dashboards to Elasticsearch package (#4597) --- .../deploy/docker/scripts/generate-logs.sh | 53 + packages/elasticsearch/changelog.yml | 5 + .../_dev/test/system/test-default-config.yml | 8 + .../agent/stream/stream.yml.hbs | 15 + .../ingest_pipeline/fields/base-fields.yml | 9 + .../ingest_pipeline/fields/ecs.yml | 21 + .../ingest_pipeline/fields/fields.yml | 45 + .../ingest_pipeline/fields/package-fields.yml | 26 + .../data_stream/ingest_pipeline/manifest.yml | 21 + ...-ea5b81a0-7fbf-11ed-8509-ddabeb9daeaf.json | 4236 +++++++++++++++++ ...lasticsearch-metrics-ingest-pipelines.json | 2710 +++++++++++ .../elasticsearch-sm-metrics.json | 22 + packages/elasticsearch/manifest.yml | 4 +- 13 files changed, 7173 insertions(+), 2 deletions(-) create mode 100644 packages/elasticsearch/data_stream/ingest_pipeline/_dev/test/system/test-default-config.yml create mode 100644 packages/elasticsearch/data_stream/ingest_pipeline/agent/stream/stream.yml.hbs create mode 100644 packages/elasticsearch/data_stream/ingest_pipeline/fields/base-fields.yml create mode 100644 packages/elasticsearch/data_stream/ingest_pipeline/fields/ecs.yml create mode 100644 packages/elasticsearch/data_stream/ingest_pipeline/fields/fields.yml create mode 100644 packages/elasticsearch/data_stream/ingest_pipeline/fields/package-fields.yml create mode 100644 packages/elasticsearch/data_stream/ingest_pipeline/manifest.yml create mode 100644 packages/elasticsearch/kibana/dashboard/elasticsearch-ea5b81a0-7fbf-11ed-8509-ddabeb9daeaf.json create mode 100644 packages/elasticsearch/kibana/dashboard/elasticsearch-metrics-ingest-pipelines.json create mode 100644 packages/elasticsearch/kibana/index_pattern/elasticsearch-sm-metrics.json diff --git a/packages/elasticsearch/_dev/deploy/docker/scripts/generate-logs.sh b/packages/elasticsearch/_dev/deploy/docker/scripts/generate-logs.sh index 7a6e3b8ec2a..75c4522d66b 100755 --- a/packages/elasticsearch/_dev/deploy/docker/scripts/generate-logs.sh +++ b/packages/elasticsearch/_dev/deploy/docker/scripts/generate-logs.sh @@ -111,7 +111,43 @@ curl --request POST \ --header 'X-Opaque-ID: myApp1' \ --header 'traceparent: 00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01' +## Ingest pipeline monitoring +## Create a pipeline +curl --request PUT \ + --url $ES_SERVICE_HOST/_ingest/pipeline/test-pipeline \ + --header "Authorization: Basic $auth" \ + --header 'Content-Type: application/json' \ + --header 'X-Opaque-ID: myApp1' \ + --header 'traceparent: 00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01' \ + --data '{ + "processors" : [ + { + "set" : { + "field": "my-keyword-field", + "value": "foo" + } + } + ] +}' + +## Create an index that uses the ingest pipeline +curl --request PUT \ + --url $ES_SERVICE_HOST/test_ip \ + --header "Authorization: Basic $auth" \ + --header 'Content-Type: application/json' \ + --header 'X-Opaque-ID: myApp1' \ + --header 'traceparent: 00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01' + +curl --request PUT \ + --url $ES_SERVICE_HOST/test_ip/_settings \ + --header "Authorization: Basic $auth" \ + --header 'Content-Type: application/json' \ + --header 'X-Opaque-ID: myApp1' \ + --header 'traceparent: 00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01' \ + --data '{"settings": { + "index.default_pipeline": "test-pipeline" +}}' while true do @@ -208,5 +244,22 @@ do copy_log_files + echo Generating ingest pipeline load + curl --request POST \ + --url $ES_SERVICE_HOST/test_ip/_bulk \ + --header "Authorization: Basic $auth" \ + --header 'Content-Type: application/json' \ + --header 'X-Opaque-ID: myApp1' \ + --header 'traceparent: 00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01' \ + --data '{ "create": {} } +{ "a":1 } +{ "create": {} } +{ "a":2 } +{ "create": {} } +{ "a":3 } +{ "create": {} } +{ "a":4 } +' + sleep 10 done diff --git a/packages/elasticsearch/changelog.yml b/packages/elasticsearch/changelog.yml index 57e6a6ccc23..35c57351600 100644 --- a/packages/elasticsearch/changelog.yml +++ b/packages/elasticsearch/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.4.0-beta1" + changes: + - description: Add ingest pipeline monitoring dataset and dashboard (experimental) + type: enhancement + link: https://github.com/elastic/integrations/issues/5141 - version: "1.3.0" changes: - description: Add conditional support to Elasticsearch log and metrics inputs diff --git a/packages/elasticsearch/data_stream/ingest_pipeline/_dev/test/system/test-default-config.yml b/packages/elasticsearch/data_stream/ingest_pipeline/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..393e054d962 --- /dev/null +++ b/packages/elasticsearch/data_stream/ingest_pipeline/_dev/test/system/test-default-config.yml @@ -0,0 +1,8 @@ +type: elasticsearch/metrics +dataset: elasticsearch.ingest_pipeline +vars: + hosts: + - "http://{{Hostname}}:9200" + username: elastic + password: changeme +data_stream: ~ diff --git a/packages/elasticsearch/data_stream/ingest_pipeline/agent/stream/stream.yml.hbs b/packages/elasticsearch/data_stream/ingest_pipeline/agent/stream/stream.yml.hbs new file mode 100644 index 00000000000..822705f1693 --- /dev/null +++ b/packages/elasticsearch/data_stream/ingest_pipeline/agent/stream/stream.yml.hbs @@ -0,0 +1,15 @@ +metricsets: ["ingest_pipeline"] +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +scope: {{scope}} +{{#if username}} +username: {{username}} +{{/if}} +{{#if password}} +password: {{password}} +{{/if}} +period: {{period}} + +ingest_pipeline.processor_sample_rate: {{ingest_pipeline_processor_sampling_rate}} \ No newline at end of file diff --git a/packages/elasticsearch/data_stream/ingest_pipeline/fields/base-fields.yml b/packages/elasticsearch/data_stream/ingest_pipeline/fields/base-fields.yml new file mode 100644 index 00000000000..a3e80e3a547 --- /dev/null +++ b/packages/elasticsearch/data_stream/ingest_pipeline/fields/base-fields.yml @@ -0,0 +1,9 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. diff --git a/packages/elasticsearch/data_stream/ingest_pipeline/fields/ecs.yml b/packages/elasticsearch/data_stream/ingest_pipeline/fields/ecs.yml new file mode 100644 index 00000000000..5d852e3c52d --- /dev/null +++ b/packages/elasticsearch/data_stream/ingest_pipeline/fields/ecs.yml @@ -0,0 +1,21 @@ +- name: '@timestamp' + external: ecs +- name: ecs.version + external: ecs +- name: event.dataset + external: ecs +- name: event.duration + external: ecs +- name: event.module + external: ecs +- name: host.name + external: ecs +- name: service.address + type: keyword + description: Service address +- name: service.type + external: ecs +- name: service.name + external: ecs +- name: error.message + external: ecs diff --git a/packages/elasticsearch/data_stream/ingest_pipeline/fields/fields.yml b/packages/elasticsearch/data_stream/ingest_pipeline/fields/fields.yml new file mode 100644 index 00000000000..336fc53ce06 --- /dev/null +++ b/packages/elasticsearch/data_stream/ingest_pipeline/fields/fields.yml @@ -0,0 +1,45 @@ +- name: elasticsearch.ingest_pipeline + type: group + release: beta + description: Runtime metrics on ingest pipeline execution + fields: + - name: name + type: wildcard + description: Name / id of the ingest pipeline + - name: total + type: group + description: Metrics on the total ingest pipeline execution, including all processors. + fields: + - name: count + type: long + description: Number of documents processed by this pipeline + - name: failed + type: long + description: Number of documented failed to process by this pipeline + - name: time.total.ms + type: long + description: Total time spent processing documents through this pipeline, inclusive of other pipelines called + - name: time.self.ms + type: long + description: Time spent processing documents through this pipeline, exclusive of other pipelines called + - name: processor + type: group + fields: + - name: type + type: keyword + description: The type of ingest processor + - name: type_tag + type: keyword + description: The type and the tag for this processor in the format ":" + - name: order_index + type: long + description: The order this processor appears in the pipeline definition + - name: count + type: long + description: Number of documents processed by this processor + - name: failed + type: long + description: Number of documented failed to process by this processor + - name: time.total.ms + type: long + description: Total time spent processing documents through this processor diff --git a/packages/elasticsearch/data_stream/ingest_pipeline/fields/package-fields.yml b/packages/elasticsearch/data_stream/ingest_pipeline/fields/package-fields.yml new file mode 100644 index 00000000000..04a501eff11 --- /dev/null +++ b/packages/elasticsearch/data_stream/ingest_pipeline/fields/package-fields.yml @@ -0,0 +1,26 @@ +- name: elasticsearch + type: group + fields: + - name: cluster.name + type: keyword + description: | + Elasticsearch cluster name + - name: cluster.id + type: keyword + description: | + Elasticsearch cluster id + - name: node + type: group + fields: + - name: id + type: keyword + description: | + Node ID + - name: name + type: keyword + description: | + Node name + - name: roles + type: keyword + description: | + Node roles diff --git a/packages/elasticsearch/data_stream/ingest_pipeline/manifest.yml b/packages/elasticsearch/data_stream/ingest_pipeline/manifest.yml new file mode 100644 index 00000000000..a44f0a2ab33 --- /dev/null +++ b/packages/elasticsearch/data_stream/ingest_pipeline/manifest.yml @@ -0,0 +1,21 @@ +type: metrics +title: Elasticsearch ingest metrics +# These metrics should stay at least beta until the metricset in Metricbeat is GA'd +release: experimental +dataset: elasticsearch.ingest_pipeline +elasticsearch: + index_template: + mappings: + dynamic: false +streams: + - input: elasticsearch/metrics + title: Ingest Pipeline metrics + description: Collect metrics on Ingest Pipelines + vars: + - name: ingest_pipeline_processor_sampling_rate + type: text + title: Processor metrics sampling rate + description: How often to collect the processor-level metrics. Number between 0 and 1. + required: true + show_user: true + default: "0.25" \ No newline at end of file diff --git a/packages/elasticsearch/kibana/dashboard/elasticsearch-ea5b81a0-7fbf-11ed-8509-ddabeb9daeaf.json b/packages/elasticsearch/kibana/dashboard/elasticsearch-ea5b81a0-7fbf-11ed-8509-ddabeb9daeaf.json new file mode 100644 index 00000000000..848bd31f390 --- /dev/null +++ b/packages/elasticsearch/kibana/dashboard/elasticsearch-ea5b81a0-7fbf-11ed-8509-ddabeb9daeaf.json @@ -0,0 +1,4236 @@ +{ + "attributes": { + "controlGroupInput": { + "chainingSystem": "HIERARCHICAL", + "controlStyle": "oneLine", + "ignoreParentSettingsJSON": "{\"ignoreFilters\":false,\"ignoreQuery\":false,\"ignoreTimerange\":false,\"ignoreValidations\":false}", + "panelsJSON": "{\"d71679da-0013-430a-982a-7aa466e582f8\":{\"order\":2,\"width\":\"small\",\"grow\":true,\"type\":\"optionsListControl\",\"explicitInput\":{\"fieldName\":\"elasticsearch.node.name\",\"title\":\"Node\",\"id\":\"d71679da-0013-430a-982a-7aa466e582f8\",\"enhancements\":{},\"selectedOptions\":[]}},\"01070ea3-1a87-4fcc-927d-18b02827b891\":{\"order\":3,\"width\":\"large\",\"grow\":true,\"type\":\"optionsListControl\",\"explicitInput\":{\"fieldName\":\"elasticsearch.ingest_pipeline.name\",\"title\":\"Pipeline\",\"id\":\"01070ea3-1a87-4fcc-927d-18b02827b891\",\"enhancements\":{},\"selectedOptions\":[]}},\"de9376f3-00fd-46cb-9b75-3909c5d17a37\":{\"order\":0,\"width\":\"small\",\"grow\":true,\"type\":\"optionsListControl\",\"explicitInput\":{\"fieldName\":\"elasticsearch.cluster.name\",\"title\":\"Cluster\",\"id\":\"de9376f3-00fd-46cb-9b75-3909c5d17a37\",\"enhancements\":{}}},\"b89ded5a-50c5-47b3-a935-ee12f9f8aaf7\":{\"order\":1,\"width\":\"small\",\"grow\":true,\"type\":\"optionsListControl\",\"explicitInput\":{\"fieldName\":\"elasticsearch.node.roles\",\"title\":\"Node Roles\",\"id\":\"b89ded5a-50c5-47b3-a935-ee12f9f8aaf7\",\"selectedOptions\":[],\"enhancements\":{}}}}" + }, + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "syncTooltips": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "enhancements": {}, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "params": { + "fontSize": 12, + "markdown": "# Overview\n\n\u003e Analyze trends in throughput and performance over time for this pipeline", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 6, + "i": "64faf306-3732-4cd2-8d17-a4fa03ca3e75", + "w": 48, + "x": 0, + "y": 0 + }, + "panelIndex": "64faf306-3732-4cd2-8d17-a4fa03ca3e75", + "title": "", + "type": "visualization", + "version": "8.5.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "elasticsearch-sm-metrics", + "name": "indexpattern-datasource-layer-3faa4240-11c3-43c8-980f-0dda1d9d3c08", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "indexpattern": { + "layers": { + "3faa4240-11c3-43c8-980f-0dda1d9d3c08": { + "columnOrder": [ + "7ee48652-d095-4cb7-af8a-c779631d4b8c" + ], + "columns": { + "7ee48652-d095-4cb7-af8a-c779631d4b8c": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Nodes running this pipeline", + "operationType": "unique_count", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "elasticsearch.node.id" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "service.type:elasticsearch or data_stream.dataset:\"elasticsearch.ingest_pipeline\" " + }, + "visualization": { + "accessor": "7ee48652-d095-4cb7-af8a-c779631d4b8c", + "layerId": "3faa4240-11c3-43c8-980f-0dda1d9d3c08", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsLegacyMetric" + }, + "enhancements": {} + }, + "gridData": { + "h": 14, + "i": "46467ea8-7ba6-43d1-b61a-d3b6e23e6742", + "w": 6, + "x": 0, + "y": 6 + }, + "panelIndex": "46467ea8-7ba6-43d1-b61a-d3b6e23e6742", + "title": "", + "type": "lens", + "version": "8.5.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "elasticsearch-sm-metrics", + "name": "indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "indexpattern": { + "layers": { + "da36c906-5865-4c1f-874d-b021fef6d425": { + "columnOrder": [ + "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61", + "d470f360-5ac9-450f-bef2-8c666d25cee7", + "d66446a1-b145-40ba-8d88-5771c524ab37", + "d66446a1-b145-40ba-8d88-5771c524ab37X1", + "d66446a1-b145-40ba-8d88-5771c524ab37X0", + "34e5f179-78e1-4975-8f3f-f1f9327a2c42", + "34e5f179-78e1-4975-8f3f-f1f9327a2c42X1", + "34e5f179-78e1-4975-8f3f-f1f9327a2c42X0" + ], + "columns": { + "34e5f179-78e1-4975-8f3f-f1f9327a2c42": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Failures", + "operationType": "formula", + "params": { + "formula": "counter_rate(max(elasticsearch.ingest_pipeline.total.failed))", + "isFormulaBroken": false + }, + "references": [ + "34e5f179-78e1-4975-8f3f-f1f9327a2c42X1" + ], + "scale": "ratio", + "timeScale": "s" + }, + "34e5f179-78e1-4975-8f3f-f1f9327a2c42X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Documents processed", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.failed" + }, + "34e5f179-78e1-4975-8f3f-f1f9327a2c42X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Documents processed", + "operationType": "counter_rate", + "references": [ + "34e5f179-78e1-4975-8f3f-f1f9327a2c42X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "d470f360-5ac9-450f-bef2-8c666d25cee7": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": true, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Processed", + "operationType": "formula", + "params": { + "formula": "counter_rate(max(elasticsearch.ingest_pipeline.total.count))", + "isFormulaBroken": false + }, + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X1" + ], + "scale": "ratio", + "timeScale": "s" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Documents processed", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Documents processed", + "operationType": "counter_rate", + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of elasticsearch.ingest_pipeline.name + 1 other", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "isBucketed": false, + "label": "Maximum of elasticsearch.ingest_pipeline.total.count", + "operationType": "max", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "multi_terms" + }, + "secondaryFields": [ + "elasticsearch.node.id" + ], + "size": 20 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.ingest_pipeline.name" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "service.type:elasticsearch or data_stream.dataset:\"elasticsearch.ingest_pipeline\" " + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "curveType": "CURVE_MONOTONE_X", + "emphasizeFitting": true, + "endValue": "None", + "fillOpacity": 0.5, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "d66446a1-b145-40ba-8d88-5771c524ab37", + "34e5f179-78e1-4975-8f3f-f1f9327a2c42" + ], + "collapseFn": "sum", + "layerId": "da36c906-5865-4c1f-874d-b021fef6d425", + "layerType": "data", + "seriesType": "area", + "splitAccessor": "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61", + "xAccessor": "d470f360-5ac9-450f-bef2-8c666d25cee7", + "yConfig": [ + { + "axisMode": "left", + "color": "#54b399", + "forAccessor": "d66446a1-b145-40ba-8d88-5771c524ab37" + }, + { + "axisMode": "left", + "color": "#d36086", + "forAccessor": "34e5f179-78e1-4975-8f3f-f1f9327a2c42" + } + ] + } + ], + "legend": { + "isVisible": false, + "maxLines": 1, + "position": "top", + "shouldTruncate": true, + "showSingleSeries": false + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "yLeftExtent": { + "mode": "full" + }, + "yLeftScale": "linear", + "yRightExtent": { + "mode": "full" + }, + "yRightScale": "linear", + "yTitle": "Documents" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 14, + "i": "62553d02-5a7d-400b-979e-36da6aef012e", + "w": 42, + "x": 6, + "y": 6 + }, + "panelIndex": "62553d02-5a7d-400b-979e-36da6aef012e", + "title": "Documents and Failures", + "type": "lens", + "version": "8.5.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "elasticsearch-sm-metrics", + "name": "indexpattern-datasource-layer-3faa4240-11c3-43c8-980f-0dda1d9d3c08", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "indexpattern": { + "layers": { + "3faa4240-11c3-43c8-980f-0dda1d9d3c08": { + "columnOrder": [ + "7ee48652-d095-4cb7-af8a-c779631d4b8c" + ], + "columns": { + "7ee48652-d095-4cb7-af8a-c779631d4b8c": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Processors in this pipeline", + "operationType": "unique_count", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.processor.order_index" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "service.type:elasticsearch or data_stream.dataset:\"elasticsearch.ingest_pipeline\" " + }, + "visualization": { + "accessor": "7ee48652-d095-4cb7-af8a-c779631d4b8c", + "layerId": "3faa4240-11c3-43c8-980f-0dda1d9d3c08", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsLegacyMetric" + }, + "enhancements": {} + }, + "gridData": { + "h": 14, + "i": "5a9aaee6-8fc7-44f5-9319-e20c8a9dd467", + "w": 6, + "x": 0, + "y": 20 + }, + "panelIndex": "5a9aaee6-8fc7-44f5-9319-e20c8a9dd467", + "title": "", + "type": "lens", + "version": "8.5.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "elasticsearch-sm-metrics", + "name": "indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "indexpattern": { + "layers": { + "da36c906-5865-4c1f-874d-b021fef6d425": { + "columnOrder": [ + "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61", + "d470f360-5ac9-450f-bef2-8c666d25cee7", + "d66446a1-b145-40ba-8d88-5771c524ab37", + "d66446a1-b145-40ba-8d88-5771c524ab37X1", + "d66446a1-b145-40ba-8d88-5771c524ab37X0", + "d66446a1-b145-40ba-8d88-5771c524ab37X6", + "d66446a1-b145-40ba-8d88-5771c524ab37X5", + "d66446a1-b145-40ba-8d88-5771c524ab37X4", + "d66446a1-b145-40ba-8d88-5771c524ab37X3", + "d66446a1-b145-40ba-8d88-5771c524ab37X2" + ], + "columns": { + "d470f360-5ac9-450f-bef2-8c666d25cee7": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": true, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Avg ms per doc", + "operationType": "formula", + "params": { + "formula": "cumulative_sum(counter_rate(max(elasticsearch.ingest_pipeline.total.time.self.ms)))/cumulative_sum(counter_rate((max(elasticsearch.ingest_pipeline.total.count))))", + "isFormulaBroken": false + }, + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X6" + ], + "scale": "ratio" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Avg time per doc", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.time.self.ms" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Avg time per doc", + "operationType": "counter_rate", + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X2": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Avg time per doc", + "operationType": "cumulative_sum", + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X1" + ], + "scale": "ratio" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X3": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Avg time per doc", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X4": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Avg time per doc", + "operationType": "counter_rate", + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X3" + ], + "scale": "ratio", + "timeScale": "s" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X5": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Avg time per doc", + "operationType": "cumulative_sum", + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X4" + ], + "scale": "ratio" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X6": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Avg time per doc", + "operationType": "math", + "params": { + "tinymathAst": { + "args": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X2", + "d66446a1-b145-40ba-8d88-5771c524ab37X5" + ], + "location": { + "max": 163, + "min": 0 + }, + "name": "divide", + "text": "cumulative_sum(counter_rate(max(elasticsearch.ingest_pipeline.total.time.self.ms)))/cumulative_sum(counter_rate((max(elasticsearch.ingest_pipeline.total.count))))", + "type": "function" + } + }, + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X2", + "d66446a1-b145-40ba-8d88-5771c524ab37X5" + ], + "scale": "ratio" + }, + "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of elasticsearch.ingest_pipeline.name + 1 other", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "isBucketed": false, + "label": "Maximum of elasticsearch.ingest_pipeline.total.count", + "operationType": "max", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "multi_terms" + }, + "secondaryFields": [ + "elasticsearch.node.id" + ], + "size": 20 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.ingest_pipeline.name" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "service.type:elasticsearch or data_stream.dataset:\"elasticsearch.ingest_pipeline\" " + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "curveType": "CURVE_MONOTONE_X", + "emphasizeFitting": true, + "endValue": "None", + "fillOpacity": 0.5, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "d66446a1-b145-40ba-8d88-5771c524ab37" + ], + "collapseFn": "avg", + "layerId": "da36c906-5865-4c1f-874d-b021fef6d425", + "layerType": "data", + "seriesType": "area", + "splitAccessor": "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61", + "xAccessor": "d470f360-5ac9-450f-bef2-8c666d25cee7", + "yConfig": [ + { + "axisMode": "left", + "color": "#54b399", + "forAccessor": "d66446a1-b145-40ba-8d88-5771c524ab37" + } + ] + } + ], + "legend": { + "isVisible": false, + "maxLines": 1, + "position": "top", + "shouldTruncate": true, + "showSingleSeries": false + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "yLeftExtent": { + "mode": "full" + }, + "yLeftScale": "linear", + "yRightExtent": { + "mode": "full" + }, + "yRightScale": "linear", + "yTitle": "" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 14, + "i": "1ba77622-28e8-4462-b806-576d073ae308", + "w": 42, + "x": 6, + "y": 20 + }, + "panelIndex": "1ba77622-28e8-4462-b806-576d073ae308", + "title": "time per document", + "type": "lens", + "version": "8.5.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "elasticsearch-sm-metrics", + "name": "indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "indexpattern": { + "layers": { + "da36c906-5865-4c1f-874d-b021fef6d425": { + "columnOrder": [ + "4a999eff-ba72-4344-9077-db8f9d27cf45", + "0bc394bd-4e71-4611-a300-e8d87d37efa6", + "f7c5cd36-1bc1-4d66-93cc-04f54459a66a", + "d66446a1-b145-40ba-8d88-5771c524ab37", + "f36a06ec-bdba-46d2-90ca-9c7de830c269", + "f36a06ec-bdba-46d2-90ca-9c7de830c269X1", + "f36a06ec-bdba-46d2-90ca-9c7de830c269X0", + "d66446a1-b145-40ba-8d88-5771c524ab37X1", + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "columns": { + "0bc394bd-4e71-4611-a300-e8d87d37efa6": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Pipeline", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderAgg": { + "customLabel": false, + "dataType": "number", + "isBucketed": false, + "label": "Count of records", + "operationType": "count", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "___records___" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": false, + "parentFormat": { + "id": "terms" + }, + "size": 500 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.ingest_pipeline.name" + }, + "4a999eff-ba72-4344-9077-db8f9d27cf45": { + "dataType": "string", + "isBucketed": true, + "label": "Top 500 values of elasticsearch.node.id", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "isBucketed": false, + "label": "Sum of elasticsearch.ingest_pipeline.total.count", + "operationType": "sum", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 500 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.node.id" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Documents", + "operationType": "formula", + "params": { + "formula": "counter_rate(max(elasticsearch.ingest_pipeline.total.count))", + "isFormulaBroken": false + }, + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X1" + ], + "scale": "ratio" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Documents", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Documents", + "operationType": "counter_rate", + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "f36a06ec-bdba-46d2-90ca-9c7de830c269": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Total time spent (s)", + "operationType": "formula", + "params": { + "formula": "counter_rate(max(elasticsearch.ingest_pipeline.total.time.self.ms))", + "isFormulaBroken": false + }, + "references": [ + "f36a06ec-bdba-46d2-90ca-9c7de830c269X1" + ], + "scale": "ratio" + }, + "f36a06ec-bdba-46d2-90ca-9c7de830c269X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Total time spent (s)", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.time.self.ms" + }, + "f36a06ec-bdba-46d2-90ca-9c7de830c269X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Total time spent (s)", + "operationType": "counter_rate", + "references": [ + "f36a06ec-bdba-46d2-90ca-9c7de830c269X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "f7c5cd36-1bc1-4d66-93cc-04f54459a66a": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "service.type:elasticsearch or data_stream.dataset:\"elasticsearch.ingest_pipeline\" " + }, + "visualization": { + "columns": [ + { + "collapseFn": "sum", + "columnId": "4a999eff-ba72-4344-9077-db8f9d27cf45", + "isTransposed": false + }, + { + "columnId": "d66446a1-b145-40ba-8d88-5771c524ab37", + "summaryRow": "none" + }, + { + "collapseFn": "sum", + "columnId": "f7c5cd36-1bc1-4d66-93cc-04f54459a66a", + "isTransposed": false + }, + { + "columnId": "f36a06ec-bdba-46d2-90ca-9c7de830c269", + "isTransposed": false + }, + { + "collapseFn": "", + "columnId": "0bc394bd-4e71-4611-a300-e8d87d37efa6", + "hidden": true, + "isTransposed": false + } + ], + "layerId": "da36c906-5865-4c1f-874d-b021fef6d425", + "layerType": "data", + "sorting": { + "columnId": "d66446a1-b145-40ba-8d88-5771c524ab37", + "direction": "desc" + } + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": { + "dynamicActions": { + "events": [ + { + "action": { + "config": { + "encodeUrl": true, + "openInNewTab": false, + "url": { + "template": "{{kibanaUrl}}/app/management/ingest/ingest_pipelines/edit/{{event.values.[0]}}" + } + }, + "factoryId": "URL_DRILLDOWN", + "name": "Edit pipeline" + }, + "eventId": "b8f207fb-daa8-4b39-b94b-e9dab4cdb4ad", + "triggers": [ + "ROW_CLICK_TRIGGER" + ] + } + ] + } + }, + "hidePanelTitles": true + }, + "gridData": { + "h": 6, + "i": "f959aa3e-3644-4f1a-a597-c3a22b9b7e0d", + "w": 23, + "x": 0, + "y": 34 + }, + "panelIndex": "f959aa3e-3644-4f1a-a597-c3a22b9b7e0d", + "title": "Pipelines processing most documents", + "type": "lens", + "version": "8.5.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "elasticsearch-sm-metrics", + "name": "indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "indexpattern": { + "layers": { + "da36c906-5865-4c1f-874d-b021fef6d425": { + "columnOrder": [ + "4a999eff-ba72-4344-9077-db8f9d27cf45", + "0bc394bd-4e71-4611-a300-e8d87d37efa6", + "f7c5cd36-1bc1-4d66-93cc-04f54459a66a", + "f36a06ec-bdba-46d2-90ca-9c7de830c269", + "f36a06ec-bdba-46d2-90ca-9c7de830c269X1", + "f36a06ec-bdba-46d2-90ca-9c7de830c269X0", + "f36a06ec-bdba-46d2-90ca-9c7de830c269X4", + "f36a06ec-bdba-46d2-90ca-9c7de830c269X3", + "f36a06ec-bdba-46d2-90ca-9c7de830c269X2", + "f36a06ec-bdba-46d2-90ca-9c7de830c269X5", + "f36a06ec-bdba-46d2-90ca-9c7de830c269X6" + ], + "columns": { + "0bc394bd-4e71-4611-a300-e8d87d37efa6": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Pipeline", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "isBucketed": false, + "label": "95th percentile of elasticsearch.ingest_pipeline.total.time.self.ms", + "operationType": "percentile", + "params": { + "percentile": 95 + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.time.self.ms" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": false, + "parentFormat": { + "id": "terms" + }, + "size": 500 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.ingest_pipeline.name" + }, + "4a999eff-ba72-4344-9077-db8f9d27cf45": { + "dataType": "string", + "isBucketed": true, + "label": "Top 500 values of elasticsearch.node.id", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "isBucketed": false, + "label": "Sum of elasticsearch.ingest_pipeline.total.count", + "operationType": "sum", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 500 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.node.id" + }, + "f36a06ec-bdba-46d2-90ca-9c7de830c269": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Avg time per doc", + "operationType": "formula", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 1, + "suffix": " ms" + } + }, + "formula": "cumulative_sum(counter_rate(max(elasticsearch.ingest_pipeline.total.time.self.ms)))/cumulative_sum(counter_rate((max(elasticsearch.ingest_pipeline.total.count))))", + "isFormulaBroken": false + }, + "references": [ + "f36a06ec-bdba-46d2-90ca-9c7de830c269X6" + ], + "scale": "ratio" + }, + "f36a06ec-bdba-46d2-90ca-9c7de830c269X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Avg time per doc", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.time.self.ms" + }, + "f36a06ec-bdba-46d2-90ca-9c7de830c269X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Avg time per doc", + "operationType": "counter_rate", + "references": [ + "f36a06ec-bdba-46d2-90ca-9c7de830c269X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "f36a06ec-bdba-46d2-90ca-9c7de830c269X2": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Avg time per doc", + "operationType": "cumulative_sum", + "references": [ + "f36a06ec-bdba-46d2-90ca-9c7de830c269X1" + ], + "scale": "ratio" + }, + "f36a06ec-bdba-46d2-90ca-9c7de830c269X3": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Avg time per doc", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + }, + "f36a06ec-bdba-46d2-90ca-9c7de830c269X4": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Avg time per doc", + "operationType": "counter_rate", + "references": [ + "f36a06ec-bdba-46d2-90ca-9c7de830c269X3" + ], + "scale": "ratio", + "timeScale": "s" + }, + "f36a06ec-bdba-46d2-90ca-9c7de830c269X5": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Avg time per doc", + "operationType": "cumulative_sum", + "references": [ + "f36a06ec-bdba-46d2-90ca-9c7de830c269X4" + ], + "scale": "ratio" + }, + "f36a06ec-bdba-46d2-90ca-9c7de830c269X6": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Avg time per doc", + "operationType": "math", + "params": { + "tinymathAst": { + "args": [ + "f36a06ec-bdba-46d2-90ca-9c7de830c269X2", + "f36a06ec-bdba-46d2-90ca-9c7de830c269X5" + ], + "location": { + "max": 163, + "min": 0 + }, + "name": "divide", + "text": "cumulative_sum(counter_rate(max(elasticsearch.ingest_pipeline.total.time.self.ms)))/cumulative_sum(counter_rate((max(elasticsearch.ingest_pipeline.total.count))))", + "type": "function" + } + }, + "references": [ + "f36a06ec-bdba-46d2-90ca-9c7de830c269X2", + "f36a06ec-bdba-46d2-90ca-9c7de830c269X5" + ], + "scale": "ratio" + }, + "f7c5cd36-1bc1-4d66-93cc-04f54459a66a": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "service.type:elasticsearch or data_stream.dataset:\"elasticsearch.ingest_pipeline\" " + }, + "visualization": { + "columns": [ + { + "collapseFn": "avg", + "columnId": "4a999eff-ba72-4344-9077-db8f9d27cf45", + "isTransposed": false + }, + { + "collapseFn": "sum", + "columnId": "f7c5cd36-1bc1-4d66-93cc-04f54459a66a", + "isTransposed": false + }, + { + "columnId": "f36a06ec-bdba-46d2-90ca-9c7de830c269", + "isTransposed": false + }, + { + "collapseFn": "", + "columnId": "0bc394bd-4e71-4611-a300-e8d87d37efa6", + "hidden": true, + "isTransposed": false + } + ], + "layerId": "da36c906-5865-4c1f-874d-b021fef6d425", + "layerType": "data", + "sorting": { + "columnId": "f36a06ec-bdba-46d2-90ca-9c7de830c269", + "direction": "desc" + } + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": { + "dynamicActions": { + "events": [ + { + "action": { + "config": { + "encodeUrl": true, + "openInNewTab": false, + "url": { + "template": "{{kibanaUrl}}/app/management/ingest/ingest_pipelines/edit/{{event.values.[0]}}" + } + }, + "factoryId": "URL_DRILLDOWN", + "name": "Edit pipeline" + }, + "eventId": "0990a8a0-42de-4cf9-b90a-82a51c5a27ea", + "triggers": [ + "ROW_CLICK_TRIGGER" + ] + } + ] + } + }, + "hidePanelTitles": true + }, + "gridData": { + "h": 6, + "i": "46db7a44-91d5-430a-b63b-9ccc90ebf512", + "w": 13, + "x": 23, + "y": 34 + }, + "panelIndex": "46db7a44-91d5-430a-b63b-9ccc90ebf512", + "title": "Slowest pipelines per document", + "type": "lens", + "version": "8.5.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "elasticsearch-sm-metrics", + "name": "indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "indexpattern": { + "layers": { + "da36c906-5865-4c1f-874d-b021fef6d425": { + "columnOrder": [ + "4a999eff-ba72-4344-9077-db8f9d27cf45", + "0bc394bd-4e71-4611-a300-e8d87d37efa6", + "f7c5cd36-1bc1-4d66-93cc-04f54459a66a", + "d66446a1-b145-40ba-8d88-5771c524ab37", + "d66446a1-b145-40ba-8d88-5771c524ab37X1", + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "columns": { + "0bc394bd-4e71-4611-a300-e8d87d37efa6": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Pipeline", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "isBucketed": false, + "label": "95th percentile of elasticsearch.ingest_pipeline.total.failed", + "operationType": "percentile", + "params": { + "percentile": 95 + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.failed" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 500 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.ingest_pipeline.name" + }, + "4a999eff-ba72-4344-9077-db8f9d27cf45": { + "dataType": "string", + "isBucketed": true, + "label": "Top 500 values of elasticsearch.node.id", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderAgg": { + "customLabel": false, + "dataType": "number", + "isBucketed": false, + "label": "Count of elasticsearch.ingest_pipeline.total.count", + "operationType": "count", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 500 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.node.id" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Failed documents", + "operationType": "formula", + "params": { + "formula": "counter_rate(max(elasticsearch.ingest_pipeline.total.failed))", + "isFormulaBroken": false + }, + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X1" + ], + "scale": "ratio" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Failed documents", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.failed" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Failed documents", + "operationType": "counter_rate", + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "f7c5cd36-1bc1-4d66-93cc-04f54459a66a": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "service.type:elasticsearch or data_stream.dataset:\"elasticsearch.ingest_pipeline\" " + }, + "visualization": { + "columns": [ + { + "collapseFn": "sum", + "columnId": "4a999eff-ba72-4344-9077-db8f9d27cf45", + "isTransposed": false + }, + { + "columnId": "d66446a1-b145-40ba-8d88-5771c524ab37", + "summaryRow": "none" + }, + { + "collapseFn": "sum", + "columnId": "f7c5cd36-1bc1-4d66-93cc-04f54459a66a", + "isTransposed": false + }, + { + "collapseFn": "", + "columnId": "0bc394bd-4e71-4611-a300-e8d87d37efa6", + "hidden": true, + "isTransposed": false + } + ], + "layerId": "da36c906-5865-4c1f-874d-b021fef6d425", + "layerType": "data", + "sorting": { + "columnId": "d66446a1-b145-40ba-8d88-5771c524ab37", + "direction": "desc" + } + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": { + "dynamicActions": { + "events": [ + { + "action": { + "config": { + "encodeUrl": true, + "openInNewTab": false, + "url": { + "template": "{{kibanaUrl}}/app/management/ingest/ingest_pipelines/edit/{{event.values.[0]}}" + } + }, + "factoryId": "URL_DRILLDOWN", + "name": "Edit pipeline" + }, + "eventId": "c2c6dab8-d750-4521-bbcc-e1b9c19455ba", + "triggers": [ + "ROW_CLICK_TRIGGER" + ] + } + ] + } + }, + "hidePanelTitles": true + }, + "gridData": { + "h": 6, + "i": "2d2c07ad-db4f-46e2-a984-4b14ce17ac84", + "w": 12, + "x": 36, + "y": 34 + }, + "panelIndex": "2d2c07ad-db4f-46e2-a984-4b14ce17ac84", + "title": "Top failing pipelines", + "type": "lens", + "version": "8.5.1" + }, + { + "embeddableConfig": { + "enhancements": {}, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "params": { + "fontSize": 12, + "markdown": "# Processors\n\n\u003e Find slow processors that may be impacting your ingestion performance or dropping documents", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 6, + "i": "08adf87f-7edd-427c-b4bb-054e53451584", + "w": 48, + "x": 0, + "y": 40 + }, + "panelIndex": "08adf87f-7edd-427c-b4bb-054e53451584", + "title": "", + "type": "visualization", + "version": "8.5.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "elasticsearch-sm-metrics", + "name": "indexpattern-datasource-layer-f16c2d3a-7f5d-491f-929a-d95c8a9ff91b", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "indexpattern": { + "layers": { + "f16c2d3a-7f5d-491f-929a-d95c8a9ff91b": { + "columnOrder": [ + "43487448-486e-4979-a681-9a49232e4a70", + "cc8105ca-0654-4846-b397-560b4375ff12", + "2a52a690-e138-4010-b111-539c435188b6", + "a4ef3a4f-4ba8-43e3-9638-3063fb7e5f0c", + "24db20d4-7483-450a-bda0-472270231a2c", + "8521fba6-caf4-4d6f-b0d9-4b9104988a4c", + "43aef93d-5091-48d0-aa45-08acbcee47b6", + "764ecc8d-65a4-4800-bfdb-3deca6ad2d0c", + "e8e9fd7d-5a03-4e78-9f5a-43e148ee2417" + ], + "columns": { + "24db20d4-7483-450a-bda0-472270231a2c": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Failures", + "operationType": "counter_rate", + "references": [ + "764ecc8d-65a4-4800-bfdb-3deca6ad2d0c" + ], + "scale": "ratio" + }, + "2a52a690-e138-4010-b111-539c435188b6": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Processor", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "fallback": true, + "type": "alphabetical" + }, + "orderDirection": "asc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "secondaryFields": [], + "size": 100 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.ingest_pipeline.processor.type_tag" + }, + "43487448-486e-4979-a681-9a49232e4a70": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "43aef93d-5091-48d0-aa45-08acbcee47b6": { + "dataType": "number", + "isBucketed": false, + "label": "Maximum of elasticsearch.ingest_pipeline.processor.count", + "operationType": "max", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.processor.count" + }, + "764ecc8d-65a4-4800-bfdb-3deca6ad2d0c": { + "dataType": "number", + "isBucketed": false, + "label": "Maximum of elasticsearch.ingest_pipeline.processor.failed", + "operationType": "max", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.processor.failed" + }, + "8521fba6-caf4-4d6f-b0d9-4b9104988a4c": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "time", + "operationType": "counter_rate", + "references": [ + "e8e9fd7d-5a03-4e78-9f5a-43e148ee2417" + ], + "scale": "ratio" + }, + "a4ef3a4f-4ba8-43e3-9638-3063fb7e5f0c": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Documents", + "operationType": "counter_rate", + "references": [ + "43aef93d-5091-48d0-aa45-08acbcee47b6" + ], + "scale": "ratio" + }, + "cc8105ca-0654-4846-b397-560b4375ff12": { + "customLabel": true, + "dataType": "number", + "isBucketed": true, + "label": "Order", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "fallback": true, + "type": "alphabetical" + }, + "orderDirection": "asc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 100 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.ingest_pipeline.processor.order_index" + }, + "e8e9fd7d-5a03-4e78-9f5a-43e148ee2417": { + "dataType": "number", + "isBucketed": false, + "label": "Maximum of elasticsearch.ingest_pipeline.processor.time.total.ms", + "operationType": "max", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.processor.time.total.ms" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "columns": [ + { + "columnId": "2a52a690-e138-4010-b111-539c435188b6", + "isTransposed": false + }, + { + "columnId": "cc8105ca-0654-4846-b397-560b4375ff12", + "isTransposed": false + }, + { + "colorMode": "cell", + "columnId": "8521fba6-caf4-4d6f-b0d9-4b9104988a4c", + "isTransposed": false, + "palette": { + "name": "negative", + "params": { + "continuity": "above", + "name": "negative", + "rangeMax": null, + "rangeMin": 0, + "reverse": false, + "stops": [ + { + "color": "#fbddd6", + "stop": 0 + }, + { + "color": "#f3bbaf", + "stop": 20 + }, + { + "color": "#e99a89", + "stop": 40 + }, + { + "color": "#db7965", + "stop": 60 + }, + { + "color": "#cc5642", + "stop": 80 + } + ] + }, + "type": "palette" + } + }, + { + "colorMode": "cell", + "columnId": "a4ef3a4f-4ba8-43e3-9638-3063fb7e5f0c", + "isTransposed": false, + "palette": { + "name": "cool", + "params": { + "continuity": "above", + "name": "cool", + "rangeMax": null, + "rangeMin": 0, + "reverse": false, + "stops": [ + { + "color": "#e0e8f2", + "stop": 0 + }, + { + "color": "#c2d2e6", + "stop": 20 + }, + { + "color": "#a2bcd9", + "stop": 40 + }, + { + "color": "#82a7cd", + "stop": 60 + }, + { + "color": "#6092c0", + "stop": 80 + } + ] + }, + "type": "palette" + } + }, + { + "colorMode": "cell", + "columnId": "24db20d4-7483-450a-bda0-472270231a2c", + "isTransposed": false, + "palette": { + "name": "negative", + "params": { + "continuity": "above", + "name": "negative", + "rangeMax": null, + "rangeMin": 0, + "reverse": false, + "stops": [ + { + "color": "#fbddd6", + "stop": 0 + }, + { + "color": "#f3bbaf", + "stop": 20 + }, + { + "color": "#e99a89", + "stop": 40 + }, + { + "color": "#db7965", + "stop": 60 + }, + { + "color": "#cc5642", + "stop": 80 + } + ] + }, + "type": "palette" + } + }, + { + "collapseFn": "sum", + "columnId": "43487448-486e-4979-a681-9a49232e4a70", + "hidden": true, + "isTransposed": false + } + ], + "layerId": "f16c2d3a-7f5d-491f-929a-d95c8a9ff91b", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": {} + }, + "gridData": { + "h": 24, + "i": "8f07b0fb-cb88-4a92-af5d-676f767b1854", + "w": 24, + "x": 0, + "y": 46 + }, + "panelIndex": "8f07b0fb-cb88-4a92-af5d-676f767b1854", + "type": "lens", + "version": "8.5.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "elasticsearch-sm-metrics", + "name": "indexpattern-datasource-layer-f16c2d3a-7f5d-491f-929a-d95c8a9ff91b", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "indexpattern": { + "layers": { + "f16c2d3a-7f5d-491f-929a-d95c8a9ff91b": { + "columnOrder": [ + "43487448-486e-4979-a681-9a49232e4a70", + "cc8105ca-0654-4846-b397-560b4375ff12", + "2a52a690-e138-4010-b111-539c435188b6", + "8521fba6-caf4-4d6f-b0d9-4b9104988a4c", + "8521fba6-caf4-4d6f-b0d9-4b9104988a4cX6", + "8521fba6-caf4-4d6f-b0d9-4b9104988a4cX5", + "8521fba6-caf4-4d6f-b0d9-4b9104988a4cX4", + "8521fba6-caf4-4d6f-b0d9-4b9104988a4cX3", + "8521fba6-caf4-4d6f-b0d9-4b9104988a4cX2", + "8521fba6-caf4-4d6f-b0d9-4b9104988a4cX1", + "8521fba6-caf4-4d6f-b0d9-4b9104988a4cX0" + ], + "columns": { + "2a52a690-e138-4010-b111-539c435188b6": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Processor", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "fallback": true, + "type": "alphabetical" + }, + "orderDirection": "asc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "secondaryFields": [], + "size": 100 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.ingest_pipeline.processor.type_tag" + }, + "43487448-486e-4979-a681-9a49232e4a70": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "8521fba6-caf4-4d6f-b0d9-4b9104988a4c": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Avg time per doc", + "operationType": "formula", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2, + "suffix": " ms" + } + }, + "formula": "cumulative_sum(counter_rate(max(elasticsearch.ingest_pipeline.processor.time.total.ms)))/cumulative_sum(counter_rate((max(elasticsearch.ingest_pipeline.processor.count))))", + "isFormulaBroken": false + }, + "references": [ + "8521fba6-caf4-4d6f-b0d9-4b9104988a4cX6" + ], + "scale": "ratio" + }, + "8521fba6-caf4-4d6f-b0d9-4b9104988a4cX0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Avg time per doc", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.processor.time.total.ms" + }, + "8521fba6-caf4-4d6f-b0d9-4b9104988a4cX1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Avg time per doc", + "operationType": "counter_rate", + "references": [ + "8521fba6-caf4-4d6f-b0d9-4b9104988a4cX0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "8521fba6-caf4-4d6f-b0d9-4b9104988a4cX2": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Avg time per doc", + "operationType": "cumulative_sum", + "references": [ + "8521fba6-caf4-4d6f-b0d9-4b9104988a4cX1" + ], + "scale": "ratio" + }, + "8521fba6-caf4-4d6f-b0d9-4b9104988a4cX3": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Avg time per doc", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.processor.count" + }, + "8521fba6-caf4-4d6f-b0d9-4b9104988a4cX4": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Avg time per doc", + "operationType": "counter_rate", + "references": [ + "8521fba6-caf4-4d6f-b0d9-4b9104988a4cX3" + ], + "scale": "ratio", + "timeScale": "s" + }, + "8521fba6-caf4-4d6f-b0d9-4b9104988a4cX5": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Avg time per doc", + "operationType": "cumulative_sum", + "references": [ + "8521fba6-caf4-4d6f-b0d9-4b9104988a4cX4" + ], + "scale": "ratio" + }, + "8521fba6-caf4-4d6f-b0d9-4b9104988a4cX6": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Avg time per doc", + "operationType": "math", + "params": { + "tinymathAst": { + "args": [ + "8521fba6-caf4-4d6f-b0d9-4b9104988a4cX2", + "8521fba6-caf4-4d6f-b0d9-4b9104988a4cX5" + ], + "location": { + "max": 172, + "min": 0 + }, + "name": "divide", + "text": "cumulative_sum(counter_rate(max(elasticsearch.ingest_pipeline.processor.time.total.ms)))/cumulative_sum(counter_rate((max(elasticsearch.ingest_pipeline.processor.count))))", + "type": "function" + } + }, + "references": [ + "8521fba6-caf4-4d6f-b0d9-4b9104988a4cX2", + "8521fba6-caf4-4d6f-b0d9-4b9104988a4cX5" + ], + "scale": "ratio" + }, + "cc8105ca-0654-4846-b397-560b4375ff12": { + "customLabel": true, + "dataType": "number", + "isBucketed": true, + "label": "Order", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "fallback": true, + "type": "alphabetical" + }, + "orderDirection": "asc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 100 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.ingest_pipeline.processor.order_index" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "columns": [ + { + "columnId": "2a52a690-e138-4010-b111-539c435188b6", + "isTransposed": false + }, + { + "columnId": "cc8105ca-0654-4846-b397-560b4375ff12", + "isTransposed": false, + "width": 122.66666666666669 + }, + { + "colorMode": "cell", + "columnId": "8521fba6-caf4-4d6f-b0d9-4b9104988a4c", + "isTransposed": false, + "palette": { + "name": "negative", + "params": { + "continuity": "above", + "name": "negative", + "rangeMax": null, + "rangeMin": 0, + "reverse": false, + "stops": [ + { + "color": "#fbddd6", + "stop": 0 + }, + { + "color": "#f3bbaf", + "stop": 20 + }, + { + "color": "#e99a89", + "stop": 40 + }, + { + "color": "#db7965", + "stop": 60 + }, + { + "color": "#cc5642", + "stop": 80 + } + ] + }, + "type": "palette" + } + }, + { + "collapseFn": "avg", + "columnId": "43487448-486e-4979-a681-9a49232e4a70", + "hidden": true, + "isTransposed": false + } + ], + "layerId": "f16c2d3a-7f5d-491f-929a-d95c8a9ff91b", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": {} + }, + "gridData": { + "h": 24, + "i": "3c8bb8ab-2c2b-41a6-9b47-de8ddf56e6f9", + "w": 24, + "x": 24, + "y": 46 + }, + "panelIndex": "3c8bb8ab-2c2b-41a6-9b47-de8ddf56e6f9", + "title": "", + "type": "lens", + "version": "8.5.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "elasticsearch-sm-metrics", + "name": "indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "indexpattern": { + "layers": { + "da36c906-5865-4c1f-874d-b021fef6d425": { + "columnOrder": [ + "d470f360-5ac9-450f-bef2-8c666d25cee7", + "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61", + "d66446a1-b145-40ba-8d88-5771c524ab37", + "d66446a1-b145-40ba-8d88-5771c524ab37X6", + "d66446a1-b145-40ba-8d88-5771c524ab37X5", + "d66446a1-b145-40ba-8d88-5771c524ab37X4", + "d66446a1-b145-40ba-8d88-5771c524ab37X3", + "d66446a1-b145-40ba-8d88-5771c524ab37X2", + "d66446a1-b145-40ba-8d88-5771c524ab37X1", + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "columns": { + "d470f360-5ac9-450f-bef2-8c666d25cee7": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": true, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "time", + "operationType": "formula", + "params": { + "formula": "cumulative_sum(counter_rate(max(elasticsearch.ingest_pipeline.processor.time.total.ms)))/cumulative_sum(counter_rate((max(elasticsearch.ingest_pipeline.processor.count))))", + "isFormulaBroken": false + }, + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X6" + ], + "scale": "ratio", + "timeScale": "s" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of time", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.processor.time.total.ms" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of time", + "operationType": "counter_rate", + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X2": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of time", + "operationType": "cumulative_sum", + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X1" + ], + "scale": "ratio" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X3": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of time", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.processor.count" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X4": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of time", + "operationType": "counter_rate", + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X3" + ], + "scale": "ratio", + "timeScale": "s" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X5": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of time", + "operationType": "cumulative_sum", + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X4" + ], + "scale": "ratio" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X6": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of time", + "operationType": "math", + "params": { + "tinymathAst": { + "args": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X2", + "d66446a1-b145-40ba-8d88-5771c524ab37X5" + ], + "location": { + "max": 172, + "min": 0 + }, + "name": "divide", + "text": "cumulative_sum(counter_rate(max(elasticsearch.ingest_pipeline.processor.time.total.ms)))/cumulative_sum(counter_rate((max(elasticsearch.ingest_pipeline.processor.count))))", + "type": "function" + } + }, + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X2", + "d66446a1-b145-40ba-8d88-5771c524ab37X5" + ], + "scale": "ratio" + }, + "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of elasticsearch.ingest_pipeline.processor.order_index + 2 others", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "isBucketed": false, + "label": "Maximum of elasticsearch.ingest_pipeline.total.count", + "operationType": "max", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": false, + "parentFormat": { + "id": "multi_terms" + }, + "secondaryFields": [ + "elasticsearch.node.name", + "elasticsearch.ingest_pipeline.processor.type_tag" + ], + "size": 100 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.ingest_pipeline.processor.order_index" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "service.type:elasticsearch or data_stream.dataset:\"elasticsearch.ingest_pipeline\" " + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "curveType": "CURVE_MONOTONE_X", + "fillOpacity": 0.5, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "d66446a1-b145-40ba-8d88-5771c524ab37" + ], + "collapseFn": "", + "layerId": "da36c906-5865-4c1f-874d-b021fef6d425", + "layerType": "data", + "seriesType": "line", + "splitAccessor": "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61", + "xAccessor": "d470f360-5ac9-450f-bef2-8c666d25cee7", + "yConfig": [ + { + "axisMode": "left", + "color": "#54b399", + "forAccessor": "d66446a1-b145-40ba-8d88-5771c524ab37" + } + ] + } + ], + "legend": { + "isVisible": true, + "maxLines": 1, + "position": "right", + "shouldTruncate": true, + "showSingleSeries": false + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "valuesInLegend": true, + "yLeftExtent": { + "mode": "full" + }, + "yLeftScale": "linear", + "yRightExtent": { + "mode": "full" + }, + "yRightScale": "linear" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 18, + "i": "d8c5f51b-a49c-4846-b372-5c258d507acb", + "w": 48, + "x": 0, + "y": 70 + }, + "panelIndex": "d8c5f51b-a49c-4846-b372-5c258d507acb", + "title": "Avg time per doc by processor", + "type": "lens", + "version": "8.5.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "elasticsearch-sm-metrics", + "name": "indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "indexpattern": { + "layers": { + "da36c906-5865-4c1f-874d-b021fef6d425": { + "columnOrder": [ + "d470f360-5ac9-450f-bef2-8c666d25cee7", + "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61", + "d66446a1-b145-40ba-8d88-5771c524ab37", + "d66446a1-b145-40ba-8d88-5771c524ab37X1", + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "columns": { + "d470f360-5ac9-450f-bef2-8c666d25cee7": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": true, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Documents", + "operationType": "formula", + "params": { + "formula": "counter_rate(max(elasticsearch.ingest_pipeline.processor.count))", + "isFormulaBroken": false + }, + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X1" + ], + "scale": "ratio", + "timeScale": "s" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Documents", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.processor.count" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Documents", + "operationType": "counter_rate", + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of elasticsearch.ingest_pipeline.processor.order_index + 2 others", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "isBucketed": false, + "label": "Maximum of elasticsearch.ingest_pipeline.total.count", + "operationType": "max", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": false, + "parentFormat": { + "id": "multi_terms" + }, + "secondaryFields": [ + "elasticsearch.ingest_pipeline.processor.type_tag", + "elasticsearch.node.name" + ], + "size": 100 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.ingest_pipeline.processor.order_index" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "service.type:elasticsearch or data_stream.dataset:\"elasticsearch.ingest_pipeline\" " + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "curveType": "CURVE_MONOTONE_X", + "fillOpacity": 0.5, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "d66446a1-b145-40ba-8d88-5771c524ab37" + ], + "collapseFn": "", + "layerId": "da36c906-5865-4c1f-874d-b021fef6d425", + "layerType": "data", + "seriesType": "line", + "splitAccessor": "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61", + "xAccessor": "d470f360-5ac9-450f-bef2-8c666d25cee7", + "yConfig": [ + { + "axisMode": "left", + "color": "#54b399", + "forAccessor": "d66446a1-b145-40ba-8d88-5771c524ab37" + } + ] + } + ], + "legend": { + "isVisible": true, + "maxLines": 1, + "position": "right", + "shouldTruncate": true, + "showSingleSeries": false + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "valuesInLegend": true, + "yLeftExtent": { + "mode": "full" + }, + "yLeftScale": "linear", + "yRightExtent": { + "mode": "full" + }, + "yRightScale": "linear" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 18, + "i": "d3fdc894-c21e-4ec4-954c-e1d9b45813e9", + "w": 48, + "x": 0, + "y": 88 + }, + "panelIndex": "d3fdc894-c21e-4ec4-954c-e1d9b45813e9", + "title": "EPS by processor", + "type": "lens", + "version": "8.5.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "elasticsearch-sm-metrics", + "name": "indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "indexpattern": { + "layers": { + "da36c906-5865-4c1f-874d-b021fef6d425": { + "columnOrder": [ + "d470f360-5ac9-450f-bef2-8c666d25cee7", + "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61", + "d66446a1-b145-40ba-8d88-5771c524ab37", + "d66446a1-b145-40ba-8d88-5771c524ab37X1", + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "columns": { + "d470f360-5ac9-450f-bef2-8c666d25cee7": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": true, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "time", + "operationType": "formula", + "params": { + "formula": "counter_rate(max(elasticsearch.ingest_pipeline.processor.time.total.ms))", + "isFormulaBroken": false + }, + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X1" + ], + "scale": "ratio", + "timeScale": "s" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Documents", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.processor.time.total.ms" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Documents", + "operationType": "counter_rate", + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of elasticsearch.ingest_pipeline.processor.order_index + 2 others", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "isBucketed": false, + "label": "Maximum of elasticsearch.ingest_pipeline.total.count", + "operationType": "max", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": false, + "parentFormat": { + "id": "multi_terms" + }, + "secondaryFields": [ + "elasticsearch.ingest_pipeline.processor.type_tag", + "elasticsearch.node.name" + ], + "size": 100 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.ingest_pipeline.processor.order_index" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "service.type:elasticsearch or data_stream.dataset:\"elasticsearch.ingest_pipeline\" " + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "curveType": "CURVE_MONOTONE_X", + "fillOpacity": 0.5, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "d66446a1-b145-40ba-8d88-5771c524ab37" + ], + "collapseFn": "", + "layerId": "da36c906-5865-4c1f-874d-b021fef6d425", + "layerType": "data", + "seriesType": "line", + "splitAccessor": "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61", + "xAccessor": "d470f360-5ac9-450f-bef2-8c666d25cee7", + "yConfig": [ + { + "axisMode": "left", + "color": "#54b399", + "forAccessor": "d66446a1-b145-40ba-8d88-5771c524ab37" + } + ] + } + ], + "legend": { + "isVisible": true, + "maxLines": 1, + "position": "right", + "shouldTruncate": true, + "showSingleSeries": false + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "valuesInLegend": true, + "yLeftExtent": { + "mode": "full" + }, + "yLeftScale": "linear", + "yRightExtent": { + "mode": "full" + }, + "yRightScale": "linear" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 18, + "i": "50a3d05a-d1b7-4eae-ba4c-da6c22f329e6", + "w": 48, + "x": 0, + "y": 106 + }, + "panelIndex": "50a3d05a-d1b7-4eae-ba4c-da6c22f329e6", + "title": "Time by Processor", + "type": "lens", + "version": "8.5.1" + }, + { + "embeddableConfig": { + "enhancements": {}, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "id": "", + "params": { + "fontSize": 12, + "markdown": "# Breakdowns by Elasticsearch node\n\n\u003e Find hotspots and poor performing Elasticsearch nodes that are impacting this pipeline's ingest performance", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 6, + "i": "32ce43ec-edb2-4e30-8fb3-df52e06c885c", + "w": 48, + "x": 0, + "y": 124 + }, + "panelIndex": "32ce43ec-edb2-4e30-8fb3-df52e06c885c", + "type": "visualization", + "version": "8.5.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "elasticsearch-sm-metrics", + "name": "indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "indexpattern": { + "layers": { + "da36c906-5865-4c1f-874d-b021fef6d425": { + "columnOrder": [ + "4a999eff-ba72-4344-9077-db8f9d27cf45", + "0bc394bd-4e71-4611-a300-e8d87d37efa6", + "f7c5cd36-1bc1-4d66-93cc-04f54459a66a", + "d66446a1-b145-40ba-8d88-5771c524ab37", + "6d9c299f-8ebb-4319-877c-4b21402d4b3f", + "f36a06ec-bdba-46d2-90ca-9c7de830c269", + "f36a06ec-bdba-46d2-90ca-9c7de830c269X1", + "f36a06ec-bdba-46d2-90ca-9c7de830c269X0", + "d66446a1-b145-40ba-8d88-5771c524ab37X1", + "d66446a1-b145-40ba-8d88-5771c524ab37X0", + "6d9c299f-8ebb-4319-877c-4b21402d4b3fX1", + "6d9c299f-8ebb-4319-877c-4b21402d4b3fX0" + ], + "columns": { + "0bc394bd-4e71-4611-a300-e8d87d37efa6": { + "dataType": "string", + "isBucketed": true, + "label": "Top 20 values of elasticsearch.ingest_pipeline.name", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderAgg": { + "customLabel": false, + "dataType": "number", + "isBucketed": false, + "label": "Count of records", + "operationType": "count", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "___records___" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 20 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.ingest_pipeline.name" + }, + "4a999eff-ba72-4344-9077-db8f9d27cf45": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Node", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "isBucketed": false, + "label": "Sum of elasticsearch.ingest_pipeline.total.count", + "operationType": "sum", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "secondaryFields": [], + "size": 20 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.node.name" + }, + "6d9c299f-8ebb-4319-877c-4b21402d4b3f": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Failures", + "operationType": "formula", + "params": { + "formula": "counter_rate(max(elasticsearch.ingest_pipeline.total.failed))", + "isFormulaBroken": false + }, + "references": [ + "6d9c299f-8ebb-4319-877c-4b21402d4b3fX1" + ], + "scale": "ratio" + }, + "6d9c299f-8ebb-4319-877c-4b21402d4b3fX0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Failures", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.failed" + }, + "6d9c299f-8ebb-4319-877c-4b21402d4b3fX1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Failures", + "operationType": "counter_rate", + "references": [ + "6d9c299f-8ebb-4319-877c-4b21402d4b3fX0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Documents", + "operationType": "formula", + "params": { + "formula": "counter_rate(max(elasticsearch.ingest_pipeline.total.count))", + "isFormulaBroken": false + }, + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X1" + ], + "scale": "ratio" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Documents", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Documents", + "operationType": "counter_rate", + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "f36a06ec-bdba-46d2-90ca-9c7de830c269": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Total time spent (s)", + "operationType": "formula", + "params": { + "formula": "counter_rate(max(elasticsearch.ingest_pipeline.total.time.self.ms))", + "isFormulaBroken": false + }, + "references": [ + "f36a06ec-bdba-46d2-90ca-9c7de830c269X1" + ], + "scale": "ratio" + }, + "f36a06ec-bdba-46d2-90ca-9c7de830c269X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Total time spent (s)", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.time.self.ms" + }, + "f36a06ec-bdba-46d2-90ca-9c7de830c269X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Total time spent (s)", + "operationType": "counter_rate", + "references": [ + "f36a06ec-bdba-46d2-90ca-9c7de830c269X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "f7c5cd36-1bc1-4d66-93cc-04f54459a66a": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "service.type:elasticsearch or data_stream.dataset:\"elasticsearch.ingest_pipeline\" " + }, + "visualization": { + "columns": [ + { + "collapseFn": "", + "columnId": "4a999eff-ba72-4344-9077-db8f9d27cf45", + "isTransposed": false + }, + { + "columnId": "d66446a1-b145-40ba-8d88-5771c524ab37", + "summaryRow": "none" + }, + { + "collapseFn": "sum", + "columnId": "f7c5cd36-1bc1-4d66-93cc-04f54459a66a", + "isTransposed": false + }, + { + "columnId": "f36a06ec-bdba-46d2-90ca-9c7de830c269", + "isTransposed": false + }, + { + "collapseFn": "sum", + "columnId": "0bc394bd-4e71-4611-a300-e8d87d37efa6", + "isTransposed": false + }, + { + "columnId": "6d9c299f-8ebb-4319-877c-4b21402d4b3f", + "isTransposed": false + } + ], + "layerId": "da36c906-5865-4c1f-874d-b021fef6d425", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": { + "dynamicActions": { + "events": [ + { + "action": { + "config": { + "encodeUrl": true, + "openInNewTab": false, + "url": { + "template": "{{kibanaUrl}}/app/monitoring#/elasticsearch/nodes/{{event.values.[0]}}\n" + } + }, + "factoryId": "URL_DRILLDOWN", + "name": "View node metrics" + }, + "eventId": "33193580-dcb3-45f8-9be5-7f0570d64227", + "triggers": [ + "ROW_CLICK_TRIGGER" + ] + } + ] + } + }, + "hidePanelTitles": false + }, + "gridData": { + "h": 18, + "i": "e4f9ecca-8f6f-43e9-bcec-c957c844305c", + "w": 48, + "x": 0, + "y": 130 + }, + "panelIndex": "e4f9ecca-8f6f-43e9-bcec-c957c844305c", + "title": "Summary by node", + "type": "lens", + "version": "8.5.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "elasticsearch-sm-metrics", + "name": "indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "indexpattern": { + "layers": { + "da36c906-5865-4c1f-874d-b021fef6d425": { + "columnOrder": [ + "d470f360-5ac9-450f-bef2-8c666d25cee7", + "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61", + "d66446a1-b145-40ba-8d88-5771c524ab37", + "d66446a1-b145-40ba-8d88-5771c524ab37X1", + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "columns": { + "d470f360-5ac9-450f-bef2-8c666d25cee7": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": true, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Time spent", + "operationType": "formula", + "params": { + "formula": "counter_rate(max(elasticsearch.ingest_pipeline.total.time.self.ms))", + "isFormulaBroken": false + }, + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X1" + ], + "scale": "ratio" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Time spent", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.time.self.ms" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Time spent", + "operationType": "counter_rate", + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61": { + "dataType": "string", + "isBucketed": true, + "label": "Top 100 values of elasticsearch.node.name", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "isBucketed": false, + "label": "Maximum of elasticsearch.ingest_pipeline.total.count", + "operationType": "max", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": false, + "parentFormat": { + "id": "terms" + }, + "secondaryFields": [], + "size": 100 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.node.name" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "service.type:elasticsearch or data_stream.dataset:\"elasticsearch.ingest_pipeline\" " + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "curveType": "CURVE_MONOTONE_X", + "fillOpacity": 0.5, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "d66446a1-b145-40ba-8d88-5771c524ab37" + ], + "collapseFn": "", + "layerId": "da36c906-5865-4c1f-874d-b021fef6d425", + "layerType": "data", + "seriesType": "area_stacked", + "splitAccessor": "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61", + "xAccessor": "d470f360-5ac9-450f-bef2-8c666d25cee7", + "yConfig": [ + { + "axisMode": "left", + "color": "#54b399", + "forAccessor": "d66446a1-b145-40ba-8d88-5771c524ab37" + } + ] + } + ], + "legend": { + "isVisible": true, + "maxLines": 1, + "position": "right", + "shouldTruncate": true, + "showSingleSeries": false + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "valuesInLegend": true, + "yLeftExtent": { + "mode": "full" + }, + "yLeftScale": "linear", + "yRightExtent": { + "mode": "full" + }, + "yRightScale": "linear" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 19, + "i": "0cc9444d-c323-4ceb-b43d-072caf360183", + "w": 24, + "x": 0, + "y": 148 + }, + "panelIndex": "0cc9444d-c323-4ceb-b43d-072caf360183", + "title": "Time spent by node", + "type": "lens", + "version": "8.5.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "elasticsearch-sm-metrics", + "name": "indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "indexpattern": { + "layers": { + "da36c906-5865-4c1f-874d-b021fef6d425": { + "columnOrder": [ + "d470f360-5ac9-450f-bef2-8c666d25cee7", + "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61", + "d66446a1-b145-40ba-8d88-5771c524ab37", + "d66446a1-b145-40ba-8d88-5771c524ab37X1", + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "columns": { + "d470f360-5ac9-450f-bef2-8c666d25cee7": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": true, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Processed", + "operationType": "formula", + "params": { + "formula": "counter_rate(max(elasticsearch.ingest_pipeline.total.count))", + "isFormulaBroken": false + }, + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X1" + ], + "scale": "ratio" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Time spent", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Time spent", + "operationType": "counter_rate", + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61": { + "dataType": "string", + "isBucketed": true, + "label": "Top 100 values of elasticsearch.node.name", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "isBucketed": false, + "label": "Maximum of elasticsearch.ingest_pipeline.total.count", + "operationType": "max", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": false, + "parentFormat": { + "id": "terms" + }, + "secondaryFields": [], + "size": 100 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.node.name" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "service.type:elasticsearch or data_stream.dataset:\"elasticsearch.ingest_pipeline\" " + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "curveType": "CURVE_MONOTONE_X", + "fillOpacity": 0.5, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "d66446a1-b145-40ba-8d88-5771c524ab37" + ], + "collapseFn": "", + "layerId": "da36c906-5865-4c1f-874d-b021fef6d425", + "layerType": "data", + "seriesType": "area_stacked", + "splitAccessor": "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61", + "xAccessor": "d470f360-5ac9-450f-bef2-8c666d25cee7", + "yConfig": [ + { + "axisMode": "left", + "color": "#54b399", + "forAccessor": "d66446a1-b145-40ba-8d88-5771c524ab37" + } + ] + } + ], + "legend": { + "isVisible": true, + "maxLines": 1, + "position": "right", + "shouldTruncate": true, + "showSingleSeries": false + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "valuesInLegend": true, + "yLeftExtent": { + "mode": "full" + }, + "yLeftScale": "linear", + "yRightExtent": { + "mode": "full" + }, + "yRightScale": "linear", + "yTitle": "Documents" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 19, + "i": "2bac6f1e-8565-49c3-8235-331c7cf45180", + "w": 24, + "x": 24, + "y": 148 + }, + "panelIndex": "2bac6f1e-8565-49c3-8235-331c7cf45180", + "title": "Documents by node", + "type": "lens", + "version": "8.5.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "elasticsearch-sm-metrics", + "name": "indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "indexpattern": { + "layers": { + "da36c906-5865-4c1f-874d-b021fef6d425": { + "columnOrder": [ + "d470f360-5ac9-450f-bef2-8c666d25cee7", + "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61", + "d66446a1-b145-40ba-8d88-5771c524ab37", + "d66446a1-b145-40ba-8d88-5771c524ab37X1", + "d66446a1-b145-40ba-8d88-5771c524ab37X0", + "d66446a1-b145-40ba-8d88-5771c524ab37X4", + "d66446a1-b145-40ba-8d88-5771c524ab37X3", + "d66446a1-b145-40ba-8d88-5771c524ab37X2" + ], + "columns": { + "d470f360-5ac9-450f-bef2-8c666d25cee7": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": true, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Time spent per document processed", + "operationType": "formula", + "params": { + "formula": "counter_rate(max(elasticsearch.ingest_pipeline.total.time.self.ms)) / counter_rate(max(elasticsearch.ingest_pipeline.total.count))", + "isFormulaBroken": false + }, + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X4" + ], + "scale": "ratio" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Time spent per document processed", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.time.self.ms" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Time spent per document processed", + "operationType": "counter_rate", + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X2": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Time spent per document processed", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X3": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Time spent per document processed", + "operationType": "counter_rate", + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X2" + ], + "scale": "ratio", + "timeScale": "s" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X4": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Time spent per document processed", + "operationType": "math", + "params": { + "tinymathAst": { + "args": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X1", + "d66446a1-b145-40ba-8d88-5771c524ab37X3" + ], + "location": { + "max": 131, + "min": 0 + }, + "name": "divide", + "text": "counter_rate(max(elasticsearch.ingest_pipeline.total.time.self.ms)) / counter_rate(max(elasticsearch.ingest_pipeline.total.count))", + "type": "function" + } + }, + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X1", + "d66446a1-b145-40ba-8d88-5771c524ab37X3" + ], + "scale": "ratio" + }, + "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of elasticsearch.ingest_pipeline.name + 1 other", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "isBucketed": false, + "label": "Maximum of elasticsearch.ingest_pipeline.total.count", + "operationType": "max", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "multi_terms" + }, + "secondaryFields": [ + "elasticsearch.node.name" + ], + "size": 8 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.ingest_pipeline.name" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "service.type:elasticsearch or data_stream.dataset:\"elasticsearch.ingest_pipeline\" " + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "curveType": "CURVE_MONOTONE_X", + "fillOpacity": 0.5, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "d66446a1-b145-40ba-8d88-5771c524ab37" + ], + "collapseFn": "", + "layerId": "da36c906-5865-4c1f-874d-b021fef6d425", + "layerType": "data", + "seriesType": "line", + "splitAccessor": "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61", + "xAccessor": "d470f360-5ac9-450f-bef2-8c666d25cee7", + "yConfig": [ + { + "axisMode": "left", + "color": "#54b399", + "forAccessor": "d66446a1-b145-40ba-8d88-5771c524ab37" + } + ] + } + ], + "legend": { + "isVisible": true, + "maxLines": 1, + "position": "right", + "shouldTruncate": true, + "showSingleSeries": false + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "valuesInLegend": true, + "yLeftExtent": { + "mode": "full" + }, + "yLeftScale": "linear", + "yRightExtent": { + "mode": "full" + }, + "yRightScale": "linear" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 18, + "i": "4fe12cde-9e47-49fa-b1db-bfe6f99790eb", + "w": 24, + "x": 0, + "y": 167 + }, + "panelIndex": "4fe12cde-9e47-49fa-b1db-bfe6f99790eb", + "title": "Time spent per document processed", + "type": "lens", + "version": "8.5.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "elasticsearch-sm-metrics", + "name": "indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "indexpattern": { + "layers": { + "da36c906-5865-4c1f-874d-b021fef6d425": { + "columnOrder": [ + "d470f360-5ac9-450f-bef2-8c666d25cee7", + "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61", + "d66446a1-b145-40ba-8d88-5771c524ab37", + "d66446a1-b145-40ba-8d88-5771c524ab37X1", + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "columns": { + "d470f360-5ac9-450f-bef2-8c666d25cee7": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": true, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Failed Documents", + "operationType": "formula", + "params": { + "formula": "counter_rate(max(elasticsearch.ingest_pipeline.total.failed))", + "isFormulaBroken": false + }, + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X1" + ], + "scale": "ratio", + "timeScale": "s" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Failed Documents", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.failed" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Failed Documents", + "operationType": "counter_rate", + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of elasticsearch.ingest_pipeline.name + 1 other", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "isBucketed": false, + "label": "Maximum of elasticsearch.ingest_pipeline.total.failed", + "operationType": "max", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.failed" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "multi_terms" + }, + "secondaryFields": [ + "elasticsearch.node.name" + ], + "size": 10 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.ingest_pipeline.name" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "service.type:elasticsearch or data_stream.dataset:\"elasticsearch.ingest_pipeline\" " + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "curveType": "CURVE_MONOTONE_X", + "fillOpacity": 0.5, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "d66446a1-b145-40ba-8d88-5771c524ab37" + ], + "collapseFn": "", + "layerId": "da36c906-5865-4c1f-874d-b021fef6d425", + "layerType": "data", + "seriesType": "line", + "splitAccessor": "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61", + "xAccessor": "d470f360-5ac9-450f-bef2-8c666d25cee7", + "yConfig": [ + { + "axisMode": "left", + "color": "#54b399", + "forAccessor": "d66446a1-b145-40ba-8d88-5771c524ab37" + } + ] + } + ], + "legend": { + "isVisible": true, + "maxLines": 1, + "position": "right", + "shouldTruncate": true, + "showSingleSeries": false + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "valuesInLegend": true, + "yLeftExtent": { + "mode": "full" + }, + "yLeftScale": "linear", + "yRightExtent": { + "mode": "full" + }, + "yRightScale": "linear" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 18, + "i": "da89ce7b-9065-4c56-affc-02eeb02723f0", + "w": 24, + "x": 24, + "y": 167 + }, + "panelIndex": "da89ce7b-9065-4c56-affc-02eeb02723f0", + "title": "Failures processed by top nodes processing top pipelines", + "type": "lens", + "version": "8.5.1" + } + ], + "timeRestore": false, + "title": "[Elasticsearch] Ingest Pipeline Detail", + "version": 1 + }, + "coreMigrationVersion": "8.5.1", + "id": "elasticsearch-ea5b81a0-7fbf-11ed-8509-ddabeb9daeaf", + "migrationVersion": { + "dashboard": "8.5.0" + }, + "references": [ + { + "id": "elasticsearch-sm-metrics", + "name": "46467ea8-7ba6-43d1-b61a-d3b6e23e6742:indexpattern-datasource-layer-3faa4240-11c3-43c8-980f-0dda1d9d3c08", + "type": "index-pattern" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "62553d02-5a7d-400b-979e-36da6aef012e:indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "5a9aaee6-8fc7-44f5-9319-e20c8a9dd467:indexpattern-datasource-layer-3faa4240-11c3-43c8-980f-0dda1d9d3c08", + "type": "index-pattern" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "1ba77622-28e8-4462-b806-576d073ae308:indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "f959aa3e-3644-4f1a-a597-c3a22b9b7e0d:indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "46db7a44-91d5-430a-b63b-9ccc90ebf512:indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "2d2c07ad-db4f-46e2-a984-4b14ce17ac84:indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "8f07b0fb-cb88-4a92-af5d-676f767b1854:indexpattern-datasource-layer-f16c2d3a-7f5d-491f-929a-d95c8a9ff91b", + "type": "index-pattern" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "3c8bb8ab-2c2b-41a6-9b47-de8ddf56e6f9:indexpattern-datasource-layer-f16c2d3a-7f5d-491f-929a-d95c8a9ff91b", + "type": "index-pattern" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "d8c5f51b-a49c-4846-b372-5c258d507acb:indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "d3fdc894-c21e-4ec4-954c-e1d9b45813e9:indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "50a3d05a-d1b7-4eae-ba4c-da6c22f329e6:indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "e4f9ecca-8f6f-43e9-bcec-c957c844305c:indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "0cc9444d-c323-4ceb-b43d-072caf360183:indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "2bac6f1e-8565-49c3-8235-331c7cf45180:indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "4fe12cde-9e47-49fa-b1db-bfe6f99790eb:indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "da89ce7b-9065-4c56-affc-02eeb02723f0:indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "controlGroup_d71679da-0013-430a-982a-7aa466e582f8:optionsListDataView", + "type": "index-pattern" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "controlGroup_01070ea3-1a87-4fcc-927d-18b02827b891:optionsListDataView", + "type": "index-pattern" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "controlGroup_de9376f3-00fd-46cb-9b75-3909c5d17a37:optionsListDataView", + "type": "index-pattern" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "controlGroup_b89ded5a-50c5-47b3-a935-ee12f9f8aaf7:optionsListDataView", + "type": "index-pattern" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/elasticsearch/kibana/dashboard/elasticsearch-metrics-ingest-pipelines.json b/packages/elasticsearch/kibana/dashboard/elasticsearch-metrics-ingest-pipelines.json new file mode 100644 index 00000000000..901be0a9f4f --- /dev/null +++ b/packages/elasticsearch/kibana/dashboard/elasticsearch-metrics-ingest-pipelines.json @@ -0,0 +1,2710 @@ +{ + "attributes": { + "controlGroupInput": { + "chainingSystem": "HIERARCHICAL", + "controlStyle": "oneLine", + "ignoreParentSettingsJSON": "{\"ignoreFilters\":false,\"ignoreQuery\":false,\"ignoreTimerange\":false,\"ignoreValidations\":false}", + "panelsJSON": "{\"d71679da-0013-430a-982a-7aa466e582f8\":{\"order\":2,\"width\":\"small\",\"grow\":true,\"type\":\"optionsListControl\",\"explicitInput\":{\"fieldName\":\"elasticsearch.node.name\",\"title\":\"Node\",\"id\":\"d71679da-0013-430a-982a-7aa466e582f8\",\"enhancements\":{},\"selectedOptions\":[]}},\"01070ea3-1a87-4fcc-927d-18b02827b891\":{\"order\":3,\"width\":\"large\",\"grow\":true,\"type\":\"optionsListControl\",\"explicitInput\":{\"fieldName\":\"elasticsearch.ingest_pipeline.name\",\"title\":\"Pipeline\",\"id\":\"01070ea3-1a87-4fcc-927d-18b02827b891\",\"enhancements\":{},\"selectedOptions\":[]}},\"de9376f3-00fd-46cb-9b75-3909c5d17a37\":{\"order\":0,\"width\":\"small\",\"grow\":true,\"type\":\"optionsListControl\",\"explicitInput\":{\"fieldName\":\"elasticsearch.cluster.name\",\"title\":\"Cluster\",\"id\":\"de9376f3-00fd-46cb-9b75-3909c5d17a37\",\"enhancements\":{}}},\"b89ded5a-50c5-47b3-a935-ee12f9f8aaf7\":{\"order\":1,\"width\":\"small\",\"grow\":true,\"type\":\"optionsListControl\",\"explicitInput\":{\"fieldName\":\"elasticsearch.node.roles\",\"title\":\"Node Roles\",\"id\":\"b89ded5a-50c5-47b3-a935-ee12f9f8aaf7\",\"selectedOptions\":[],\"enhancements\":{}}}}" + }, + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": "System pipelines", + "disabled": false, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "key": "elasticsearch.ingest_pipeline.name", + "negate": true, + "params": { + "query": ".fleet_final_pipeline-1" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "elasticsearch.ingest_pipeline.name": ".fleet_final_pipeline-1" + } + } + } + ], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "syncTooltips": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "enhancements": {}, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "params": { + "fontSize": 12, + "markdown": "# Overview\n\n\u003e Analyze trends in throughput and performance for overall ingest processing", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 6, + "i": "64faf306-3732-4cd2-8d17-a4fa03ca3e75", + "w": 48, + "x": 0, + "y": 0 + }, + "panelIndex": "64faf306-3732-4cd2-8d17-a4fa03ca3e75", + "title": "", + "type": "visualization", + "version": "8.5.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "elasticsearch-sm-metrics", + "name": "indexpattern-datasource-layer-3faa4240-11c3-43c8-980f-0dda1d9d3c08", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "indexpattern": { + "layers": { + "3faa4240-11c3-43c8-980f-0dda1d9d3c08": { + "columnOrder": [ + "7ee48652-d095-4cb7-af8a-c779631d4b8c" + ], + "columns": { + "7ee48652-d095-4cb7-af8a-c779631d4b8c": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Active pipelines", + "operationType": "unique_count", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.name" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "service.type:elasticsearch or data_stream.dataset:\"elasticsearch.ingest_pipeline\" " + }, + "visualization": { + "accessor": "7ee48652-d095-4cb7-af8a-c779631d4b8c", + "layerId": "3faa4240-11c3-43c8-980f-0dda1d9d3c08", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsLegacyMetric" + }, + "enhancements": {} + }, + "gridData": { + "h": 14, + "i": "e5ab9dda-69bd-4138-8df1-812e4a376d09", + "w": 6, + "x": 0, + "y": 6 + }, + "panelIndex": "e5ab9dda-69bd-4138-8df1-812e4a376d09", + "type": "lens", + "version": "8.5.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "elasticsearch-sm-metrics", + "name": "indexpattern-datasource-layer-3faa4240-11c3-43c8-980f-0dda1d9d3c08", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "indexpattern": { + "layers": { + "3faa4240-11c3-43c8-980f-0dda1d9d3c08": { + "columnOrder": [ + "7ee48652-d095-4cb7-af8a-c779631d4b8c" + ], + "columns": { + "7ee48652-d095-4cb7-af8a-c779631d4b8c": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Nodes running pipelines", + "operationType": "unique_count", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "elasticsearch.node.id" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "service.type:elasticsearch or data_stream.dataset:\"elasticsearch.ingest_pipeline\" " + }, + "visualization": { + "accessor": "7ee48652-d095-4cb7-af8a-c779631d4b8c", + "layerId": "3faa4240-11c3-43c8-980f-0dda1d9d3c08", + "layerType": "data" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsLegacyMetric" + }, + "enhancements": {} + }, + "gridData": { + "h": 14, + "i": "46467ea8-7ba6-43d1-b61a-d3b6e23e6742", + "w": 6, + "x": 6, + "y": 6 + }, + "panelIndex": "46467ea8-7ba6-43d1-b61a-d3b6e23e6742", + "title": "", + "type": "lens", + "version": "8.5.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "elasticsearch-sm-metrics", + "name": "indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "indexpattern": { + "layers": { + "da36c906-5865-4c1f-874d-b021fef6d425": { + "columnOrder": [ + "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61", + "d470f360-5ac9-450f-bef2-8c666d25cee7", + "d66446a1-b145-40ba-8d88-5771c524ab37", + "d66446a1-b145-40ba-8d88-5771c524ab37X1", + "d66446a1-b145-40ba-8d88-5771c524ab37X0", + "65cdf833-eab2-4485-be00-fca498ccdd85", + "65cdf833-eab2-4485-be00-fca498ccdd85X1", + "65cdf833-eab2-4485-be00-fca498ccdd85X0" + ], + "columns": { + "65cdf833-eab2-4485-be00-fca498ccdd85": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Failures", + "operationType": "formula", + "params": { + "formula": "counter_rate(max(elasticsearch.ingest_pipeline.total.failed))", + "isFormulaBroken": false + }, + "references": [ + "65cdf833-eab2-4485-be00-fca498ccdd85X1" + ], + "scale": "ratio", + "timeScale": "s" + }, + "65cdf833-eab2-4485-be00-fca498ccdd85X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Documents processed", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.failed" + }, + "65cdf833-eab2-4485-be00-fca498ccdd85X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Documents processed", + "operationType": "counter_rate", + "references": [ + "65cdf833-eab2-4485-be00-fca498ccdd85X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "d470f360-5ac9-450f-bef2-8c666d25cee7": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": true, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Documents processed", + "operationType": "formula", + "params": { + "formula": "counter_rate(max(elasticsearch.ingest_pipeline.total.count))", + "isFormulaBroken": false + }, + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X1" + ], + "scale": "ratio", + "timeScale": "s" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Events", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Events", + "operationType": "counter_rate", + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of elasticsearch.ingest_pipeline.name + 1 other", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "isBucketed": false, + "label": "Maximum of elasticsearch.ingest_pipeline.total.count", + "operationType": "max", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "multi_terms" + }, + "secondaryFields": [ + "elasticsearch.node.id" + ], + "size": 20 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.ingest_pipeline.name" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "service.type:elasticsearch or data_stream.dataset:\"elasticsearch.ingest_pipeline\" " + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "curveType": "CURVE_MONOTONE_X", + "emphasizeFitting": true, + "endValue": "None", + "fillOpacity": 0.5, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "d66446a1-b145-40ba-8d88-5771c524ab37", + "65cdf833-eab2-4485-be00-fca498ccdd85" + ], + "collapseFn": "sum", + "layerId": "da36c906-5865-4c1f-874d-b021fef6d425", + "layerType": "data", + "seriesType": "area", + "splitAccessor": "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61", + "xAccessor": "d470f360-5ac9-450f-bef2-8c666d25cee7", + "yConfig": [ + { + "axisMode": "left", + "color": "#54b399", + "forAccessor": "d66446a1-b145-40ba-8d88-5771c524ab37" + }, + { + "color": "#e7664c", + "forAccessor": "65cdf833-eab2-4485-be00-fca498ccdd85" + } + ] + } + ], + "legend": { + "isVisible": false, + "maxLines": 1, + "position": "top", + "shouldTruncate": true, + "showSingleSeries": false + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "yLeftExtent": { + "mode": "full" + }, + "yLeftScale": "linear", + "yRightExtent": { + "mode": "full" + }, + "yRightScale": "linear" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 14, + "i": "62553d02-5a7d-400b-979e-36da6aef012e", + "w": 36, + "x": 12, + "y": 6 + }, + "panelIndex": "62553d02-5a7d-400b-979e-36da6aef012e", + "title": "Total documents and failures", + "type": "lens", + "version": "8.5.1" + }, + { + "embeddableConfig": { + "enhancements": {}, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "id": "", + "params": { + "fontSize": 12, + "markdown": "# Breakdowns by pipeline and node\n\n\u003e Find poor performing or high throughput pipelines that may be impacting your ingest capacity\n\u003e Click the \"+\" icon next to a pipeline name in table to view the detailed pipeline and processor metrics", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 6, + "i": "32ce43ec-edb2-4e30-8fb3-df52e06c885c", + "w": 48, + "x": 0, + "y": 20 + }, + "panelIndex": "32ce43ec-edb2-4e30-8fb3-df52e06c885c", + "type": "visualization", + "version": "8.5.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "elasticsearch-sm-metrics", + "name": "indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "indexpattern": { + "layers": { + "da36c906-5865-4c1f-874d-b021fef6d425": { + "columnOrder": [ + "4a999eff-ba72-4344-9077-db8f9d27cf45", + "0bc394bd-4e71-4611-a300-e8d87d37efa6", + "f7c5cd36-1bc1-4d66-93cc-04f54459a66a", + "d66446a1-b145-40ba-8d88-5771c524ab37", + "f36a06ec-bdba-46d2-90ca-9c7de830c269", + "f36a06ec-bdba-46d2-90ca-9c7de830c269X1", + "f36a06ec-bdba-46d2-90ca-9c7de830c269X0", + "d66446a1-b145-40ba-8d88-5771c524ab37X1", + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "columns": { + "0bc394bd-4e71-4611-a300-e8d87d37efa6": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Pipeline", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderAgg": { + "customLabel": false, + "dataType": "number", + "isBucketed": false, + "label": "Count of records", + "operationType": "count", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "___records___" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": false, + "parentFormat": { + "id": "terms" + }, + "size": 500 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.ingest_pipeline.name" + }, + "4a999eff-ba72-4344-9077-db8f9d27cf45": { + "dataType": "string", + "isBucketed": true, + "label": "Top 500 values of elasticsearch.node.id", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "isBucketed": false, + "label": "Sum of elasticsearch.ingest_pipeline.total.count", + "operationType": "sum", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 500 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.node.id" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Documents", + "operationType": "formula", + "params": { + "formula": "counter_rate(max(elasticsearch.ingest_pipeline.total.count))", + "isFormulaBroken": false + }, + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X1" + ], + "scale": "ratio" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Documents", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Documents", + "operationType": "counter_rate", + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "f36a06ec-bdba-46d2-90ca-9c7de830c269": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Total time spent", + "operationType": "formula", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 0, + "suffix": " s" + } + }, + "formula": "counter_rate(max(elasticsearch.ingest_pipeline.total.time.self.ms))", + "isFormulaBroken": false + }, + "references": [ + "f36a06ec-bdba-46d2-90ca-9c7de830c269X1" + ], + "scale": "ratio" + }, + "f36a06ec-bdba-46d2-90ca-9c7de830c269X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Total time spent", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.time.self.ms" + }, + "f36a06ec-bdba-46d2-90ca-9c7de830c269X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Total time spent", + "operationType": "counter_rate", + "references": [ + "f36a06ec-bdba-46d2-90ca-9c7de830c269X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "f7c5cd36-1bc1-4d66-93cc-04f54459a66a": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "service.type:elasticsearch or data_stream.dataset:\"elasticsearch.ingest_pipeline\" " + }, + "visualization": { + "columns": [ + { + "collapseFn": "sum", + "columnId": "4a999eff-ba72-4344-9077-db8f9d27cf45", + "isTransposed": false + }, + { + "columnId": "d66446a1-b145-40ba-8d88-5771c524ab37", + "summaryRow": "sum" + }, + { + "collapseFn": "sum", + "columnId": "f7c5cd36-1bc1-4d66-93cc-04f54459a66a", + "isTransposed": false + }, + { + "columnId": "f36a06ec-bdba-46d2-90ca-9c7de830c269", + "isTransposed": false, + "summaryRow": "sum" + }, + { + "collapseFn": "", + "columnId": "0bc394bd-4e71-4611-a300-e8d87d37efa6", + "isTransposed": false + } + ], + "layerId": "da36c906-5865-4c1f-874d-b021fef6d425", + "layerType": "data", + "sorting": { + "columnId": "d66446a1-b145-40ba-8d88-5771c524ab37", + "direction": "desc" + } + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": { + "dynamicActions": { + "events": [ + { + "action": { + "config": { + "openInNewTab": false, + "useCurrentDateRange": true, + "useCurrentFilters": true + }, + "factoryId": "DASHBOARD_TO_DASHBOARD_DRILLDOWN", + "name": "View pipeline metrics" + }, + "eventId": "6f8f9ee3-9811-43de-8208-6e60ce2bb2af", + "triggers": [ + "FILTER_TRIGGER" + ] + } + ] + } + }, + "hidePanelTitles": false + }, + "gridData": { + "h": 37, + "i": "f959aa3e-3644-4f1a-a597-c3a22b9b7e0d", + "w": 17, + "x": 0, + "y": 26 + }, + "panelIndex": "f959aa3e-3644-4f1a-a597-c3a22b9b7e0d", + "title": "Top pipelines by documents and time", + "type": "lens", + "version": "8.5.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "elasticsearch-sm-metrics", + "name": "indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "indexpattern": { + "layers": { + "da36c906-5865-4c1f-874d-b021fef6d425": { + "columnOrder": [ + "d470f360-5ac9-450f-bef2-8c666d25cee7", + "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61", + "d66446a1-b145-40ba-8d88-5771c524ab37", + "d66446a1-b145-40ba-8d88-5771c524ab37X1", + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "columns": { + "d470f360-5ac9-450f-bef2-8c666d25cee7": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": true, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Documents", + "operationType": "formula", + "params": { + "formula": "counter_rate(max(elasticsearch.ingest_pipeline.total.count))", + "isFormulaBroken": false + }, + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X1" + ], + "scale": "ratio", + "timeScale": "s" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Events", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Events", + "operationType": "counter_rate", + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of elasticsearch.ingest_pipeline.name + 1 other", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "isBucketed": false, + "label": "Maximum of elasticsearch.ingest_pipeline.total.count", + "operationType": "max", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": false, + "parentFormat": { + "id": "multi_terms" + }, + "secondaryFields": [ + "elasticsearch.node.name" + ], + "size": 10 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.ingest_pipeline.name" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "service.type:elasticsearch or data_stream.dataset:\"elasticsearch.ingest_pipeline\" " + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "curveType": "CURVE_MONOTONE_X", + "fillOpacity": 0.5, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "d66446a1-b145-40ba-8d88-5771c524ab37" + ], + "collapseFn": "", + "layerId": "da36c906-5865-4c1f-874d-b021fef6d425", + "layerType": "data", + "seriesType": "line", + "splitAccessor": "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61", + "xAccessor": "d470f360-5ac9-450f-bef2-8c666d25cee7", + "yConfig": [ + { + "axisMode": "left", + "color": "#54b399", + "forAccessor": "d66446a1-b145-40ba-8d88-5771c524ab37" + } + ] + } + ], + "legend": { + "isVisible": true, + "maxLines": 1, + "position": "right", + "shouldTruncate": true, + "showSingleSeries": false + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "valuesInLegend": true, + "yLeftExtent": { + "mode": "full" + }, + "yLeftScale": "linear", + "yRightExtent": { + "mode": "full" + }, + "yRightScale": "linear" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 19, + "i": "d3fdc894-c21e-4ec4-954c-e1d9b45813e9", + "w": 31, + "x": 17, + "y": 26 + }, + "panelIndex": "d3fdc894-c21e-4ec4-954c-e1d9b45813e9", + "title": "Top pipelines by documents", + "type": "lens", + "version": "8.5.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "elasticsearch-sm-metrics", + "name": "indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "indexpattern": { + "layers": { + "da36c906-5865-4c1f-874d-b021fef6d425": { + "columnOrder": [ + "d470f360-5ac9-450f-bef2-8c666d25cee7", + "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61", + "d66446a1-b145-40ba-8d88-5771c524ab37", + "d66446a1-b145-40ba-8d88-5771c524ab37X1", + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "columns": { + "d470f360-5ac9-450f-bef2-8c666d25cee7": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": true, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Time spent (ms)", + "operationType": "formula", + "params": { + "formula": "counter_rate(max(elasticsearch.ingest_pipeline.total.time.self.ms))", + "isFormulaBroken": false + }, + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X1" + ], + "scale": "ratio" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Time spent", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.time.self.ms" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Time spent", + "operationType": "counter_rate", + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of elasticsearch.ingest_pipeline.name + 1 other", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "isBucketed": false, + "label": "Maximum of elasticsearch.ingest_pipeline.total.count", + "operationType": "max", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": false, + "parentFormat": { + "id": "multi_terms" + }, + "secondaryFields": [ + "elasticsearch.node.name" + ], + "size": 8 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.ingest_pipeline.name" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "service.type:elasticsearch or data_stream.dataset:\"elasticsearch.ingest_pipeline\" " + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "curveType": "CURVE_MONOTONE_X", + "fillOpacity": 0.5, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "d66446a1-b145-40ba-8d88-5771c524ab37" + ], + "collapseFn": "", + "layerId": "da36c906-5865-4c1f-874d-b021fef6d425", + "layerType": "data", + "seriesType": "line", + "splitAccessor": "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61", + "xAccessor": "d470f360-5ac9-450f-bef2-8c666d25cee7", + "yConfig": [ + { + "axisMode": "left", + "color": "#54b399", + "forAccessor": "d66446a1-b145-40ba-8d88-5771c524ab37" + } + ] + } + ], + "legend": { + "isVisible": true, + "maxLines": 1, + "position": "right", + "shouldTruncate": true, + "showSingleSeries": false + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "valuesInLegend": true, + "yLeftExtent": { + "mode": "full" + }, + "yLeftScale": "linear", + "yRightExtent": { + "mode": "full" + }, + "yRightScale": "linear" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 18, + "i": "0cc9444d-c323-4ceb-b43d-072caf360183", + "w": 31, + "x": 17, + "y": 45 + }, + "panelIndex": "0cc9444d-c323-4ceb-b43d-072caf360183", + "title": "Top pipelines by total processing time", + "type": "lens", + "version": "8.5.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "elasticsearch-sm-metrics", + "name": "indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "indexpattern": { + "layers": { + "da36c906-5865-4c1f-874d-b021fef6d425": { + "columnOrder": [ + "4a999eff-ba72-4344-9077-db8f9d27cf45", + "0bc394bd-4e71-4611-a300-e8d87d37efa6", + "f7c5cd36-1bc1-4d66-93cc-04f54459a66a", + "f36a06ec-bdba-46d2-90ca-9c7de830c269", + "f36a06ec-bdba-46d2-90ca-9c7de830c269X1", + "f36a06ec-bdba-46d2-90ca-9c7de830c269X0", + "f36a06ec-bdba-46d2-90ca-9c7de830c269X4", + "f36a06ec-bdba-46d2-90ca-9c7de830c269X3", + "f36a06ec-bdba-46d2-90ca-9c7de830c269X2", + "f36a06ec-bdba-46d2-90ca-9c7de830c269X5", + "f36a06ec-bdba-46d2-90ca-9c7de830c269X6" + ], + "columns": { + "0bc394bd-4e71-4611-a300-e8d87d37efa6": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Pipeline", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "isBucketed": false, + "label": "95th percentile of elasticsearch.ingest_pipeline.total.time.self.ms", + "operationType": "percentile", + "params": { + "percentile": 95 + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.time.self.ms" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": false, + "parentFormat": { + "id": "terms" + }, + "size": 500 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.ingest_pipeline.name" + }, + "4a999eff-ba72-4344-9077-db8f9d27cf45": { + "dataType": "string", + "isBucketed": true, + "label": "Top 500 values of elasticsearch.node.id", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "isBucketed": false, + "label": "Sum of elasticsearch.ingest_pipeline.total.count", + "operationType": "sum", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 500 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.node.id" + }, + "f36a06ec-bdba-46d2-90ca-9c7de830c269": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Avg time per doc", + "operationType": "formula", + "params": { + "format": { + "id": "number", + "params": { + "decimals": 1, + "suffix": " ms" + } + }, + "formula": "cumulative_sum(counter_rate(max(elasticsearch.ingest_pipeline.total.time.self.ms)))/cumulative_sum(counter_rate((max(elasticsearch.ingest_pipeline.total.count))))", + "isFormulaBroken": false + }, + "references": [ + "f36a06ec-bdba-46d2-90ca-9c7de830c269X6" + ], + "scale": "ratio" + }, + "f36a06ec-bdba-46d2-90ca-9c7de830c269X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Avg time per doc", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.time.self.ms" + }, + "f36a06ec-bdba-46d2-90ca-9c7de830c269X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Avg time per doc", + "operationType": "counter_rate", + "references": [ + "f36a06ec-bdba-46d2-90ca-9c7de830c269X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "f36a06ec-bdba-46d2-90ca-9c7de830c269X2": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Avg time per doc", + "operationType": "cumulative_sum", + "references": [ + "f36a06ec-bdba-46d2-90ca-9c7de830c269X1" + ], + "scale": "ratio" + }, + "f36a06ec-bdba-46d2-90ca-9c7de830c269X3": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Avg time per doc", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + }, + "f36a06ec-bdba-46d2-90ca-9c7de830c269X4": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Avg time per doc", + "operationType": "counter_rate", + "references": [ + "f36a06ec-bdba-46d2-90ca-9c7de830c269X3" + ], + "scale": "ratio", + "timeScale": "s" + }, + "f36a06ec-bdba-46d2-90ca-9c7de830c269X5": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Avg time per doc", + "operationType": "cumulative_sum", + "references": [ + "f36a06ec-bdba-46d2-90ca-9c7de830c269X4" + ], + "scale": "ratio" + }, + "f36a06ec-bdba-46d2-90ca-9c7de830c269X6": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Avg time per doc", + "operationType": "math", + "params": { + "tinymathAst": { + "args": [ + "f36a06ec-bdba-46d2-90ca-9c7de830c269X2", + "f36a06ec-bdba-46d2-90ca-9c7de830c269X5" + ], + "location": { + "max": 163, + "min": 0 + }, + "name": "divide", + "text": "cumulative_sum(counter_rate(max(elasticsearch.ingest_pipeline.total.time.self.ms)))/cumulative_sum(counter_rate((max(elasticsearch.ingest_pipeline.total.count))))", + "type": "function" + } + }, + "references": [ + "f36a06ec-bdba-46d2-90ca-9c7de830c269X2", + "f36a06ec-bdba-46d2-90ca-9c7de830c269X5" + ], + "scale": "ratio" + }, + "f7c5cd36-1bc1-4d66-93cc-04f54459a66a": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "service.type:elasticsearch or data_stream.dataset:\"elasticsearch.ingest_pipeline\" " + }, + "visualization": { + "columns": [ + { + "collapseFn": "avg", + "columnId": "4a999eff-ba72-4344-9077-db8f9d27cf45", + "isTransposed": false + }, + { + "collapseFn": "sum", + "columnId": "f7c5cd36-1bc1-4d66-93cc-04f54459a66a", + "isTransposed": false + }, + { + "columnId": "f36a06ec-bdba-46d2-90ca-9c7de830c269", + "isTransposed": false + }, + { + "collapseFn": "", + "columnId": "0bc394bd-4e71-4611-a300-e8d87d37efa6", + "isTransposed": false + } + ], + "layerId": "da36c906-5865-4c1f-874d-b021fef6d425", + "layerType": "data", + "sorting": { + "columnId": "f36a06ec-bdba-46d2-90ca-9c7de830c269", + "direction": "desc" + } + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": { + "dynamicActions": { + "events": [ + { + "action": { + "config": { + "openInNewTab": false, + "useCurrentDateRange": true, + "useCurrentFilters": true + }, + "factoryId": "DASHBOARD_TO_DASHBOARD_DRILLDOWN", + "name": "View pipeline metrics" + }, + "eventId": "7f5437c6-2143-478d-85f2-d9315bf312b9", + "triggers": [ + "FILTER_TRIGGER" + ] + } + ] + } + }, + "hidePanelTitles": false + }, + "gridData": { + "h": 18, + "i": "46db7a44-91d5-430a-b63b-9ccc90ebf512", + "w": 17, + "x": 0, + "y": 63 + }, + "panelIndex": "46db7a44-91d5-430a-b63b-9ccc90ebf512", + "title": "Slowest pipelines", + "type": "lens", + "version": "8.5.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "elasticsearch-sm-metrics", + "name": "indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "indexpattern": { + "layers": { + "da36c906-5865-4c1f-874d-b021fef6d425": { + "columnOrder": [ + "d470f360-5ac9-450f-bef2-8c666d25cee7", + "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61", + "d66446a1-b145-40ba-8d88-5771c524ab37", + "d66446a1-b145-40ba-8d88-5771c524ab37X1", + "d66446a1-b145-40ba-8d88-5771c524ab37X0", + "d66446a1-b145-40ba-8d88-5771c524ab37X4", + "d66446a1-b145-40ba-8d88-5771c524ab37X3", + "d66446a1-b145-40ba-8d88-5771c524ab37X2" + ], + "columns": { + "d470f360-5ac9-450f-bef2-8c666d25cee7": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": true, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Time spent per document processed (ms)", + "operationType": "formula", + "params": { + "formula": "counter_rate(max(elasticsearch.ingest_pipeline.total.time.self.ms)) / counter_rate(max(elasticsearch.ingest_pipeline.total.count))", + "isFormulaBroken": false + }, + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X4" + ], + "scale": "ratio" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Time spent per document processed", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.time.self.ms" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Time spent per document processed", + "operationType": "counter_rate", + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X2": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Time spent per document processed", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X3": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Time spent per document processed", + "operationType": "counter_rate", + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X2" + ], + "scale": "ratio", + "timeScale": "s" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X4": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Time spent per document processed", + "operationType": "math", + "params": { + "tinymathAst": { + "args": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X1", + "d66446a1-b145-40ba-8d88-5771c524ab37X3" + ], + "location": { + "max": 130, + "min": 0 + }, + "name": "divide", + "text": "counter_rate(max(elasticsearch.ingest_pipeline.total.time.self.ms)) / counter_rate(max(elasticsearch.ingest_pipeline.total.count))", + "type": "function" + } + }, + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X1", + "d66446a1-b145-40ba-8d88-5771c524ab37X3" + ], + "scale": "ratio" + }, + "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of elasticsearch.ingest_pipeline.name + 1 other", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "isBucketed": false, + "label": "Maximum of elasticsearch.ingest_pipeline.total.count", + "operationType": "max", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "multi_terms" + }, + "secondaryFields": [ + "elasticsearch.node.name" + ], + "size": 8 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.ingest_pipeline.name" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "service.type:elasticsearch or data_stream.dataset:\"elasticsearch.ingest_pipeline\" " + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "curveType": "CURVE_MONOTONE_X", + "fillOpacity": 0.5, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "d66446a1-b145-40ba-8d88-5771c524ab37" + ], + "collapseFn": "", + "layerId": "da36c906-5865-4c1f-874d-b021fef6d425", + "layerType": "data", + "seriesType": "line", + "splitAccessor": "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61", + "xAccessor": "d470f360-5ac9-450f-bef2-8c666d25cee7", + "yConfig": [ + { + "axisMode": "left", + "color": "#54b399", + "forAccessor": "d66446a1-b145-40ba-8d88-5771c524ab37" + } + ] + } + ], + "legend": { + "isVisible": true, + "maxLines": 1, + "position": "right", + "shouldTruncate": true, + "showSingleSeries": false + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "valuesInLegend": true, + "yLeftExtent": { + "mode": "full" + }, + "yLeftScale": "linear", + "yRightExtent": { + "mode": "full" + }, + "yRightScale": "linear" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 18, + "i": "4fe12cde-9e47-49fa-b1db-bfe6f99790eb", + "w": 31, + "x": 17, + "y": 63 + }, + "panelIndex": "4fe12cde-9e47-49fa-b1db-bfe6f99790eb", + "title": "Average time per document", + "type": "lens", + "version": "8.5.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "elasticsearch-sm-metrics", + "name": "indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "indexpattern": { + "layers": { + "da36c906-5865-4c1f-874d-b021fef6d425": { + "columnOrder": [ + "4a999eff-ba72-4344-9077-db8f9d27cf45", + "0bc394bd-4e71-4611-a300-e8d87d37efa6", + "f7c5cd36-1bc1-4d66-93cc-04f54459a66a", + "d66446a1-b145-40ba-8d88-5771c524ab37", + "d66446a1-b145-40ba-8d88-5771c524ab37X1", + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "columns": { + "0bc394bd-4e71-4611-a300-e8d87d37efa6": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Pipeline", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "isBucketed": false, + "label": "95th percentile of elasticsearch.ingest_pipeline.total.failed", + "operationType": "percentile", + "params": { + "percentile": 95 + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.failed" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 500 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.ingest_pipeline.name" + }, + "4a999eff-ba72-4344-9077-db8f9d27cf45": { + "dataType": "string", + "isBucketed": true, + "label": "Top 500 values of elasticsearch.node.id", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderAgg": { + "customLabel": false, + "dataType": "number", + "isBucketed": false, + "label": "Count of elasticsearch.ingest_pipeline.total.count", + "operationType": "count", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 500 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.node.id" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Failed documents", + "operationType": "formula", + "params": { + "formula": "counter_rate(max(elasticsearch.ingest_pipeline.total.failed))", + "isFormulaBroken": false + }, + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X1" + ], + "scale": "ratio" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Failed documents", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.failed" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Failed documents", + "operationType": "counter_rate", + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "f7c5cd36-1bc1-4d66-93cc-04f54459a66a": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "service.type:elasticsearch or data_stream.dataset:\"elasticsearch.ingest_pipeline\" " + }, + "visualization": { + "columns": [ + { + "collapseFn": "sum", + "columnId": "4a999eff-ba72-4344-9077-db8f9d27cf45", + "isTransposed": false + }, + { + "columnId": "d66446a1-b145-40ba-8d88-5771c524ab37", + "summaryRow": "none" + }, + { + "collapseFn": "sum", + "columnId": "f7c5cd36-1bc1-4d66-93cc-04f54459a66a", + "isTransposed": false + }, + { + "collapseFn": "", + "columnId": "0bc394bd-4e71-4611-a300-e8d87d37efa6", + "isTransposed": false + } + ], + "layerId": "da36c906-5865-4c1f-874d-b021fef6d425", + "layerType": "data", + "sorting": { + "columnId": "d66446a1-b145-40ba-8d88-5771c524ab37", + "direction": "desc" + } + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsDatatable" + }, + "enhancements": { + "dynamicActions": { + "events": [ + { + "action": { + "config": { + "openInNewTab": false, + "useCurrentDateRange": true, + "useCurrentFilters": true + }, + "factoryId": "DASHBOARD_TO_DASHBOARD_DRILLDOWN", + "name": "View pipeline metrics" + }, + "eventId": "645f0eae-a920-4536-94ae-93453846b9d8", + "triggers": [ + "FILTER_TRIGGER" + ] + } + ] + } + }, + "hidePanelTitles": false + }, + "gridData": { + "h": 18, + "i": "2d2c07ad-db4f-46e2-a984-4b14ce17ac84", + "w": 17, + "x": 0, + "y": 81 + }, + "panelIndex": "2d2c07ad-db4f-46e2-a984-4b14ce17ac84", + "title": "Top failing pipelines", + "type": "lens", + "version": "8.5.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "elasticsearch-sm-metrics", + "name": "indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "indexpattern": { + "layers": { + "da36c906-5865-4c1f-874d-b021fef6d425": { + "columnOrder": [ + "d470f360-5ac9-450f-bef2-8c666d25cee7", + "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61", + "d66446a1-b145-40ba-8d88-5771c524ab37", + "d66446a1-b145-40ba-8d88-5771c524ab37X1", + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "columns": { + "d470f360-5ac9-450f-bef2-8c666d25cee7": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "dropPartials": true, + "includeEmptyRows": true, + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Failed Documents", + "operationType": "formula", + "params": { + "formula": "counter_rate(max(elasticsearch.ingest_pipeline.total.failed))", + "isFormulaBroken": false + }, + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X1" + ], + "scale": "ratio", + "timeScale": "s" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Failed Documents", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.failed" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Failed Documents", + "operationType": "counter_rate", + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "scale": "ratio", + "timeScale": "s" + }, + "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of elasticsearch.ingest_pipeline.name + 1 other", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "isBucketed": false, + "label": "Maximum of elasticsearch.ingest_pipeline.total.failed", + "operationType": "max", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.failed" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "multi_terms" + }, + "secondaryFields": [ + "elasticsearch.node.name" + ], + "size": 10 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.ingest_pipeline.name" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "service.type:elasticsearch or data_stream.dataset:\"elasticsearch.ingest_pipeline\" " + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "curveType": "CURVE_MONOTONE_X", + "fillOpacity": 0.5, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "d66446a1-b145-40ba-8d88-5771c524ab37" + ], + "collapseFn": "", + "layerId": "da36c906-5865-4c1f-874d-b021fef6d425", + "layerType": "data", + "seriesType": "line", + "splitAccessor": "e8c1f37d-ecec-4ff7-b8e4-1d50c947ac61", + "xAccessor": "d470f360-5ac9-450f-bef2-8c666d25cee7", + "yConfig": [ + { + "axisMode": "left", + "color": "#54b399", + "forAccessor": "d66446a1-b145-40ba-8d88-5771c524ab37" + } + ] + } + ], + "legend": { + "isVisible": true, + "maxLines": 1, + "position": "right", + "shouldTruncate": true, + "showSingleSeries": false + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "valuesInLegend": true, + "yLeftExtent": { + "mode": "full" + }, + "yLeftScale": "linear", + "yRightExtent": { + "mode": "full" + }, + "yRightScale": "linear" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 18, + "i": "da89ce7b-9065-4c56-affc-02eeb02723f0", + "w": 31, + "x": 17, + "y": 81 + }, + "panelIndex": "da89ce7b-9065-4c56-affc-02eeb02723f0", + "title": "Failures processed by top nodes processing top pipelines", + "type": "lens", + "version": "8.5.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "elasticsearch-sm-metrics", + "name": "indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "indexpattern": { + "layers": { + "da36c906-5865-4c1f-874d-b021fef6d425": { + "columnOrder": [ + "4a999eff-ba72-4344-9077-db8f9d27cf45", + "a3252405-b001-460f-bb01-494dc15447ba", + "d66446a1-b145-40ba-8d88-5771c524ab37", + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "columns": { + "4a999eff-ba72-4344-9077-db8f9d27cf45": { + "dataType": "string", + "isBucketed": true, + "label": "Top 5 values of elasticsearch.ingest_pipeline.name", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "isBucketed": false, + "label": "Sum of elasticsearch.ingest_pipeline.total.count", + "operationType": "sum", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": false, + "parentFormat": { + "id": "terms" + }, + "size": 5 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.ingest_pipeline.name" + }, + "a3252405-b001-460f-bb01-494dc15447ba": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Node", + "operationType": "terms", + "params": { + "accuracyMode": false, + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "isBucketed": false, + "label": "Maximum of elasticsearch.ingest_pipeline.total.count", + "operationType": "max", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "secondaryFields": [], + "size": 25 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.node.id" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Documents", + "operationType": "formula", + "params": { + "formula": "max(elasticsearch.ingest_pipeline.total.count)", + "isFormulaBroken": false + }, + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "scale": "ratio" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Documents", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "service.type:elasticsearch or data_stream.dataset:\"elasticsearch.ingest_pipeline\" " + }, + "visualization": { + "gridConfig": { + "isCellLabelVisible": true, + "isXAxisLabelVisible": true, + "isXAxisTitleVisible": false, + "isYAxisLabelVisible": true, + "isYAxisTitleVisible": false, + "type": "heatmap_grid" + }, + "layerId": "da36c906-5865-4c1f-874d-b021fef6d425", + "layerType": "data", + "legend": { + "isVisible": false, + "position": "right", + "type": "heatmap_legend" + }, + "palette": { + "accessor": "d66446a1-b145-40ba-8d88-5771c524ab37", + "name": "warm", + "params": { + "continuity": "above", + "name": "warm", + "rangeMax": null, + "rangeMin": 0, + "reverse": false, + "stops": [ + { + "color": "#f7e0b8", + "stop": 0 + }, + { + "color": "#f2c596", + "stop": 20 + }, + { + "color": "#eca976", + "stop": 40 + }, + { + "color": "#e78c5b", + "stop": 60 + }, + { + "color": "#e7664c", + "stop": 80 + } + ] + }, + "type": "palette" + }, + "shape": "heatmap", + "valueAccessor": "d66446a1-b145-40ba-8d88-5771c524ab37", + "xAccessor": "4a999eff-ba72-4344-9077-db8f9d27cf45", + "yAccessor": "a3252405-b001-460f-bb01-494dc15447ba" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsHeatmap" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 16, + "i": "0b317030-44f2-46d1-9790-c1a7e0ac0fae", + "w": 48, + "x": 0, + "y": 99 + }, + "panelIndex": "0b317030-44f2-46d1-9790-c1a7e0ac0fae", + "title": "Node x pipeline hotspots - by documents processed", + "type": "lens", + "version": "8.5.1" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "elasticsearch-sm-metrics", + "name": "indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "indexpattern": { + "layers": { + "da36c906-5865-4c1f-874d-b021fef6d425": { + "columnOrder": [ + "4a999eff-ba72-4344-9077-db8f9d27cf45", + "a3252405-b001-460f-bb01-494dc15447ba", + "d66446a1-b145-40ba-8d88-5771c524ab37", + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "columns": { + "4a999eff-ba72-4344-9077-db8f9d27cf45": { + "dataType": "string", + "isBucketed": true, + "label": "Top 5 values of elasticsearch.ingest_pipeline.name", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "isBucketed": false, + "label": "Sum of elasticsearch.ingest_pipeline.total.count", + "operationType": "sum", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": false, + "parentFormat": { + "id": "terms" + }, + "size": 5 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.ingest_pipeline.name" + }, + "a3252405-b001-460f-bb01-494dc15447ba": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Node", + "operationType": "terms", + "params": { + "accuracyMode": false, + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "isBucketed": false, + "label": "Maximum of elasticsearch.ingest_pipeline.total.count", + "operationType": "max", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.count" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "secondaryFields": [], + "size": 25 + }, + "scale": "ordinal", + "sourceField": "elasticsearch.node.id" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Time spent", + "operationType": "formula", + "params": { + "formula": "max(elasticsearch.ingest_pipeline.total.time.self.ms)", + "isFormulaBroken": false + }, + "references": [ + "d66446a1-b145-40ba-8d88-5771c524ab37X0" + ], + "scale": "ratio" + }, + "d66446a1-b145-40ba-8d88-5771c524ab37X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Documents", + "operationType": "max", + "params": { + "emptyAsNull": false + }, + "scale": "ratio", + "sourceField": "elasticsearch.ingest_pipeline.total.time.self.ms" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "service.type:elasticsearch or data_stream.dataset:\"elasticsearch.ingest_pipeline\" " + }, + "visualization": { + "gridConfig": { + "isCellLabelVisible": true, + "isXAxisLabelVisible": true, + "isXAxisTitleVisible": false, + "isYAxisLabelVisible": true, + "isYAxisTitleVisible": false, + "type": "heatmap_grid" + }, + "layerId": "da36c906-5865-4c1f-874d-b021fef6d425", + "layerType": "data", + "legend": { + "isVisible": false, + "position": "right", + "type": "heatmap_legend" + }, + "palette": { + "accessor": "d66446a1-b145-40ba-8d88-5771c524ab37", + "name": "warm", + "params": { + "continuity": "above", + "name": "warm", + "rangeMax": null, + "rangeMin": 0, + "reverse": false, + "stops": [ + { + "color": "#f7e0b8", + "stop": 0 + }, + { + "color": "#f2c596", + "stop": 20 + }, + { + "color": "#eca976", + "stop": 40 + }, + { + "color": "#e78c5b", + "stop": 60 + }, + { + "color": "#e7664c", + "stop": 80 + } + ] + }, + "type": "palette" + }, + "shape": "heatmap", + "valueAccessor": "d66446a1-b145-40ba-8d88-5771c524ab37", + "xAccessor": "4a999eff-ba72-4344-9077-db8f9d27cf45", + "yAccessor": "a3252405-b001-460f-bb01-494dc15447ba" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsHeatmap" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 16, + "i": "6c703288-f2a0-42e8-8125-ee5d80a15bdf", + "w": 48, + "x": 0, + "y": 115 + }, + "panelIndex": "6c703288-f2a0-42e8-8125-ee5d80a15bdf", + "title": "Node x pipeline hotspots - by time spent", + "type": "lens", + "version": "8.5.1" + } + ], + "timeRestore": false, + "title": "[Elasticsearch] Ingest Pipelines", + "version": 1 + }, + "coreMigrationVersion": "8.5.1", + "id": "elasticsearch-metrics-ingest-pipelines", + "migrationVersion": { + "dashboard": "8.5.0" + }, + "references": [ + { + "id": "elasticsearch-sm-metrics", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "e5ab9dda-69bd-4138-8df1-812e4a376d09:indexpattern-datasource-layer-3faa4240-11c3-43c8-980f-0dda1d9d3c08", + "type": "index-pattern" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "46467ea8-7ba6-43d1-b61a-d3b6e23e6742:indexpattern-datasource-layer-3faa4240-11c3-43c8-980f-0dda1d9d3c08", + "type": "index-pattern" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "62553d02-5a7d-400b-979e-36da6aef012e:indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "f959aa3e-3644-4f1a-a597-c3a22b9b7e0d:indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + }, + { + "id": "elasticsearch-ea5b81a0-7fbf-11ed-8509-ddabeb9daeaf", + "name": "f959aa3e-3644-4f1a-a597-c3a22b9b7e0d:drilldown:DASHBOARD_TO_DASHBOARD_DRILLDOWN:6f8f9ee3-9811-43de-8208-6e60ce2bb2af:dashboardId", + "type": "dashboard" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "d3fdc894-c21e-4ec4-954c-e1d9b45813e9:indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "0cc9444d-c323-4ceb-b43d-072caf360183:indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "46db7a44-91d5-430a-b63b-9ccc90ebf512:indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + }, + { + "id": "elasticsearch-ea5b81a0-7fbf-11ed-8509-ddabeb9daeaf", + "name": "46db7a44-91d5-430a-b63b-9ccc90ebf512:drilldown:DASHBOARD_TO_DASHBOARD_DRILLDOWN:7f5437c6-2143-478d-85f2-d9315bf312b9:dashboardId", + "type": "dashboard" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "4fe12cde-9e47-49fa-b1db-bfe6f99790eb:indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "2d2c07ad-db4f-46e2-a984-4b14ce17ac84:indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + }, + { + "id": "elasticsearch-ea5b81a0-7fbf-11ed-8509-ddabeb9daeaf", + "name": "2d2c07ad-db4f-46e2-a984-4b14ce17ac84:drilldown:DASHBOARD_TO_DASHBOARD_DRILLDOWN:645f0eae-a920-4536-94ae-93453846b9d8:dashboardId", + "type": "dashboard" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "da89ce7b-9065-4c56-affc-02eeb02723f0:indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "0b317030-44f2-46d1-9790-c1a7e0ac0fae:indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "6c703288-f2a0-42e8-8125-ee5d80a15bdf:indexpattern-datasource-layer-da36c906-5865-4c1f-874d-b021fef6d425", + "type": "index-pattern" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "controlGroup_d71679da-0013-430a-982a-7aa466e582f8:optionsListDataView", + "type": "index-pattern" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "controlGroup_01070ea3-1a87-4fcc-927d-18b02827b891:optionsListDataView", + "type": "index-pattern" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "controlGroup_de9376f3-00fd-46cb-9b75-3909c5d17a37:optionsListDataView", + "type": "index-pattern" + }, + { + "id": "elasticsearch-sm-metrics", + "name": "controlGroup_b89ded5a-50c5-47b3-a935-ee12f9f8aaf7:optionsListDataView", + "type": "index-pattern" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/elasticsearch/kibana/index_pattern/elasticsearch-sm-metrics.json b/packages/elasticsearch/kibana/index_pattern/elasticsearch-sm-metrics.json new file mode 100644 index 00000000000..df98ae560d1 --- /dev/null +++ b/packages/elasticsearch/kibana/index_pattern/elasticsearch-sm-metrics.json @@ -0,0 +1,22 @@ +{ + "attributes": { + "fieldAttrs": "{}", + "fieldFormatMap": "{}", + "fields": "[]", + "name": "Stack Monitoring Metrics", + "runtimeFieldMap": "{}", + "sourceFilters": "[]", + "timeFieldName": "@timestamp", + "title": "metrics-*,metricbeat-*,.monitoring-*", + "typeMeta": "{}" + }, + "coreMigrationVersion": "8.5.0", + "id": "elasticsearch-sm-metrics", + "migrationVersion": { + "index-pattern": "8.0.0" + }, + "references": [], + "type": "index-pattern", + "updated_at": "2022-12-12T16:28:45.196Z", + "version": "Wzg0OCwxXQ==" +} diff --git a/packages/elasticsearch/manifest.yml b/packages/elasticsearch/manifest.yml index d02fa3750da..befa94fd124 100644 --- a/packages/elasticsearch/manifest.yml +++ b/packages/elasticsearch/manifest.yml @@ -1,6 +1,6 @@ name: elasticsearch title: Elasticsearch -version: 1.3.0 +version: 1.4.0-beta1 description: Elasticsearch Integration type: integration icons: @@ -12,7 +12,7 @@ format_version: 1.0.0 license: basic categories: ["elastic_stack", "datastore"] conditions: - kibana.version: ^8.5.0 + kibana.version: ^8.7.0 policy_templates: - name: elasticsearch title: Elasticsearch logs and metrics