Convert Istio dashboards to Kibana Lens (#5268)

* convert dashboards from TSVB to Lens * add data_set.datastream pre-filter * some other layout optimizations
elastic · Mar 7, 2023 · 3c02fde · 3c02fde
1 parent 96e8c05
commit 3c02fde
Show file tree

Hide file tree

Showing 12 changed files with 4,711 additions and 2,511 deletions.
diff --git a/packages/istio/changelog.yml b/packages/istio/changelog.yml
@@ -1,3 +1,9 @@
+# newer versions go on top
+- version: "0.2.4"
+  changes:
+    - description: Convert dashboards to Lens and other fixes
+      type: enhancement # can be one of: enhancement, bugfix, breaking-change
+      link: https://github.com/elastic/integrations/pull/5268
 - version: "0.2.3"
   changes:
     - description: Fix Access Log Common Format Ingest Grok Pattern for IPv6

diff --git a/...ges/istio/data_stream/access_logs/_dev/test/pipeline/test-access-failed.log-expected.json b/...ges/istio/data_stream/access_logs/_dev/test/pipeline/test-access-failed.log-expected.json
@@ -11,7 +11,7 @@
                 ],
                 "created": "2020-04-28T11:07:58.223Z",
                 "id": "5735b2d0-4949-9384-b082-9d4c368eb9dc",
-                "ingested": "2022-09-09T09:23:51.058741510Z",
+                "ingested": "2023-02-10T18:34:31.109258337Z",
                 "kind": "event",
                 "module": "istio",
                 "original": "[2022-06-22T10:17:14.118Z] \"GET /status/418 HTTP/1.1\" 503 UH no_healthy_upstream - \"-\" 0 19 1 - \"-\" \"curl/7.83.1-DEV\" \"5735b2d0-4949-9384-b082-9d4c368eb9dc\" \"httpbin:8000\" \"-\" outbound|8000||httpbin.default.svc.cluster.local - 10.99.37.119:8000 172.17.0.6:44690 - default",
@@ -104,7 +104,7 @@
                 "created": "2020-04-28T11:07:58.223Z",
                 "duration": 25000000,
                 "id": "adc1465d-589f-9119-bdcf-2ea373ff7d7a",
-                "ingested": "2022-09-09T09:23:51.058748926Z",
+                "ingested": "2023-02-10T18:34:31.109293295Z",
                 "kind": "event",
                 "module": "istio",
                 "original": "[2022-06-22T10:27:00.362Z] \"GET /status/418 HTTP/1.1\" 418 - via_upstream - \"-\" 0 135 25 25 \"-\" \"curl/7.83.1-DEV\" \"adc1465d-589f-9119-bdcf-2ea373ff7d7a\" \"httpbin:8000\" \"172.17.0.7:80\" outbound|8000||httpbin.default.svc.cluster.local 172.17.0.6:57130 10.99.37.119:8000 172.17.0.6:50514 - default",
@@ -195,7 +195,7 @@
                 ],
                 "created": "2020-04-28T11:07:58.223Z",
                 "id": "5169ae9d-b98e-9d3c-bf74-2fb5bc5102f3",
-                "ingested": "2022-09-09T09:23:51.058750343Z",
+                "ingested": "2023-02-10T18:34:31.109295503Z",
                 "kind": "event",
                 "module": "istio",
                 "original": "[2022-09-01T09:50:40.396Z] \"GET /favicon.ico HTTP/1.1\" 404 NR route_not_found - \"-\" 0 0 0 - \"10.124.2.1\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36\" \"5169ae9d-b98e-9d3c-bf74-2fb5bc5102f3\" \"35.232.196.88\" \"-\" - - 10.124.2.10:8080 10.124.2.1:51978 - -",
@@ -284,7 +284,7 @@
                 ],
                 "created": "2020-04-28T11:07:58.223Z",
                 "id": "ae9ca454-2d21-956c-9cf4-813929267bb5",
-                "ingested": "2022-09-09T09:23:51.058751593Z",
+                "ingested": "2023-02-10T18:34:31.109306003Z",
                 "kind": "event",
                 "module": "istio",
                 "original": "[2022-09-01T11:33:22.455Z] \"GET /portal/redlion HTTP/1.1\" 404 NR route_not_found - \"-\" 0 0 0 - \"10.124.2.1\" \"Mozilla/5.0 zgrab/0.x\" \"ae9ca454-2d21-956c-9cf4-813929267bb5\" \"35.232.196.88\" \"-\" - - 10.124.2.10:8080 10.124.2.1:46713 - -",
@@ -363,7 +363,7 @@
             },
             "event": {
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-09-09T09:23:51.058752676Z",
+                "ingested": "2023-02-10T18:34:31.109307670Z",
                 "kind": "event",
                 "module": "istio",
                 "original": "[2022-09-01T09:36:33.553Z] \"- - HTTP/1.1\" 400 DPE http1.codec_error - \"-\" 0 11 46 - \"-\" \"-\" \"-\" \"-\" \"-\" - - 10.124.2.10:8080 10.128.0.60:51643 - -",
@@ -429,7 +429,7 @@
                 ],
                 "created": "2020-04-28T11:07:58.223Z",
                 "id": "7c20db9a-f48b-9a68-a158-12ca79dfdd05",
-                "ingested": "2022-09-09T09:23:51.058753843Z",
+                "ingested": "2023-02-10T18:34:31.109312920Z",
                 "kind": "event",
                 "module": "istio",
                 "original": "[2022-09-01T13:15:57.905Z] \"GET /productpage HTTP/1.1\" 503 UH no_healthy_upstream - \"-\" 0 19 0 - \"10.128.0.60\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36\" \"7c20db9a-f48b-9a68-a158-12ca79dfdd05\" \"35.232.196.88\" \"-\" outbound|9080||productpage.default.svc.cluster.local - 10.124.2.10:8080 10.128.0.60:21742 - -",
@@ -526,7 +526,7 @@
                 ],
                 "created": "2020-04-28T11:07:58.223Z",
                 "id": "56446678-64f3-9fbf-9c4a-eca01fafba4d",
-                "ingested": "2022-09-09T09:23:51.058754968Z",
+                "ingested": "2023-02-10T18:34:31.109314253Z",
                 "kind": "event",
                 "module": "istio",
                 "original": "[2022-09-01T13:15:09.085Z] \"GET /reviews/0 HTTP/1.1\" 503 UF upstream_reset_before_response_started{connection_failure,delayed_connect_error:_111} - \"-\" 0 145 0 - \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36\" \"56446678-64f3-9fbf-9c4a-eca01fafba4d\" \"reviews:9080\" \"10.124.2.14:9080\" inbound|9080|| - 10.124.2.14:9080 10.124.1.6:48816 outbound_.9080_._.reviews.default.svc.cluster.local default",
@@ -627,7 +627,7 @@
                 ],
                 "created": "2020-04-28T11:07:58.223Z",
                 "id": "2dc64af2-bd13-9962-819e-0b72064e7909",
-                "ingested": "2022-09-09T09:23:51.058756010Z",
+                "ingested": "2023-02-10T18:34:31.109338545Z",
                 "kind": "event",
                 "module": "istio",
                 "original": "[2022-09-01T13:08:55.358Z] \"GET /productpage HTTP/1.1\" 0 DC downstream_remote_disconnect - \"-\" 0 0 746 - \"10.128.0.60\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36\" \"2dc64af2-bd13-9962-819e-0b72064e7909\" \"35.232.196.88\" \"10.124.1.6:9080\" inbound|9080|| 127.0.0.6:38601 10.124.1.6:9080 10.128.0.60:0 outbound_.9080_._.productpage.default.svc.cluster.local default",

diff --git a/.../istio/data_stream/access_logs/_dev/test/pipeline/test-access-json-logs.log-expected.json b/.../istio/data_stream/access_logs/_dev/test/pipeline/test-access-json-logs.log-expected.json
@@ -17,7 +17,7 @@
                 "created": "2020-04-28T11:07:58.223Z",
                 "duration": 14000000,
                 "id": "9fcc7101-13bd-9753-98d6-820806088d9d",
-                "ingested": "2022-09-09T09:23:51.109644635Z",
+                "ingested": "2023-02-10T18:34:31.180442670Z",
                 "kind": "event",
                 "module": "istio",
                 "original": "{\"authority\":\"reviews:9080\",\"bytes_received\":0,\"bytes_sent\":358,\"connection_termination_details\":null,\"downstream_local_address\":\"10.124.1.5:9080\",\"downstream_remote_address\":\"10.124.0.13:43612\",\"duration\":15,\"method\":\"GET\",\"path\":\"/reviews/0\",\"protocol\":\"HTTP/1.1\",\"request_id\":\"9fcc7101-13bd-9753-98d6-820806088d9d\",\"requested_server_name\":\"outbound_.9080_._.reviews.default.svc.cluster.local\",\"response_code\":200,\"response_code_details\":\"via_upstream\",\"response_flags\":\"-\",\"route_name\":\"default\",\"start_time\":\"2022-09-08T12:51:05.656Z\",\"upstream_cluster\":\"inbound|9080||\",\"upstream_host\":\"10.124.1.5:9080\",\"upstream_local_address\":\"127.0.0.6:37685\",\"upstream_service_time\":\"14\",\"upstream_transport_failure_reason\":null,\"user_agent\":\"curl/7.79.1\",\"x_forwarded_for\":null}",
@@ -115,7 +115,7 @@
                 "created": "2020-04-28T11:07:58.223Z",
                 "duration": 14000000,
                 "id": "9fcc7101-13bd-9753-98d6-820806088d9d",
-                "ingested": "2022-09-09T09:23:51.109652093Z",
+                "ingested": "2023-02-10T18:34:31.180460587Z",
                 "kind": "event",
                 "module": "istio",
                 "original": "{\"authority\":\"reviews:9080\",\"bytes_received\":0,\"bytes_sent\":358,\"connection_termination_details\":null,\"downstream_local_address\":\"10.124.1.5:9080\",\"downstream_remote_address\":\"[2a02:cf40::4e36]:5000\",\"duration\":15,\"method\":\"GET\",\"path\":\"/reviews/0\",\"protocol\":\"HTTP/1.1\",\"request_id\":\"9fcc7101-13bd-9753-98d6-820806088d9d\",\"requested_server_name\":\"outbound_.9080_._.reviews.default.svc.cluster.local\",\"response_code\":200,\"response_code_details\":\"via_upstream\",\"response_flags\":\"-\",\"route_name\":\"default\",\"start_time\":\"2022-09-08T12:51:05.656Z\",\"upstream_cluster\":\"inbound|9080||\",\"upstream_host\":\"10.124.1.5:9080\",\"upstream_local_address\":\"127.0.0.6:37685\",\"upstream_service_time\":\"14\",\"upstream_transport_failure_reason\":null,\"user_agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.114 Safari/537.36\",\"x_forwarded_for\":null}",
@@ -217,7 +217,7 @@
             },
             "event": {
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-09-09T09:23:51.109653385Z",
+                "ingested": "2023-02-10T18:34:31.180462420Z",
                 "kind": "event",
                 "module": "istio",
                 "original": "{\"authority\":\"-\",\"bytes_received\":0,\"bytes_sent\":11,\"connection_termination_details\":null,\"downstream_local_address\":\"10.124.2.10:8080\",\"downstream_remote_address\":\"10.128.0.60:51643\",\"duration\":46,\"method\":\"-\",\"path\":\"-\",\"protocol\":\"HTTP/1.1\",\"request_id\":\"-\",\"requested_server_name\":\"-\",\"response_code\":400,\"response_code_details\":\"http1.codec_error\",\"response_flags\":\"DPE\",\"route_name\":\"-\",\"start_time\":\"2022-09-01T09:36:33.553Z\",\"upstream_cluster\":null,\"upstream_host\":\"-\",\"upstream_local_address\":null,\"upstream_service_time\":null,\"upstream_transport_failure_reason\":\"-\",\"user_agent\":\"-\",\"x_forwarded_for\":null}",

diff --git a/packages/istio/data_stream/access_logs/_dev/test/pipeline/test-access-logs.log-expected.json b/packages/istio/data_stream/access_logs/_dev/test/pipeline/test-access-logs.log-expected.json
@@ -17,7 +17,7 @@
                 "created": "2020-04-28T11:07:58.223Z",
                 "duration": 1000000,
                 "id": "785918d6-06b6-9312-bf77-6d9bd968dc21",
-                "ingested": "2022-09-09T09:23:51.149047676Z",
+                "ingested": "2023-02-10T18:34:31.225572837Z",
                 "kind": "event",
                 "module": "istio",
                 "original": "[2022-07-20T09:52:24.955Z] \"GET /details/0 HTTP/1.1\" 200 - via_upstream - \"-\" 0 178 2 1 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.114 Safari/537.36\" \"785918d6-06b6-9312-bf77-6d9bd968dc21\" \"details:9080\" \"10.68.2.10:9080\" inbound|9080|| 127.0.0.6:47889 10.68.2.10:9080 10.68.1.7:39696 outbound_.9080_._.details.default.svc.cluster.local default",
@@ -120,7 +120,7 @@
                 "created": "2020-04-28T11:07:58.223Z",
                 "duration": 3000000,
                 "id": "96bf54e0-6dac-9c1f-8090-b4acd6f779ef",
-                "ingested": "2022-09-09T09:23:51.149055218Z",
+                "ingested": "2023-02-10T18:34:31.225592337Z",
                 "kind": "event",
                 "module": "istio",
                 "original": "[2022-07-20T09:52:25.642Z] \"GET /static/bootstrap/fonts/glyphicons-halflings-regular.woff2 HTTP/1.1\" 200 - via_upstream - \"-\" 0 18028 3 3 \"10.128.0.64\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.114 Safari/537.36\" \"96bf54e0-6dac-9c1f-8090-b4acd6f779ef\" \"35.184.82.199\" \"10.68.1.7:9080\" inbound|9080|| 127.0.0.6:41929 10.68.1.7:9080 10.128.0.64:0 outbound_.9080_._.productpage.default.svc.cluster.local default",
@@ -222,7 +222,7 @@
                 "created": "2020-04-28T11:07:58.223Z",
                 "duration": 29000000,
                 "id": "f5dd9a93-895b-9a0d-a830-44694ed10c4c",
-                "ingested": "2022-09-09T09:23:51.149056635Z",
+                "ingested": "2023-02-10T18:34:31.225594253Z",
                 "kind": "event",
                 "module": "istio",
                 "original": "[2022-08-22T10:51:35.926Z] \"GET /reviews/0 HTTP/1.1\" 200 - via_upstream - \"-\" 0 437 30 29 \"-\" \"curl/7.79.1\" \"f5dd9a93-895b-9a0d-a830-44694ed10c4c\" \"reviews:9080\" \"10.124.0.12:9080\" inbound|9080|| 127.0.0.6:50151 10.124.0.12:9080 10.124.2.6:54438 outbound_.9080_._.reviews.default.svc.cluster.local default",
@@ -320,7 +320,7 @@
                 "created": "2020-04-28T11:07:58.223Z",
                 "duration": 12000000,
                 "id": "00bcf1d7-12c4-93b9-b36a-bc0faac8e428",
-                "ingested": "2022-09-09T09:23:51.149057885Z",
+                "ingested": "2023-02-10T18:34:31.225597295Z",
                 "kind": "event",
                 "module": "istio",
                 "original": "[2022-08-22T13:08:23.188Z] \"GET /reviews/0 HTTP/1.1\" 200 - via_upstream - \"-\" 0 442 12 12 \"-\" \"curl/7.79.1\" \"00bcf1d7-12c4-93b9-b36a-bc0faac8e428\" \"reviews:9080\" \"10.124.1.5:9080\" outbound|9080||reviews.default.svc.cluster.local 10.124.2.6:50960 10.0.18.132:9080 10.124.2.6:52930 - default",
@@ -417,7 +417,7 @@
                 "created": "2020-04-28T11:07:58.223Z",
                 "duration": 34000000,
                 "id": "00bcf1d7-12c4-93b9-b36a-bc0faac8e428",
-                "ingested": "2022-09-09T09:23:51.149059051Z",
+                "ingested": "2023-02-10T18:34:31.225598628Z",
                 "kind": "event",
                 "module": "istio",
                 "original": "[2022-08-22T13:08:23.169Z] \"GET /productpage HTTP/1.1\" 200 - via_upstream - \"-\" 0 5294 35 34 \"10.128.0.161\" \"curl/7.79.1\" \"00bcf1d7-12c4-93b9-b36a-bc0faac8e428\" \"35.222.123.213\" \"10.124.2.6:9080\" inbound|9080|| 127.0.0.6:47203 10.124.2.6:9080 10.128.0.161:0 outbound_.9080_._.productpage.default.svc.cluster.local default",
@@ -514,7 +514,7 @@
                 "created": "2020-04-28T11:07:58.223Z",
                 "duration": 3000000,
                 "id": "d3bbd98a-ab12-9e96-b539-a419c2c8091f",
-                "ingested": "2022-09-09T09:23:51.149060093Z",
+                "ingested": "2023-02-10T18:34:31.225599837Z",
                 "kind": "event",
                 "module": "istio",
                 "original": "[2022-08-22T13:08:23.448Z] \"GET /details/0 HTTP/1.1\" 200 - via_upstream - \"-\" 0 178 3 3 \"-\" \"curl/7.79.1\" \"d3bbd98a-ab12-9e96-b539-a419c2c8091f\" \"details:9080\" \"10.124.0.10:9080\" outbound|9080||details.default.svc.cluster.local 10.124.2.6:58408 10.0.16.224:9080 10.124.2.6:49098 - default",
@@ -611,7 +611,7 @@
                 "created": "2020-04-28T11:07:58.223Z",
                 "duration": 0,
                 "id": "72f12c1b-8a44-9a62-b28e-2296da5b1118",
-                "ingested": "2022-09-09T09:23:51.149061093Z",
+                "ingested": "2023-02-10T18:34:31.225601087Z",
                 "kind": "event",
                 "module": "istio",
                 "original": "[2022-08-22T13:20:22.460Z] \"GET /ratings/0 HTTP/1.1\" 200 - via_upstream - \"-\" 0 48 1 0 \"-\" \"curl/7.79.1\" \"72f12c1b-8a44-9a62-b28e-2296da5b1118\" \"ratings:9080\" \"10.124.0.11:9080\" inbound|9080|| 127.0.0.6:38951 10.124.0.11:9080 10.124.0.12:58774 outbound_.9080_._.ratings.default.svc.cluster.local default",
@@ -709,7 +709,7 @@
                 "created": "2020-04-28T11:07:58.223Z",
                 "duration": 0,
                 "id": "72f12c1b-8a44-9a62-b28e-2296da5b1118",
-                "ingested": "2022-09-09T09:23:51.149062135Z",
+                "ingested": "2023-02-10T18:34:31.225602337Z",
                 "kind": "event",
                 "module": "istio",
                 "original": "[2022-08-22T13:20:22.460Z] \"GET /ratings/0 HTTP/1.1\" 200 - via_upstream - \"-\" 0 48 1 0 \"-\" \"curl/7.79.1\" \"72f12c1b-8a44-9a62-b28e-2296da5b1118\" \"ratings:9080\" \"[2a02:cf40::7]:3000\" inbound|9080|| 127.0.0.6:38951 10.124.0.11:9080 [2a02:cf40::4e36]:5000 outbound_.9080_._.ratings.default.svc.cluster.local default",
@@ -816,7 +816,7 @@
                 "created": "2020-04-28T11:07:58.223Z",
                 "duration": 1000000,
                 "id": "785918d6-06b6-9312-bf77-6d9bd968dc21",
-                "ingested": "2022-09-09T09:23:51.149063135Z",
+                "ingested": "2023-02-10T18:34:31.225603545Z",
                 "kind": "event",
                 "module": "istio",
                 "original": "[2022-07-20T09:52:24.955Z] \"GET /details/0 HTTP/1.1\" 200 - via_upstream - \"-\" 0 178 2 1 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.114 Safari/537.36\" \"785918d6-06b6-9312-bf77-6d9bd968dc21\" \"details:9080\" \"10.68.2.10:9080\" inbound|9080|| 127.0.0.6:47889 10.68.2.10:9080 89.160.20.156:39696 outbound_.9080_._.details.default.svc.cluster.local default",

diff --git a/packages/istio/data_stream/istiod_metrics/elasticsearch/ingest_pipeline/default.yml b/packages/istio/data_stream/istiod_metrics/elasticsearch/ingest_pipeline/default.yml
@@ -9,7 +9,7 @@ processors:
       ignore_missing: true
   - set:
       field: ecs.version
-      value: '8.4.0'
+      value: '8.6.0'
   - set:
       field: event.module
       value: istio

diff --git a/packages/istio/data_stream/istiod_metrics/sample_event.json b/packages/istio/data_stream/istiod_metrics/sample_event.json
@@ -34,7 +34,7 @@
     },
     "@timestamp": "2022-09-23T09:30:56.055Z",
     "ecs": {
-        "version": "8.4.0"
+        "version": "8.6.0"
     },
     "data_stream": {
         "namespace": "default",

diff --git a/packages/istio/data_stream/proxy_metrics/elasticsearch/ingest_pipeline/default.yml b/packages/istio/data_stream/proxy_metrics/elasticsearch/ingest_pipeline/default.yml
@@ -9,7 +9,7 @@ processors:
       ignore_missing: true
   - set:
       field: ecs.version
-      value: '8.4.0'
+      value: '8.6.0'
   - set:
       field: event.module
       value: istio

diff --git a/packages/istio/data_stream/proxy_metrics/sample_event.json b/packages/istio/data_stream/proxy_metrics/sample_event.json
@@ -6,7 +6,7 @@
         "type": "metrics"
     },
     "ecs": {
-        "version": "8.4.0"
+        "version": "8.6.0"
     },
     "event": {
         "agent_id_status": "verified",

diff --git a/packages/istio/docs/README.md b/packages/istio/docs/README.md
@@ -279,7 +279,7 @@ An example event for `istiod` looks as following:
     },
     "@timestamp": "2022-09-23T09:30:56.055Z",
     "ecs": {
-        "version": "8.4.0"
+        "version": "8.6.0"
     },
     "data_stream": {
         "namespace": "default",
@@ -336,7 +336,7 @@ An example event for `proxy` looks as following:
         "type": "metrics"
     },
     "ecs": {
-        "version": "8.4.0"
+        "version": "8.6.0"
     },
     "event": {
         "agent_id_status": "verified",