Fix Grok Pattern in Istio Access Log Pipeline (#5316)

* Resolved 5218 * Fix IPv6 addresses in tests * Fix community id expected
elastic · Feb 20, 2023 · 6e3956f · 6e3956f
1 parent 02df9e2
commit 6e3956f
Show file tree

Hide file tree

Showing 5 changed files with 106 additions and 3 deletions.
diff --git a/packages/istio/changelog.yml b/packages/istio/changelog.yml
@@ -1,4 +1,8 @@
-# newer versions go on top
+- version: "0.2.3"
+  changes:
+    - description: Fix Access Log Common Format Ingest Grok Pattern for IPv6
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/5316
 - version: "0.2.2"
   changes:
     - description: Monitor Istiod service

diff --git a/packages/istio/data_stream/access_logs/_dev/test/pipeline/test-access-logs.log b/packages/istio/data_stream/access_logs/_dev/test/pipeline/test-access-logs.log
@@ -7,3 +7,4 @@
 [2022-08-22T13:20:22.460Z] "GET /ratings/0 HTTP/1.1" 200 - via_upstream - "-" 0 48 1 0 "-" "curl/7.79.1" "72f12c1b-8a44-9a62-b28e-2296da5b1118" "ratings:9080" "10.124.0.11:9080" inbound|9080|| 127.0.0.6:38951 10.124.0.11:9080 10.124.0.12:58774 outbound_.9080_._.ratings.default.svc.cluster.local default
 [2022-08-22T13:20:22.460Z] "GET /ratings/0 HTTP/1.1" 200 - via_upstream - "-" 0 48 1 0 "-" "curl/7.79.1" "72f12c1b-8a44-9a62-b28e-2296da5b1118" "ratings:9080" "[2a02:cf40::7]:3000" inbound|9080|| 127.0.0.6:38951 10.124.0.11:9080 [2a02:cf40::4e36]:5000 outbound_.9080_._.ratings.default.svc.cluster.local default
 [2022-07-20T09:52:24.955Z] "GET /details/0 HTTP/1.1" 200 - via_upstream - "-" 0 178 2 1 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.114 Safari/537.36" "785918d6-06b6-9312-bf77-6d9bd968dc21" "details:9080" "10.68.2.10:9080" inbound|9080|| 127.0.0.6:47889 10.68.2.10:9080 89.160.20.156:39696 outbound_.9080_._.details.default.svc.cluster.local default
+[2023-02-08T22:00:26.503Z] "GET / HTTP/1.1" 200 - via_upstream - "-" 0 568 1 0 "192.168.1.1" "curl/7.79.1" "f825485a-6c59-4eaf-b712-9a172662b355" "elastic.domain.com:9200" "[2a02:cf40::4e36]:9200" outbound|9200||elasticsearch-es-http.elastic-dev.svc.cluster.local [2a02:cf40::7]:33224 10.20.20.236:9200 192.168.1.1:60754 elastic.domain.com -
diff --git a/packages/istio/data_stream/access_logs/_dev/test/pipeline/test-access-logs.log-expected.json b/packages/istio/data_stream/access_logs/_dev/test/pipeline/test-access-logs.log-expected.json
@@ -919,6 +919,104 @@
                 },
                 "version": "103.0.5060.114"
             }
+        },
+        {
+            "@timestamp": "2023-02-08T22:00:26.503Z",
+            "destination": {
+                "address": "[2a02:cf40::4e36]:9200",
+                "ip": "2a02:cf40::4e36",
+                "port": 9200
+            },
+            "ecs": {
+                "version": "8.3.0"
+            },
+            "event": {
+                "category": [
+                    "web"
+                ],
+                "created": "2020-04-28T11:07:58.223Z",
+                "duration": 0,
+                "id": "f825485a-6c59-4eaf-b712-9a172662b355",
+                "ingested": "2022-09-09T09:23:51.149061093Z",
+                "kind": "event",
+                "module": "istio",
+                "original": "[2023-02-08T22:00:26.503Z] \"GET / HTTP/1.1\" 200 - via_upstream - \"-\" 0 568 1 0 \"192.168.1.1\" \"curl/7.79.1\" \"f825485a-6c59-4eaf-b712-9a172662b355\" \"elastic.domain.com:9200\" \"[2a02:cf40::4e36]:9200\" outbound|9200||elasticsearch-es-http.elastic-dev.svc.cluster.local [2a02:cf40::7]:33224 10.20.20.236:9200 192.168.1.1:60754 elastic.domain.com -",
+                "outcome": "success",
+                "type": [
+                    "access"
+                ]
+            },
+            "http": {
+                "request": {
+                    "body": {
+                        "bytes": 568
+                    },
+                    "id": "f825485a-6c59-4eaf-b712-9a172662b355",
+                    "method": "GET"
+                },
+                "response": {
+                    "body": {
+                        "bytes": 0
+                    },
+                    "status_code": 200
+                },
+                "version": "1.1"
+            },
+            "istio": {
+                "access": {
+                    "authority": "elastic.domain.com:9200",
+                    "bytes": {
+                        "received": 0,
+                        "sent": 568
+                    },
+                    "downstream": {
+                        "local_address": "10.20.20.236:9200",
+                        "remote_address": "192.168.1.1:60754"
+                    },
+                    "duration": 1,
+                    "requested_server_name": "elastic.domain.com",
+                    "response": {
+                        "code_details": "via_upstream"
+                    },
+                    "upstream": {
+                        "cluster": "outbound|9200||elasticsearch-es-http.elastic-dev.svc.cluster.local",
+                        "host": "[2a02:cf40::4e36]:9200",
+                        "local_address": "[2a02:cf40::7]:33224",
+                        "service_time": 0
+                    },
+                    "x_forwarded_for": "192.168.1.1"
+                }
+            },
+            "network": {
+                "community_id": "1:CJwABDSOfTg+6pEYPICwnKbcl/M=",
+                "protocol": "http",
+                "transport": "tcp"
+            },
+            "related": {
+                "ip": [
+                    "192.168.1.1",
+                    "2a02:cf40::4e36"
+                ]
+            },
+            "source": {
+                "address": "192.168.1.1:60754",
+                "ip": "192.168.1.1",
+                "port": 60754
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "url": {
+                "original": "/"
+            },
+            "user_agent": {
+                "device": {
+                    "name": "Other"
+                },
+                "name": "curl",
+                "original": "curl/7.79.1",
+                "version": "7.79.1"
+            }
         }
     ]
 }
diff --git a/packages/istio/data_stream/access_logs/elasticsearch/ingest_pipeline/format-common.yml b/packages/istio/data_stream/access_logs/elasticsearch/ingest_pipeline/format-common.yml
@@ -9,7 +9,7 @@ processors:
   - grok:
       field: istio.access.message
       patterns:
-        - '"(-|%{DATA:http.request.method}) (-|%{DATA:url.original}) (-|%{DATA:istio.access.protocol})" (-|%{NUMBER:http.response.status_code}) (-|%{DATA:istio.access.response.flags}) (-|%{DATA:istio.access.response.code_details}) (-|%{DATA:istio.access.connection_termination_details}) "(-|%{DATA:istio.access.upstream.transport_failure_reason})" %{NUMBER:istio.access.bytes.received} %{NUMBER:istio.access.bytes.sent} (-|%{NUMBER:istio.access.duration}) (-|%{NUMBER:istio.access.upstream.service_time}) "(-|%{DATA:istio.access.x_forwarded_for})" "(-|%{DATA:user_agent.original})" "(-|%{DATA:http.request.id})" "(-|%{DATA:istio.access.authority})" "(-|%{DATA:istio.access.upstream.host})" (-|%{DATA:istio.access.upstream.cluster}) (-|%{HOSTPORT:istio.access.upstream.local_address}) (-|%{HOSTPORT:istio.access.downstream.local_address}) (-|%{DATA:istio.access.downstream.remote_address}) (-|%{DATA:istio.access.requested_server_name}) (-|%{GREEDYDATA:istio.access.route_name})'
+        - '"(-|%{DATA:http.request.method}) (-|%{DATA:url.original}) (-|%{DATA:istio.access.protocol})" (-|%{NUMBER:http.response.status_code}) (-|%{DATA:istio.access.response.flags}) (-|%{DATA:istio.access.response.code_details}) (-|%{DATA:istio.access.connection_termination_details}) "(-|%{DATA:istio.access.upstream.transport_failure_reason})" %{NUMBER:istio.access.bytes.received} %{NUMBER:istio.access.bytes.sent} (-|%{NUMBER:istio.access.duration}) (-|%{NUMBER:istio.access.upstream.service_time}) "(-|%{DATA:istio.access.x_forwarded_for})" "(-|%{DATA:user_agent.original})" "(-|%{DATA:http.request.id})" "(-|%{DATA:istio.access.authority})" "(-|%{DATA:istio.access.upstream.host})" (-|%{DATA:istio.access.upstream.cluster}) (-|%{DATA:istio.access.upstream.local_address}) (-|%{DATA:istio.access.downstream.local_address}) (-|%{DATA:istio.access.downstream.remote_address}) (-|%{DATA:istio.access.requested_server_name}) (-|%{GREEDYDATA:istio.access.route_name})'
       ignore_missing: true
   - remove:
       field: istio.access.message

diff --git a/packages/istio/manifest.yml b/packages/istio/manifest.yml
@@ -3,7 +3,7 @@ name: istio
 title: Istio
 description: Collect logs and metrics from the service mesh Istio with Elastic Agent.
 type: integration
-version: 0.2.2
+version: 0.2.3
 release: beta
 license: basic
 categories: