Fix url scheme grok pattern

elastic · Jun 13, 2023 · bbf1378 · bbf1378
1 parent 4af3e60
commit bbf1378
Show file tree

Hide file tree

Showing 4 changed files with 61 additions and 150 deletions.
diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc
@@ -82,6 +82,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only.
 - Fix the ingest pipeline for mysql slowlog to parse schema name with dash {pull}34371[34372]
 - Fix the multiple host support for mongodb module {pull}34624[34624]
 - Skip HTTPJSON flakey test. {issue}34929[34929] {pull}35138[35138]
+- Fix ingest pipeline for panw module to parse url scheme correctly {pull}35757[35757]
 
 ==== Added
 

diff --git a/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml b/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml
@@ -437,7 +437,7 @@ processors:
   - grok:
       field: url.original
       patterns:
-        - '(%{ANY:url.scheme}\:\/\/)?(%{USERNAME:url.username}(\:%{PASSWORD:url.password})?\@)?%{DOMAIN:url.domain}(\:%{POSINT:url.port})?(%{PATH:url.path})?(\?%{QUERY:url.query})?(\#%{ANY:url.fragment})?'
+        - '(%{URIPROTO:url.scheme}\:\/\/)?(%{USERNAME:url.username}(\:%{PASSWORD:url.password})?\@)?%{DOMAIN:url.domain}(\:%{POSINT:url.port})?(%{PATH:url.path})?(\?%{QUERY:url.query})?(\#%{ANY:url.fragment})?'
       ignore_missing: true
       pattern_definitions:
         USERNAME: '[^\:]*'

diff --git a/x-pack/filebeat/module/panw/panos/test/pan_inc_threat.log b/x-pack/filebeat/module/panw/panos/test/pan_inc_threat.log
@@ -46,7 +46,7 @@ Oct 30 09:53:38 1,2012/10/30 09:53:38,01606001116,THREAT,url,1,2012/04/10 04:37:
 Oct 30 09:53:48 1,2012/10/30 09:53:48,01606001116,THREAT,url,1,2012/04/10 04:37:27,192.168.0.2,69.43.161.154,0.0.0.0,0.0.0.0,rule1,crusher,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,2012/04/10 04:37:27,23659,1,58847,80,0,0,0x200000,tcp,block-url,"tubemov.com/",(9999),adult-and-pornography,informational,client-to-server,0,0x0,192.168.0.0-192.168.255.255,United States,0,
 Oct 30 09:53:58 1,2012/10/30 09:53:58,01606001116,THREAT,url,1,2012/04/10 04:37:25,192.168.0.2,208.91.196.252,0.0.0.0,0.0.0.0,rule1,crusher,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,2012/04/10 04:37:25,23782,1,58841,80,0,0,0x200000,tcp,block-url,"pagesinxt.com/?dn=teenstube.us&flrdr=yes&nxte=js",(9999),malware-sites,informational,client-to-server,0,0x0,192.168.0.0-192.168.255.255,Virgin Islands British,0,
 Oct 30 09:55:23 1,2012/10/30 09:55:23,01606001116,THREAT,url,1,2012/04/10 04:37:05,192.168.0.2,208.73.210.29,0.0.0.0,0.0.0.0,rule1,crusher,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,2012/04/10 04:37:06,23239,1,58795,80,0,0,0x200000,tcp,block-url,"movfree.com/",(9999),spyware-and-adware,informational,client-to-server,0,0x0,192.168.0.0-192.168.255.255,United States,0,
-Oct 30 09:56:23 1,2012/10/30 09:56:23,01606001116,THREAT,url,1,2012/04/10 04:36:51,192.168.0.2,67.43.156.12,0.0.0.0,0.0.0.0,rule1,crusher,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,2012/04/10 04:36:52,22479,1,58753,80,0,0,0x200000,tcp,block-url,"gometascan.com/",(9999),malware-sites,informational,client-to-server,0,0x0,192.168.0.0-192.168.255.255,United States,0,
+Oct 30 09:56:23 1,2012/10/30 09:56:23,01606001116,THREAT,url,1,2012/04/10 04:36:51,192.168.0.2,67.43.156.12,0.0.0.0,0.0.0.0,rule1,crusher,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,2012/04/10 04:36:52,22479,1,58753,80,0,0,0x200000,tcp,block-url,"www.sportspar.de/widgets/index/refreshStatistic?requestPage=/&requestController=index&referer=https://www.google.com/",(9999),malware-sites,informational,client-to-server,0,0x0,192.168.0.0-192.168.255.255,United States,0,
 Oct 30 09:57:33 1,2012/10/30 09:57:33,01606001116,THREAT,url,1,2012/04/10 04:36:39,192.168.0.2,67.43.156.12,0.0.0.0,0.0.0.0,rule1,crusher,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,2012/04/10 04:36:40,21458,1,58708,80,0,0,0x200000,tcp,block-url,"antivirus-powerful-scannerv2.com/download/Install_11-1.exe",(9999),malware-sites,informational,client-to-server,0,0x0,192.168.0.0-192.168.255.255,United States,0,
 Oct 30 09:57:38 1,2012/10/30 09:57:38,01606001116,THREAT,url,1,2012/04/10 04:36:38,192.168.0.2,67.43.156.12,0.0.0.0,0.0.0.0,rule1,crusher,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,2012/04/10 04:36:39,21577,1,58707,80,0,0,0x200000,tcp,block-url,"antivirus-powerful-scannerv2.com/1/?id=11-1&back==TQzyDTyMUQNMI=N",(9999),malware-sites,informational,client-to-server,0,0x0,192.168.0.0-192.168.255.255,United States,0,
 Mar 25 23:58:52 1,2013/03/25 23:58:52,1606001116,THREAT,url,1,2012/04/10 04:36:27,192.168.0.2,67.43.156.12,0.0.0.0,0.0.0.0,rule1,crusher,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,2012/04/10 04:36:28,21487,1,58603,80,0,0,0x200000,tcp,block-url,"basdzsdas.com/poker/config.bin",(9999),malware-sites,informational,client-to-server,0,0x0,192.168.0.0-192.168.255.255,United States,0,
@@ -97,4 +97,4 @@ Mar 25 23:59:47 1,2013/03/25 23:59:47,01606001116,THREAT,data,1,2012/04/09 03:49
 Mar 25 23:59:47 1,2013/03/25 23:59:47,01606001116,THREAT,data,1,2012/04/09 03:53:41,81.2.69.143,192.168.0.2,0.0.0.0,0.0.0.0,rule1,,jordy,google-maps,vsys1,untrust,trust,ethernet1/2,ethernet1/1,forwardAll,2012/04/09 03:53:45,914,1,80,60822,0,0,0x200000,tcp,reset-both,"js",PII(60000),any,informational,server-to-client,0,0x0,United States,192.168.0.0-192.168.255.255,0,
 Mar 25 23:59:47 1,2013/03/25 23:59:47,01606001116,THREAT,data,1,2012/04/09 03:55:23,81.2.69.143,192.168.0.2,0.0.0.0,0.0.0.0,rule1,,jordy,google-maps,vsys1,untrust,trust,ethernet1/2,ethernet1/1,forwardAll,2012/04/09 03:55:28,1475,1,80,61105,0,0,0x200000,tcp,reset-both,"js",PII(60000),any,informational,server-to-client,0,0x0,United States,192.168.0.0-192.168.255.255,0,
 Mar 25 23:59:47 1,2013/03/25 23:59:47,01606001116,THREAT,data,1,2012/04/09 03:55:52,81.2.69.193,192.168.0.2,0.0.0.0,0.0.0.0,rule1,,jordy,google-analytics,vsys1,untrust,trust,ethernet1/2,ethernet1/1,forwardAll,2012/04/09 03:55:57,883,1,80,60782,0,0,0x200000,tcp,alert,"ga.js",PII(60000),any,informational,server-to-client,0,0x0,United States,192.168.0.0-192.168.255.255,0,
-Mar 25 23:59:47 1,2013/03/25 23:59:47,01606001116,THREAT,data,1,2012/04/09 04:03:55,81.2.69.143,192.168.0.2,0.0.0.0,0.0.0.0,rule1,,jordy,google-maps,vsys1,untrust,trust,ethernet1/2,ethernet1/1,forwardAll,2012/04/09 04:04:00,1965,1,80,61470,0,0,0x200000,tcp,reset-both,"js",PII(60000),any,informational,server-to-client,0,0x0,United States,192.168.0.0-192.168.255.255,0,
+Mar 25 23:59:47 1,2013/03/25 23:59:47,01606001116,THREAT,data,1,2012/04/09 04:03:55,81.2.69.143,192.168.0.2,0.0.0.0,0.0.0.0,rule1,,jordy,google-maps,vsys1,untrust,trust,ethernet1/2,ethernet1/1,forwardAll,2012/04/09 04:04:00,1965,1,80,61470,0,0,0x200000,tcp,reset-both,"js",PII(60000),any,informational,server-to-client,0,0x0,United States,192.168.0.0-192.168.255.255,0,