added geoip ingest node pipeline

README.md

@@ -128,6 +128,7 @@ becomes
             "es_api_key" : "APIKEY",
             "es_scheme" : "http",
             "index_template" : "es_stuff/index-template.json",
+            "ingest_node_template" : "es_stuff/evtxnom_pipeline.json",
             "ecs_map_file" : "es_stuff/ecs_map.json",
             "ecs_mode" : true,
             "delete_old_indexes" : false
@@ -147,6 +148,7 @@ becomes
 | es_api_key | string | base64 encoded api key (for api auth) |
 | es_scheme| string | http or https (for security you will be using https) |
 | index_template | string | path to index template, ive included one under es_stuff/index-template.json, You do not need to edit this for a custom index name as it will be done by the plugin |
+| ingest_node_template | string | path to ingest node json file, default one is provided, set to false if you dont want to use this , but it is currently used for geoip |
 | ecs_map_file | string | path to ecs map |
 | ecs_mode | string | if set to false no ecs mapping is done, the logs are still ecs structured ie under winlog.* just no processing ) |
 

config.json

@@ -26,6 +26,7 @@
             "es_api_key" : "APIKEY",
             "es_scheme" : "http",
             "index_template" : "es_stuff/index-template.json",
+            "ingest_node_template" : "es_stuff/evtxnom_pipeline.json",
             "ecs_map_file" : "es_stuff/ecs_map.json",
             "ecs_mode" : true,
             "delete_old_indexes" : true

es_stuff/ecs_map.json

@@ -8,7 +8,8 @@
                 "event.type" : "start",
                 "user.name" : "%%%%winlog.event_data.targetusername",
                 "user.id" : "%%%%winlog.event_data.targetusersid",
-                "user.domain" : "%%%%winlog.event_data.targetdomainname"
+                "user.domain" : "%%%%winlog.event_data.targetdomainname",
+                "source.ip" : "%%%%winlog.event_data.ipaddress"
             },
             "4672" : {
                 "event.kind" : "event",

es_stuff/evtxnom_pipeline.json

@@ -0,0 +1,81 @@
+{
+    "description" : "EVTXNOM Elasticsearch Ingest Pipeline",
+    "processors" : [
+        {
+            "remove": {
+              "if" : "ctx.source?.ip == '-'",
+              "field": "source.ip",
+              "ignore_missing": true
+            }
+        },
+        {
+            "geoip" : {
+              "if" : "ctx.source?.geo == null",
+              "field" : "source.ip",
+              "target_field" : "source.geo",
+              "ignore_missing" : true
+            }
+          },
+          {
+            "geoip" : {
+              "if" : "ctx.destination?.geo == null",
+              "field" : "destination.ip",
+              "target_field" : "destination.geo",
+              "ignore_missing" : true
+            }
+          },
+          {
+            "geoip" : {
+              "database_file" : "GeoLite2-ASN.mmdb",
+              "field" : "source.ip",
+              "target_field" : "source.as",
+              "properties" : [
+                "asn",
+                "organization_name"
+              ],
+              "ignore_missing" : true
+            }
+          },
+          {
+            "geoip" : {
+              "field" : "destination.ip",
+              "target_field" : "destination.as",
+              "properties" : [
+                "asn",
+                "organization_name"
+              ],
+              "ignore_missing" : true,
+              "database_file" : "GeoLite2-ASN.mmdb"
+            }
+          },
+          {
+            "rename" : {
+              "field" : "source.as.asn",
+              "target_field" : "source.as.number",
+              "ignore_missing" : true
+            }
+          },
+          {
+            "rename" : {
+              "target_field" : "source.as.organization.name",
+              "ignore_missing" : true,
+              "field" : "source.as.organization_name"
+            }
+          },
+          {
+            "rename" : {
+              "target_field" : "destination.as.number",
+              "ignore_missing" : true,
+              "field" : "destination.as.asn"
+            }
+          },
+          {
+            "rename" : {
+              "field" : "destination.as.organization_name",
+              "target_field" : "destination.as.organization.name",
+              "ignore_missing" : true
+            }
+          }
+
+    ]
+  }

lib/nom.py

@@ -37,6 +37,7 @@ def __init__(self,config,parsing_config):
         self.welm_map = load_welm_map(parsing_config['welm']['mapping_file'])
         self.welm_mode = parsing_config['welm']['enabled']
         self.delete = config['delete_old_indexes']
+        self.pipeline = config['ingest_node_template']
         self.prep_es()
     def load_ecs(self,filename):
         with open(filename,'r') as in_file:
@@ -88,6 +89,11 @@ def prep_es(self):
         # if option is set, delete existing indexes, todo handle patterns
         if self.delete:
             es.indices.delete(index=self.es_index, ignore=[400, 404])
+        # put es ingest node pipeline
+        if self.pipeline:
+            with open(self.pipeline,"r") as t_file:
+                template = json.load(t_file)
+            es.ingest.put_pipeline('evtx_nom', template, params=None, headers=None)
         return es
     def ingest_file(self,filename):
         # Process 1 file ah ah ah
@@ -129,6 +135,8 @@ def prepare_actions(self,filename):
                 '_index': self.es_index,
                 '_source': self.process_ecs(source)
             }
+            if self.pipeline:
+                action['pipeline'] = 'evtx_nom'
             yield action
     def parse_date(self,datestring):
         # Parse Date to Python object ISO 8601/ RFC3339