diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..397b4a7
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+*.log
diff --git a/loganon b/loganon
index 34d7eff..7a5bef2 100755
--- a/loganon
+++ b/loganon
@@ -1,19 +1,19 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# copyright sys4 AG 2015
# This file is part of loganon.
-#
+#
# loganon is free software: you can redistribute it and/or modify it under the
# terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
-#
+#
# loganon is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
-#
+#
# You should have received a copy of the GNU Lesser General Public License
# along with loganon. If not, see .
@@ -21,10 +21,16 @@ import os
import sys
import re
import yaml
+import magic
+import io
from getopt import getopt
from netaddr import IPAddress, IPNetwork
+from ipaddress import ip_address
from netaddr.core import AddrFormatError
+import hashlib
+#import zlib
+#from random import randint
try:
from collections import OrderedDict
@@ -46,7 +52,7 @@ def ordered_load(stream, Loader=yaml.Loader, object_pairs_hook=OrderedDict):
def usage():
"""Print a simple usage to stdout
"""
- print """%s [options]
+ print("""%s [options]
-h, --help prints out this help
-i, --input=file log file to read
@@ -59,7 +65,7 @@ Optional:
-6, --mask6=number number of bits to mask an IPv6 address
-t, --test test pattern and print output to stdout
- """ % os.path.basename(__file__)
+ """ % os.path.basename(__file__))
def main():
"""Main application
@@ -92,6 +98,13 @@ def main():
ipv4 = re.compile("[1-9][0-9]{0,2}\.[0-9.]{3,7}\.[0-9]{1,3}")
ipv6 = re.compile("([1-9a-fA-F][0-9a-fA-F]{3}):"
"[0-9a-fA-F:]{2,29}[0-9a-fA-F]{1,4}")
+ domain = re.compile("([^\s=\"\(\):]*\.)?[a-zA-Z0-9][a-zA-Z0-9-]{1,61}[a-zA-Z0-9]\.[a-zA-Z]{2,}")
+ syslog_prio = re.compile("(auth|cron|daemon|kern|local[0-7]|lpr|mail|news|user|uucp)\.(info|notice|warning|err|alert|warn|debug|emerg|crit)", re.IGNORECASE)
+
+ def get_encoding(file):
+ blob = open(file, "rb").read()
+ m = magic.Magic(mime_encoding=True)
+ return m.from_buffer(blob)
# Read command line options
try:
@@ -135,10 +148,10 @@ def main():
usage()
sys.exit(os.EX_USAGE)
- except Exception, e:
+ except Exception as e:
print >> sys.stderr, "Syntax error: %s" % e
sys.exit(os.EX_USAGE)
-
+
# Read all rules
try:
for rule in iter(rules):
@@ -147,22 +160,22 @@ def main():
yaml.SafeLoader,
OrderedDict))
- except IOError, e:
+ except IOError as e:
print >> sys.stderr, "IOError: %s" % e
sys.exit(os.EX_IOERR)
- except Exception, e:
+ except Exception as e:
print >> sys.stderr, "Unknown error: %s" % e
sys.exit(os.EX_USAGE)
-
+
# Build macro dictionary
for rule_entity in iter(rules_collection):
- for service, ruledef in rule_entity.iteritems():
- for rulename, rulepattern in ruledef.iteritems():
+ for service, ruledef in rule_entity.items():
+ for rulename, rulepattern in ruledef.items():
search = None
replace = None
for patterndef in iter(rulepattern):
- for actiondesc, actiondef in patterndef.iteritems():
+ for actiondesc, actiondef in patterndef.items():
if actiondesc == "search":
search = actiondef
if actiondesc == "replace":
@@ -175,26 +188,26 @@ def main():
sys.exit(os.EX_USAGE)
try:
rule_data[rulename] = (re.compile(search), replace)
- except Exception, e:
+ except Exception as e:
print >> sys.stderr, ("Syntax error in or "
" pattern: %s" % e)
sys.exit(os.EX_USAGE)
# Open input and output files
try:
- fd_in = open(fdinarg, "r")
+ fd_in = io.open(fdinarg, "r", encoding=get_encoding(fdinarg))
if not test:
fd_out = open(fdoutarg, "w")
- except IOError, e:
- print >> sys.stderr, "IOError: %s" % e
+ except IOError as e:
+ print("IOError: %s" % e, file=sys.stderr)
sys.exit(os.EX_IOERR)
- except Exception, e:
- print >> sys.stderr, "Unknown error: %s" % e
+ except Exception as e:
+ print("Unknown error: %s" % e, file=sys.stderr)
sys.exit(os.EX_USAGE)
-
- def reduce_ip(matchobj):
+
+ def maybe_ip(matchobj):
maybe_ip = False
# simple tests
@@ -217,8 +230,10 @@ def main():
test = matchobj.group(0).split(":")
if len(test) >= 2:
maybe_ip = True
+ return maybe_ip
- if maybe_ip:
+ def reduce_ip(matchobj):
+ if maybe_ip(matchobj):
try:
ip = IPAddress(matchobj.group(0))
except AddrFormatError:
@@ -235,25 +250,105 @@ def main():
else:
return matchobj.group(0)
+ def ips(start, end):
+ '''Return IPs in IPv4 range, inclusive.'''
+ start_int = int(ip_address(start).packed.hex(), 16)
+ end_int = int(ip_address(end).packed.hex(), 16)
+ return [ip_address(ip).exploded for ip in range(start_int, end_int)]
+
+ iprepo_global = ips('1.2.0.1', '1.2.20.254')
+ iprepo_private = ips('10.10.0.1', '10.10.20.254')
+ ip_map = {}
+
+ def map_ip(matchobj):
+ if maybe_ip(matchobj):
+ try:
+ ip = IPAddress(matchobj.group(0))
+ except AddrFormatError:
+ # might be something else than an IPv6 address
+ return matchobj.group(0)
+
+ if ip.version == 4:
+ if not str(ip) in ip_map:
+ if ip.is_private():
+ ip_map[str(ip)] = iprepo_private.pop(0)
+ else:
+ ip_map[str(ip)] = iprepo_global.pop(0)
+ return ip_map[str(ip)]
+ #return str(bitmask4 & ip)
+ else:
+ return str(bitmask6 & ip)
+
+ return str(ip)
+
+ else:
+ return matchobj.group(0)
+
+ def string_hash(_str):
+ _hash = list(hashlib.md5(_str.encode()).hexdigest())
+ ret = ''
+ for i in range(0,len(_hash),2):
+ ret += map_char(_hash[i] + _hash[i+1])
+ return ret
+
+ domain_map = {}
+
+ def map_domain(matchobj):
+ _domain = matchobj.group(0)
+ if re.match(syslog_prio, _domain) is not None:
+ return _domain
+ if not _domain in domain_map:
+ parts = _domain.split('.')
+ for idx in range(len(parts)):
+ parts[idx] = string_hash(parts[idx])
+ domain_map[_domain] = '.'.join(parts)
+ #print(_domain + '=' + domain_map[_domain])
+ return domain_map[_domain]
+
+ def map_char(hex):
+ ic = int(hex, 16)
+ offset = int(ic / 255 * 50)
+ if offset <= 25:
+ return chr(65 + offset)
+ else:
+ return chr(72 + offset)
+
+ string_map = {}
+
+ def map_string(_str):
+ if not _str in string_map:
+ string_map[_str] = string_hash(_str)
+ return string_map[_str]
+
while True:
line = fd_in.readline()
if not line:
break
# Phase 1 - search and replace pattern
- for key, value in rule_data.iteritems():
+ for key, value in rule_data.items():
try:
- linenew = value[0].sub(value[1], line)
+ replace = value[1]
+ if '_MAP_' in replace:
+ found = value[0].search(line)
+ if found:
+ _str = found.group(1)
+ replace = replace.replace('_MAP_', map_string(_str))
+ linenew = value[0].sub(replace, line)
if linenew is not None:
line = linenew
- except Exception, e:
+ except Exception as e:
print >> sys.stderr, e
# Phase 2 - find IPv4/IPv6 address
- line = re.sub(ipv4, reduce_ip, line)
+ line = re.sub(ipv4, map_ip, line)
line = re.sub(ipv6, reduce_ip, line)
+
+ # Phase 3 - search and replace domains
+ line = re.sub(domain, map_domain, line)
+
if test:
- print line.strip()
+ print(line.strip())
else:
fd_out.write(line)
diff --git a/rules/exchange.rules b/rules/exchange.rules
new file mode 100644
index 0000000..a54d503
--- /dev/null
+++ b/rules/exchange.rules
@@ -0,0 +1,38 @@
+# copyright sys4 AG 2015
+
+# This file is part of loganon.
+#
+# loganon is free software: you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# loganon is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with loganon. If not, see .
+
+
+exchange:
+ sender-address:
+ - search: "sender-address:([^@]+)@"
+ - replace: "sender-address:_MAP_@"
+ return-path:
+ - search: "return-path:([^@]+)@"
+ - replace: "return-path:_MAP_@"
+ recipient-address:
+ - search: "recipient-address:([^@]+)@"
+ - replace: "recipient-address:_MAP_@"
+ UserID:
+ - search: "UserID:([^,]+)"
+ - replace: "UserID:_MAP_"
+ AccountName:
+ - search: "AccountName:([^,]+)"
+ - replace: "AccountName:_MAP_"
+ Domain:
+ - search: "Domain:([^,]+)"
+ - replace: "Domain:_MAP_"
+# vim: syn=yaml ts=2 sw=2 expandtab
diff --git a/rules/fortigate.rules b/rules/fortigate.rules
new file mode 100644
index 0000000..0c90a49
--- /dev/null
+++ b/rules/fortigate.rules
@@ -0,0 +1,44 @@
+# copyright sys4 AG 2015
+
+# This file is part of loganon.
+#
+# loganon is free software: you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# loganon is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with loganon. If not, see .
+
+fortigate:
+ devname:
+ - search: "devname=[^ ]+"
+ - replace: "devname=DEVNAME"
+ devid:
+ - search: "devid=[^ ]+"
+ - replace: "devid=DEVID"
+ #srcip:
+ # - search: "srcip=[^ ]+"
+ # - replace: "srcip=IP"
+ #dstip:
+ # - search: "dstip=[^ ]+"
+ # - replace: "dstip=IP"
+ srcintf:
+ - search: "srcintf=[^ ]+"
+ - replace: "srcintf=\"SRCINTF\""
+ dstintf:
+ - search: "dstintf=[^ ]+"
+ - replace: "dstintf=\"DSTINTF\""
+ srccountry:
+ - search: "srccountry=[^ ]+"
+ - replace: "srccountry=\"COUNTRY\""
+ dstcountry:
+ - search: "dstcountry=[^ ]+"
+ - replace: "dstcountry=\"COUNTRY\""
+
+# vim: syn=yaml ts=2 sw=2 expandtab
diff --git a/rules/infoblox.rules b/rules/infoblox.rules
new file mode 100644
index 0000000..a95c6c2
--- /dev/null
+++ b/rules/infoblox.rules
@@ -0,0 +1,44 @@
+# copyright sys4 AG 2015
+
+# This file is part of loganon.
+#
+# loganon is free software: you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# loganon is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with loganon. If not, see .
+
+fortigate:
+ devname:
+ - search: "devname=[^ ]+"
+ - replace: "devname=DEVNAME"
+ devid:
+ - search: "devid=[^ ]+"
+ - replace: "devid=DEVID"
+ #srcip:
+ # - search: "srcip=[^ ]+"
+ # - replace: "srcip=IP"
+ #dstip:
+ # - search: "dstip=[^ ]+"
+ # - replace: "dstip=IP"
+ srcintf:
+ - search: "srcintf=[^ ]+"
+ - replace: "srcintf=SRCINTF"
+ dstintf:
+ - search: "dstintf=[^ ]+"
+ - replace: "dstintf=DSTINTF"
+ srccountry:
+ - search: "srccountry=[^ ]+"
+ - replace: "srccountry=\"COUNTRY\""
+ dstcountry:
+ - search: "dstcountry=[^ ]+"
+ - replace: "dstcountry=\"COUNTRY\""
+
+# vim: syn=yaml ts=2 sw=2 expandtab
diff --git a/rules/webproxy.rules b/rules/webproxy.rules
new file mode 100644
index 0000000..65832cc
--- /dev/null
+++ b/rules/webproxy.rules
@@ -0,0 +1,23 @@
+# copyright sys4 AG 2015
+
+# This file is part of loganon.
+#
+# loganon is free software: you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# loganon is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with loganon. If not, see .
+
+webproxy:
+ user:
+ - search: "user=\"([^\"]+)\""
+ - replace: "user=\"_MAP_\""
+
+# vim: syn=yaml ts=2 sw=2 expandtab