From 0d37a62e8bffc4bb9da4c52b486c31b4dac6d60e Mon Sep 17 00:00:00 2001 From: Miles Florence Date: Wed, 12 Feb 2020 03:37:19 -0800 Subject: [PATCH 1/7] Updated vendor lib to python3 --- analyzers/DNSDB/dnsdb_query.py | 200 +++++++++++++++++++++------------ 1 file changed, 129 insertions(+), 71 deletions(-) diff --git a/analyzers/DNSDB/dnsdb_query.py b/analyzers/DNSDB/dnsdb_query.py index 0fad57c99..57b38e254 100755 --- a/analyzers/DNSDB/dnsdb_query.py +++ b/analyzers/DNSDB/dnsdb_query.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2013 by Farsight Security, Inc. # @@ -14,8 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Python3 compatability by: https://github.com/guyddr/dnsdb-query + import calendar import errno +import json import locale import optparse import os @@ -23,30 +26,33 @@ import sys import time import urllib -import urllib2 -from cStringIO import StringIO +from io import StringIO -try: - import json -except ImportError: - import simplejson as json +import urllib3 -DEFAULT_CONFIG_FILE = '/etc/dnsdb-query.conf' +DEFAULT_CONFIG_FILES = filter(os.path.isfile, ('/etc/dnsdb-query.conf', os.path.expanduser('~/.dnsdb-query.conf'))) DEFAULT_DNSDB_SERVER = 'https://api.dnsdb.info' +DEFAULT_HTTP_PROXY = '' +DEFAULT_HTTPS_PROXY = '' cfg = None options = None +debug = False # set to True to print raw queries and responses locale.setlocale(locale.LC_ALL, '') + class QueryError(Exception): pass + class DnsdbClient(object): - def __init__(self, server, apikey, limit=None): + def __init__(self, server, apikey, limit=None, http_proxy=None, https_proxy=None): self.server = server self.apikey = apikey self.limit = limit + self.http_proxy = http_proxy + self.https_proxy = https_proxy def query_rrset(self, oname, rrtype=None, bailiwick=None, before=None, after=None): if bailiwick: @@ -71,6 +77,7 @@ def query_rdata_ip(self, rdata_ip, before=None, after=None): return self._query(path, before, after) def _query(self, path, before=None, after=None): + res = [] url = '%s/lookup/%s' % (self.server, path) params = {} @@ -85,57 +92,80 @@ def _query(self, path, before=None, after=None): if after: params['time_last_after'] = after if params: - url += '?{0}'.format(urllib.urlencode(params)) - - req = urllib2.Request(url) - req.add_header('Accept', 'application/json') - req.add_header('X-Api-Key', self.apikey) - http = urllib2.urlopen(req) - while True: - line = http.readline() - if not line: - break - yield json.loads(line) + url += '?{0}'.format(urllib.parse.urlencode(params)) + + if self.https_proxy: + manager = urllib3.ProxyManager(self.https_proxy) + elif self.http_proxy: + manager = urllib3.ProxyManager(self.http_proxy) + else: + manager = urllib3.PoolManager() + + headers = { + 'Accept': 'application/json', + 'X-Api-Key': self.apikey + } + + if debug: + sys.stderr.write(";; query URL =" + url) + + try: + r = manager.request(method='GET', url=url, headers=headers) + + json_data = r.data.decode('utf-8') + if json_data: + json_list = json_data.splitlines() + for line in json_list: + yield json.loads(line) + + except (urllib3.exceptions.HTTPError) as e: + raise QueryError(str(e)) + def quote(path): - return urllib.quote(path, safe='') + return urllib.parse.quote(path, safe='') + def sec_to_text(ts): return time.strftime('%Y-%m-%d %H:%M:%S -0000', time.gmtime(ts)) + def rrset_to_text(m): s = StringIO() - if 'bailiwick' in m: - s.write(';; bailiwick: %s\n' % m['bailiwick']) + try: + if 'bailiwick' in m: + s.write(';; bailiwick: %s\n' % m['bailiwick']) + + if 'count' in m: + s.write(';; count: %s\n' % locale.format('%d', m['count'], True)) - if 'count' in m: - s.write(';; count: %s\n' % locale.format('%d', m['count'], True)) + if 'time_first' in m: + s.write(';; first seen: %s\n' % sec_to_text(m['time_first'])) + if 'time_last' in m: + s.write(';; last seen: %s\n' % sec_to_text(m['time_last'])) - if 'time_first' in m: - s.write(';; first seen: %s\n' % sec_to_text(m['time_first'])) - if 'time_last' in m: - s.write(';; last seen: %s\n' % sec_to_text(m['time_last'])) + if 'zone_time_first' in m: + s.write(';; first seen in zone file: %s\n' % sec_to_text(m['zone_time_first'])) + if 'zone_time_last' in m: + s.write(';; last seen in zone file: %s\n' % sec_to_text(m['zone_time_last'])) - if 'zone_time_first' in m: - s.write(';; first seen in zone file: %s\n' % sec_to_text(m['zone_time_first'])) - if 'zone_time_last' in m: - s.write(';; last seen in zone file: %s\n' % sec_to_text(m['zone_time_last'])) + if 'rdata' in m: + for rdata in m['rdata']: + s.write('%s IN %s %s\n' % (m['rrname'], m['rrtype'], rdata)) - if 'rdata' in m: - for rdata in m['rdata']: - s.write('%s IN %s %s\n' % (m['rrname'], m['rrtype'], rdata)) + s.seek(0) + return s.read() + finally: + s.close() - s.seek(0) - return s.read() def rdata_to_text(m): return '%s IN %s %s' % (m['rrname'], m['rrtype'], m['rdata']) -def parse_config(cfg_fname): + +def parse_config(cfg_files): config = {} - cfg_files = filter(os.path.isfile, - (cfg_fname, os.path.expanduser('~/.dnsdb-query.conf'))) if not cfg_files: raise IOError(errno.ENOENT, 'dnsdb_query: No config files found') @@ -148,6 +178,7 @@ def parse_config(cfg_fname): return config + def time_parse(s): try: epoch = int(s) @@ -169,38 +200,56 @@ def time_parse(s): m = re.match(r'^(?=\d)(?:(\d+)w)?(?:(\d+)d)?(?:(\d+)h)?(?:(\d+)m)?(?:(\d+)s?)?$', s, re.I) if m: - return -1*(int(m.group(1) or 0)*604800 + \ - int(m.group(2) or 0)*86400+ \ - int(m.group(3) or 0)*3600+ \ - int(m.group(4) or 0)*60+ \ - int(m.group(5) or 0)) + return -1 * (int(m.group(1) or 0) * 604800 + \ + int(m.group(2) or 0) * 86400 + \ + int(m.group(3) or 0) * 3600 + \ + int(m.group(4) or 0) * 60 + \ + int(m.group(5) or 0)) raise ValueError('Invalid time: "%s"' % s) + +def epipe_wrapper(func): + def f(*args, **kwargs): + try: + return func(*args, **kwargs) + except IOError as e: + if e.errno == errno.EPIPE: + sys.exit(e.errno) + raise + + return f + + +@epipe_wrapper def main(): global cfg global options + global debug - parser = optparse.OptionParser(epilog='Time formats are: "%Y-%m-%d", "%Y-%m-%d %H:%M:%S", "%d" (UNIX timestamp), "-%d" (Relative time in seconds), BIND format (e.g. 1w1h, (w)eek, (d)ay, (h)our, (m)inute, (s)econd)') - parser.add_option('-c', '--config', dest='config', type='string', - help='config file', default=DEFAULT_CONFIG_FILE) + parser = optparse.OptionParser( + epilog='Time formats are: "%Y-%m-%d", "%Y-%m-%d %H:%M:%S", "%d" (UNIX timestamp), "-%d" (Relative time in seconds), BIND format (e.g. 1w1h, (w)eek, (d)ay, (h)our, (m)inute, (s)econd)') + parser.add_option('-c', '--config', dest='config', + help='config file', action='append') parser.add_option('-r', '--rrset', dest='rrset', type='string', - help='rrset [/[/BAILIWICK]]') + help='rrset [/[/BAILIWICK]]') parser.add_option('-n', '--rdataname', dest='rdata_name', type='string', - help='rdata name [/]') + help='rdata name [/]') parser.add_option('-i', '--rdataip', dest='rdata_ip', type='string', - help='rdata ip ') + help='rdata ip ') parser.add_option('-t', '--rrtype', dest='rrtype', type='string', - help='rrset or rdata rrtype') + help='rrset or rdata rrtype') parser.add_option('-b', '--bailiwick', dest='bailiwick', type='string', - help='rrset bailiwick') + help='rrset bailiwick') parser.add_option('-s', '--sort', dest='sort', type='string', help='sort key') parser.add_option('-R', '--reverse', dest='reverse', action='store_true', default=False, - help='reverse sort') + help='reverse sort') parser.add_option('-j', '--json', dest='json', action='store_true', default=False, - help='output in JSON format') + help='output in JSON format') parser.add_option('-l', '--limit', dest='limit', type='int', default=0, - help='limit number of results') + help='limit number of results') + parser.add_option('-d', '--debug', dest='debug', action='store_true', default=False, + help='print debug output') parser.add_option('', '--before', dest='before', type='string', help='only output results seen before this time') parser.add_option('', '--after', dest='after', type='string', help='only output results seen after this time') @@ -210,32 +259,40 @@ def main(): parser.print_help() sys.exit(1) + debug = options.debug + try: if options.before: options.before = time_parse(options.before) - except ValueError, e: - print 'Could not parse before: {}'.format(options.before) + except ValueError as e: + print('Could not parse before: {}'.format(options.before)) try: if options.after: options.after = time_parse(options.after) - except ValueError, e: - print 'Could not parse after: {}'.format(options.after) + except ValueError as e: + print('Could not parse after: {}'.format(options.after)) try: - cfg = parse_config(options.config) - except IOError, e: - sys.stderr.write(e.message) + cfg = parse_config(options.config or DEFAULT_CONFIG_FILES) + except IOError as e: + sys.stderr.writable(str(e)) sys.exit(1) - if not 'DNSDB_SERVER' in cfg: cfg['DNSDB_SERVER'] = DEFAULT_DNSDB_SERVER + if not 'HTTP_PROXY' in cfg: + cfg['HTTP_PROXY'] = DEFAULT_HTTP_PROXY + if not 'HTTPS_PROXY' in cfg: + cfg['HTTPS_PROXY'] = DEFAULT_HTTPS_PROXY if not 'APIKEY' in cfg: sys.stderr.write('dnsdb_query: APIKEY not defined in config file\n') sys.exit(1) - client = DnsdbClient(cfg['DNSDB_SERVER'], cfg['APIKEY'], options.limit) + client = DnsdbClient(cfg['DNSDB_SERVER'], cfg['APIKEY'], + limit=options.limit, + http_proxy=cfg['HTTP_PROXY'], + https_proxy=cfg['HTTPS_PROXY']) if options.rrset: if options.rrtype or options.bailiwick: qargs = (options.rrset, options.rrtype, options.bailiwick) @@ -246,7 +303,7 @@ def main(): fmt_func = rrset_to_text elif options.rdata_name: if options.rrtype: - qargs = (options.rdata_name, options.rrtype, options.bailiwick) + qargs = (options.rdata_name, options.rrtype) else: qargs = (options.rdata_name.split('/', 1)) @@ -269,14 +326,15 @@ def main(): if not options.sort in results[0]: sort_keys = results[0].keys() sort_keys.sort() - sys.stderr.write('dnsdb_query: invalid sort key "%s". valid sort keys are %s\n' % (options.sort, ', '.join(sort_keys))) + sys.stderr.write('dnsdb_query: invalid sort key "%s". valid sort keys are %s\n' % ( + options.sort, ', '.join(sort_keys))) sys.exit(1) results.sort(key=lambda r: r[options.sort], reverse=options.reverse) for res in results: sys.stdout.write('%s\n' % fmt_func(res)) - except (urllib2.HTTPError, urllib2.URLError), e: - print >>sys.stderr, str(e) + except QueryError as e: + sys.stderr.write(e) sys.exit(1) if __name__ == '__main__': - main() + main() \ No newline at end of file From c63353e619abea4e51bbaa098e3d556156540bc4 Mon Sep 17 00:00:00 2001 From: Miles Florence Date: Wed, 12 Feb 2020 03:38:00 -0800 Subject: [PATCH 2/7] Minor fix to address different error handling in py3 --- analyzers/DNSDB/dnsdb.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/analyzers/DNSDB/dnsdb.py b/analyzers/DNSDB/dnsdb.py index 19af57c97..689e9f71e 100755 --- a/analyzers/DNSDB/dnsdb.py +++ b/analyzers/DNSDB/dnsdb.py @@ -1,7 +1,7 @@ -#!/usr/bin/env python2 -# encoding: utf-8 +#!/usr/bin/env python3 + import datetime -from urllib2 import HTTPError +from urllib3.exceptions import HTTPError from dnsdb_query import DnsdbClient, QueryError from cortexutils.analyzer import Analyzer @@ -65,12 +65,10 @@ def run(self): "records": map(lambda r: self.update_date('time_first', self.update_date('time_last', r)), self.execute_dnsdb_service(client)) }) - except HTTPError, e: - if e.code != 404: - self.unexpectedError(e) - else: - self.report({"records": []}) + except Exception as e: + self.unexpectedError(e) + self.report({"records": []}) if __name__ == '__main__': - DnsDbAnalyzer().run() + DnsDbAnalyzer().run() \ No newline at end of file From acbc28540dda9554538d732498f01ffff2412f6b Mon Sep 17 00:00:00 2001 From: Miles Florence Date: Wed, 12 Feb 2020 03:39:34 -0800 Subject: [PATCH 3/7] JSON and datetime were moved to stdlib --- analyzers/DNSDB/requirements.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/analyzers/DNSDB/requirements.txt b/analyzers/DNSDB/requirements.txt index 43fe0b391..d71f30df0 100644 --- a/analyzers/DNSDB/requirements.txt +++ b/analyzers/DNSDB/requirements.txt @@ -1,4 +1,2 @@ -datetime -simplejson cortexutils future \ No newline at end of file From ed78adad83d9f7b5818d2746cc164fe0103cc097 Mon Sep 17 00:00:00 2001 From: Miles Florence Date: Wed, 12 Feb 2020 03:40:04 -0800 Subject: [PATCH 4/7] future lib was a bandage left over from python2 --- analyzers/DNSDB/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/analyzers/DNSDB/requirements.txt b/analyzers/DNSDB/requirements.txt index d71f30df0..8ad52a568 100644 --- a/analyzers/DNSDB/requirements.txt +++ b/analyzers/DNSDB/requirements.txt @@ -1,2 +1 @@ cortexutils -future \ No newline at end of file From 2de24a406edb49a1e05b634add9606538593f1f6 Mon Sep 17 00:00:00 2001 From: Miles Florence Date: Wed, 12 Feb 2020 03:40:47 -0800 Subject: [PATCH 5/7] Adding requests as it will be the log-term solution --- analyzers/DNSDB/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/analyzers/DNSDB/requirements.txt b/analyzers/DNSDB/requirements.txt index 8ad52a568..6aabc3cfa 100644 --- a/analyzers/DNSDB/requirements.txt +++ b/analyzers/DNSDB/requirements.txt @@ -1 +1,2 @@ cortexutils +requests From 0b7a7f96723ca9001ec7f891a96a3631bafef3e6 Mon Sep 17 00:00:00 2001 From: Miles Florence Date: Wed, 12 Feb 2020 03:41:10 -0800 Subject: [PATCH 6/7] vbump to python:3 --- analyzers/DNSDB/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analyzers/DNSDB/Dockerfile b/analyzers/DNSDB/Dockerfile index b178e3d01..b7db084a0 100644 --- a/analyzers/DNSDB/Dockerfile +++ b/analyzers/DNSDB/Dockerfile @@ -1,4 +1,4 @@ -FROM python:2 +FROM python:3 WORKDIR /worker COPY . DNSDB From e44489431087ce69655ddcae04d06afb235f33bf Mon Sep 17 00:00:00 2001 From: Davide Arcuri Date: Mon, 13 Apr 2020 15:13:37 +0200 Subject: [PATCH 7/7] fix map object vs json --- analyzers/DNSDB/dnsdb.py | 4 ++-- analyzers/DNSDB/dnsdb_query.py | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/analyzers/DNSDB/dnsdb.py b/analyzers/DNSDB/dnsdb.py index 689e9f71e..8d13e20f4 100755 --- a/analyzers/DNSDB/dnsdb.py +++ b/analyzers/DNSDB/dnsdb.py @@ -62,8 +62,8 @@ def run(self): try: client = DnsdbClient(self.dnsdb_server, self.dnsdb_key) self.report({ - "records": map(lambda r: self.update_date('time_first', self.update_date('time_last', r)), - self.execute_dnsdb_service(client)) + "records": list(map(lambda r: self.update_date('time_first', self.update_date('time_last', r)), + self.execute_dnsdb_service(client))) }) except Exception as e: self.unexpectedError(e) diff --git a/analyzers/DNSDB/dnsdb_query.py b/analyzers/DNSDB/dnsdb_query.py index 57b38e254..a055ea0eb 100755 --- a/analyzers/DNSDB/dnsdb_query.py +++ b/analyzers/DNSDB/dnsdb_query.py @@ -111,12 +111,14 @@ def _query(self, path, before=None, after=None): try: r = manager.request(method='GET', url=url, headers=headers) - - json_data = r.data.decode('utf-8') - if json_data: - json_list = json_data.splitlines() - for line in json_list: - yield json.loads(line) + if r.status == 200: + json_data = r.data.decode('utf-8') + if json_data: + json_list = json_data.splitlines() + for line in json_list: + yield json.loads(line) + else: + raise QueryError(r.text) except (urllib3.exceptions.HTTPError) as e: raise QueryError(str(e))