From 3d9fd6ce5b016b7f69a52a128504260499cafffb Mon Sep 17 00:00:00 2001 From: Kyle Maxwell Date: Mon, 15 Sep 2014 16:18:01 -0500 Subject: [PATCH 1/6] Ignore combine.cfg and .cfg, latter used for safety --- .gitignore | 2 ++ combine.cfg => combine-example.cfg | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) rename combine.cfg => combine-example.cfg (67%) diff --git a/.gitignore b/.gitignore index ea9756b..6326487 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,5 @@ harvest.json enriched.json .ipynb_checkpoints +combine.cfg +.cfg diff --git a/combine.cfg b/combine-example.cfg similarity index 67% rename from combine.cfg rename to combine-example.cfg index eb78a7f..2b28bfd 100644 --- a/combine.cfg +++ b/combine-example.cfg @@ -3,8 +3,8 @@ inbound_urls = inbound_urls.txt outbound_urls = outbound_urls.txt [Winnower] -dnsdb_server = https://api.dnsdb.info/ -dnsdb_api = YOUR_API_KEY_HERE +dnsdb_server = https://dnsdb-api.mlsecproject.org +dnsdb_api = YOUR_KEY_HERE enrich_dns = 1 enrich_ip = 1 From 4cf281d1b8cbffcc34fd014bb04fa75c5ee343eb Mon Sep 17 00:00:00 2001 From: Kyle Maxwell Date: Mon, 15 Sep 2014 16:19:26 -0500 Subject: [PATCH 2/6] Ignore local test data --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 6326487..9c5aeba 100644 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,6 @@ enriched.json .ipynb_checkpoints combine.cfg .cfg + +# test local files +test*.txt From f511d4795340ff6b13a9d223853cc1bfd168ce30 Mon Sep 17 00:00:00 2001 From: Kyle Maxwell Date: Mon, 15 Sep 2014 16:21:33 -0500 Subject: [PATCH 3/6] Use correct endpoint --- combine-example.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/combine-example.cfg b/combine-example.cfg index 2b28bfd..eb78a7f 100644 --- a/combine-example.cfg +++ b/combine-example.cfg @@ -3,8 +3,8 @@ inbound_urls = inbound_urls.txt outbound_urls = outbound_urls.txt [Winnower] -dnsdb_server = https://dnsdb-api.mlsecproject.org -dnsdb_api = YOUR_KEY_HERE +dnsdb_server = https://api.dnsdb.info/ +dnsdb_api = YOUR_API_KEY_HERE enrich_dns = 1 enrich_ip = 1 From 901cfc984efa7f9c7d25e7ce0904865afb4b4fe9 Mon Sep 17 00:00:00 2001 From: Kyle Maxwell Date: Mon, 15 Sep 2014 16:38:30 -0500 Subject: [PATCH 4/6] Test for existence of combine.cfg --- baler.py | 7 ++++++- reaper.py | 10 +++++++--- winnower.py | 7 ++++++- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/baler.py b/baler.py index 43e842d..6151abc 100755 --- a/baler.py +++ b/baler.py @@ -9,7 +9,12 @@ def tiq_output(reg_file, enr_file): config = ConfigParser.ConfigParser() - config.read('combine.cfg') + cfg_success = config.read('combine.cfg') + if not cfg_success: + sys.stderr.write('Could not read combine.cfg.\n') + sys.stderr.write('HINT: edit combine-example.cfg and save as combine.cfg.\n') + return + tiq_dir = os.path.join(config.get('Baler', 'tiq_directory'), 'data') today = dt.datetime.today().strftime('%Y%m%d') diff --git a/reaper.py b/reaper.py index 2be451a..6b064b7 100755 --- a/reaper.py +++ b/reaper.py @@ -9,8 +9,12 @@ def exception_handler(request, exception): def reap(file_name): - config = ConfigParser.ConfigParser() - config.read('combine.cfg') + config = ConfigParser.SafeConfigParser(allow_no_value=False) + cfg_success = config.read('combine.cfg') + if not cfg_success: + sys.stderr.write('Could not read combine.cfg.\n') + sys.stderr.write('HINT: edit combine-example.cfg and save as combine.cfg.\n') + return inbound_url_file = config.get('Reaper', 'inbound_urls') outbound_url_file = config.get('Reaper', 'outbound_urls') @@ -30,7 +34,7 @@ def reap(file_name): reqs = [grequests.get(url, headers=headers) for url in outbound_urls] outbound_responses = grequests.map(reqs) outbound_harvest = [(response.url, response.status_code, response.text) for response in outbound_responses] - + sys.stderr.write('Storing raw feeds in %s\n' % file_name) harvest = {'inbound': inbound_harvest, 'outbound': outbound_harvest} diff --git a/winnower.py b/winnower.py index 1bdaed9..d95d0fa 100755 --- a/winnower.py +++ b/winnower.py @@ -80,7 +80,12 @@ def reserved(address): def winnow(in_file, out_file, enr_file): config = ConfigParser.ConfigParser(allow_no_value=True) - config.read('combine.cfg') + cfg_success = config.read('combine.cfg') + if not cfg_success: + sys.stderr.write('Could not read combine.cfg.\n') + sys.stderr.write('HINT: edit combine-example.cfg and save as combine.cfg.\n') + return + server = config.get('Winnower', 'dnsdb_server') api = config.get('Winnower', 'dnsdb_api') enrich_ip = config.get('Winnower', 'enrich_ip') From 0b77dd3aea22579e59a7993d6113d3275f6e2806 Mon Sep 17 00:00:00 2001 From: Kyle Maxwell Date: Mon, 15 Sep 2014 19:27:53 -0500 Subject: [PATCH 5/6] Use SafeConfigParser everywhere per doc recommendation --- baler.py | 2 +- winnower.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/baler.py b/baler.py index 6151abc..d3f1189 100755 --- a/baler.py +++ b/baler.py @@ -8,7 +8,7 @@ def tiq_output(reg_file, enr_file): - config = ConfigParser.ConfigParser() + config = ConfigParser.SafeConfigParser() cfg_success = config.read('combine.cfg') if not cfg_success: sys.stderr.write('Could not read combine.cfg.\n') diff --git a/winnower.py b/winnower.py index d95d0fa..7667e84 100755 --- a/winnower.py +++ b/winnower.py @@ -79,7 +79,7 @@ def reserved(address): def winnow(in_file, out_file, enr_file): - config = ConfigParser.ConfigParser(allow_no_value=True) + config = ConfigParser.SafeConfigParser(allow_no_value=True) cfg_success = config.read('combine.cfg') if not cfg_success: sys.stderr.write('Could not read combine.cfg.\n') From 501a99f682ab9db789368a97bf77f9e01ef2b8bc Mon Sep 17 00:00:00 2001 From: Alexandre Pinto Date: Mon, 15 Sep 2014 18:04:58 -0700 Subject: [PATCH 6/6] Added checking for the config file at the beginning of each step of the process. This is begging for refactoring (#68) --- baler.py | 9 ++++++++- reaper.py | 2 +- thresher.py | 9 +++++++++ winnower.py | 2 +- 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/baler.py b/baler.py index d3f1189..f35d21f 100755 --- a/baler.py +++ b/baler.py @@ -11,7 +11,7 @@ def tiq_output(reg_file, enr_file): config = ConfigParser.SafeConfigParser() cfg_success = config.read('combine.cfg') if not cfg_success: - sys.stderr.write('Could not read combine.cfg.\n') + sys.stderr.write('tiq_output: Could not read combine.cfg.\n') sys.stderr.write('HINT: edit combine-example.cfg and save as combine.cfg.\n') return @@ -93,6 +93,13 @@ def bale_enr_csvgz(harvest, output_file): def bale(input_file, output_file, output_format): + config = ConfigParser.SafeConfigParser() + cfg_success = config.read('combine.cfg') + if not cfg_success: + sys.stderr.write('Baler: Could not read combine.cfg.\n') + sys.stderr.write('HINT: edit combine-example.cfg and save as combine.cfg.\n') + return + sys.stderr.write('Reading processed data from %s\n' % input_file) with open(input_file, 'rb') as f: harvest = json.load(f) diff --git a/reaper.py b/reaper.py index 6b064b7..6f75a9d 100755 --- a/reaper.py +++ b/reaper.py @@ -12,7 +12,7 @@ def reap(file_name): config = ConfigParser.SafeConfigParser(allow_no_value=False) cfg_success = config.read('combine.cfg') if not cfg_success: - sys.stderr.write('Could not read combine.cfg.\n') + sys.stderr.write('Reaper: Could not read combine.cfg.\n') sys.stderr.write('HINT: edit combine-example.cfg and save as combine.cfg.\n') return diff --git a/thresher.py b/thresher.py index f2c263b..a8e3092 100755 --- a/thresher.py +++ b/thresher.py @@ -1,3 +1,4 @@ +import ConfigParser import bs4 import datetime import feedparser @@ -134,6 +135,14 @@ def process_malwaregroup(response, source, direction): def thresh(input_file, output_file): + + config = ConfigParser.SafeConfigParser(allow_no_value=False) + cfg_success = config.read('combine.cfg') + if not cfg_success: + sys.stderr.write('Thresher: Could not read combine.cfg.\n') + sys.stderr.write('HINT: edit combine-example.cfg and save as combine.cfg.\n') + return + sys.stderr.write('Loading raw feed data from %s\n' % input_file) with open(input_file, 'rb') as f: crop = json.load(f) diff --git a/winnower.py b/winnower.py index 7667e84..7828c0e 100755 --- a/winnower.py +++ b/winnower.py @@ -82,7 +82,7 @@ def winnow(in_file, out_file, enr_file): config = ConfigParser.SafeConfigParser(allow_no_value=True) cfg_success = config.read('combine.cfg') if not cfg_success: - sys.stderr.write('Could not read combine.cfg.\n') + sys.stderr.write('Winnower: Could not read combine.cfg.\n') sys.stderr.write('HINT: edit combine-example.cfg and save as combine.cfg.\n') return