diff --git a/.gitignore b/.gitignore index ea9756b..9c5aeba 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,8 @@ harvest.json enriched.json .ipynb_checkpoints +combine.cfg +.cfg + +# test local files +test*.txt diff --git a/baler.py b/baler.py index 43e842d..f35d21f 100755 --- a/baler.py +++ b/baler.py @@ -8,8 +8,13 @@ def tiq_output(reg_file, enr_file): - config = ConfigParser.ConfigParser() - config.read('combine.cfg') + config = ConfigParser.SafeConfigParser() + cfg_success = config.read('combine.cfg') + if not cfg_success: + sys.stderr.write('tiq_output: Could not read combine.cfg.\n') + sys.stderr.write('HINT: edit combine-example.cfg and save as combine.cfg.\n') + return + tiq_dir = os.path.join(config.get('Baler', 'tiq_directory'), 'data') today = dt.datetime.today().strftime('%Y%m%d') @@ -88,6 +93,13 @@ def bale_enr_csvgz(harvest, output_file): def bale(input_file, output_file, output_format): + config = ConfigParser.SafeConfigParser() + cfg_success = config.read('combine.cfg') + if not cfg_success: + sys.stderr.write('Baler: Could not read combine.cfg.\n') + sys.stderr.write('HINT: edit combine-example.cfg and save as combine.cfg.\n') + return + sys.stderr.write('Reading processed data from %s\n' % input_file) with open(input_file, 'rb') as f: harvest = json.load(f) diff --git a/combine.cfg b/combine-example.cfg similarity index 100% rename from combine.cfg rename to combine-example.cfg diff --git a/reaper.py b/reaper.py index 2be451a..6f75a9d 100755 --- a/reaper.py +++ b/reaper.py @@ -9,8 +9,12 @@ def exception_handler(request, exception): def reap(file_name): - config = ConfigParser.ConfigParser() - config.read('combine.cfg') + config = ConfigParser.SafeConfigParser(allow_no_value=False) + cfg_success = config.read('combine.cfg') + if not cfg_success: + sys.stderr.write('Reaper: Could not read combine.cfg.\n') + sys.stderr.write('HINT: edit combine-example.cfg and save as combine.cfg.\n') + return inbound_url_file = config.get('Reaper', 'inbound_urls') outbound_url_file = config.get('Reaper', 'outbound_urls') @@ -30,7 +34,7 @@ def reap(file_name): reqs = [grequests.get(url, headers=headers) for url in outbound_urls] outbound_responses = grequests.map(reqs) outbound_harvest = [(response.url, response.status_code, response.text) for response in outbound_responses] - + sys.stderr.write('Storing raw feeds in %s\n' % file_name) harvest = {'inbound': inbound_harvest, 'outbound': outbound_harvest} diff --git a/thresher.py b/thresher.py index 963b609..f112d84 100755 --- a/thresher.py +++ b/thresher.py @@ -1,3 +1,4 @@ +import ConfigParser import bs4 import datetime import feedparser @@ -134,6 +135,14 @@ def process_malwaregroup(response, source, direction): def thresh(input_file, output_file): + + config = ConfigParser.SafeConfigParser(allow_no_value=False) + cfg_success = config.read('combine.cfg') + if not cfg_success: + sys.stderr.write('Thresher: Could not read combine.cfg.\n') + sys.stderr.write('HINT: edit combine-example.cfg and save as combine.cfg.\n') + return + sys.stderr.write('Loading raw feed data from %s\n' % input_file) with open(input_file, 'rb') as f: crop = json.load(f) diff --git a/winnower.py b/winnower.py index 0472f32..62efe13 100755 --- a/winnower.py +++ b/winnower.py @@ -80,8 +80,13 @@ def reserved(address): def winnow(in_file, out_file, enr_file): - config = ConfigParser.ConfigParser(allow_no_value=True) - config.read('combine.cfg') + config = ConfigParser.SafeConfigParser(allow_no_value=True) + cfg_success = config.read('combine.cfg') + if not cfg_success: + sys.stderr.write('Winnower: Could not read combine.cfg.\n') + sys.stderr.write('HINT: edit combine-example.cfg and save as combine.cfg.\n') + return + server = config.get('Winnower', 'dnsdb_server') api = config.get('Winnower', 'dnsdb_api') enrich_ip = config.get('Winnower', 'enrich_ip')