Merge pull request #72 from mlsecproject/i68_config

Fix config (closes #68)

.gitignore

@@ -55,3 +55,8 @@ harvest.json
 enriched.json
 
 .ipynb_checkpoints
+combine.cfg
+.cfg
+
+# test local files
+test*.txt

baler.py

@@ -8,8 +8,13 @@
 
 
 def tiq_output(reg_file, enr_file):
-    config = ConfigParser.ConfigParser()
-    config.read('combine.cfg')
+    config = ConfigParser.SafeConfigParser()
+    cfg_success = config.read('combine.cfg')
+    if not cfg_success:
+        sys.stderr.write('tiq_output: Could not read combine.cfg.\n')
+        sys.stderr.write('HINT: edit combine-example.cfg and save as combine.cfg.\n')
+        return
+
     tiq_dir = os.path.join(config.get('Baler', 'tiq_directory'), 'data')
     today = dt.datetime.today().strftime('%Y%m%d')
 
@@ -88,6 +93,13 @@ def bale_enr_csvgz(harvest, output_file):
 
 
 def bale(input_file, output_file, output_format):
+    config = ConfigParser.SafeConfigParser()
+    cfg_success = config.read('combine.cfg')
+    if not cfg_success:
+        sys.stderr.write('Baler: Could not read combine.cfg.\n')
+        sys.stderr.write('HINT: edit combine-example.cfg and save as combine.cfg.\n')
+        return
+
     sys.stderr.write('Reading processed data from %s\n' % input_file)
     with open(input_file, 'rb') as f:
         harvest = json.load(f)

reaper.py

@@ -9,8 +9,12 @@ def exception_handler(request, exception):
 
 
 def reap(file_name):
-    config = ConfigParser.ConfigParser()
-    config.read('combine.cfg')
+    config = ConfigParser.SafeConfigParser(allow_no_value=False)
+    cfg_success = config.read('combine.cfg')
+    if not cfg_success:
+        sys.stderr.write('Reaper: Could not read combine.cfg.\n')
+        sys.stderr.write('HINT: edit combine-example.cfg and save as combine.cfg.\n')
+        return
 
     inbound_url_file = config.get('Reaper', 'inbound_urls')
     outbound_url_file = config.get('Reaper', 'outbound_urls')
@@ -30,7 +34,7 @@ def reap(file_name):
     reqs = [grequests.get(url, headers=headers) for url in outbound_urls]
     outbound_responses = grequests.map(reqs)
     outbound_harvest = [(response.url, response.status_code, response.text) for response in outbound_responses]
-    
+
     sys.stderr.write('Storing raw feeds in %s\n' % file_name)
     harvest = {'inbound': inbound_harvest, 'outbound': outbound_harvest}
 

thresher.py

@@ -1,3 +1,4 @@
+import ConfigParser
 import bs4
 import datetime
 import feedparser
@@ -134,6 +135,14 @@ def process_malwaregroup(response, source, direction):
 
 
 def thresh(input_file, output_file):
+
+    config = ConfigParser.SafeConfigParser(allow_no_value=False)
+    cfg_success = config.read('combine.cfg')
+    if not cfg_success:
+        sys.stderr.write('Thresher: Could not read combine.cfg.\n')
+        sys.stderr.write('HINT: edit combine-example.cfg and save as combine.cfg.\n')
+        return
+
     sys.stderr.write('Loading raw feed data from %s\n' % input_file)
     with open(input_file, 'rb') as f:
         crop = json.load(f)

winnower.py

@@ -80,8 +80,13 @@ def reserved(address):
 
 
 def winnow(in_file, out_file, enr_file):
-    config = ConfigParser.ConfigParser(allow_no_value=True)
-    config.read('combine.cfg')
+    config = ConfigParser.SafeConfigParser(allow_no_value=True)
+    cfg_success = config.read('combine.cfg')
+    if not cfg_success:
+        sys.stderr.write('Winnower: Could not read combine.cfg.\n')
+        sys.stderr.write('HINT: edit combine-example.cfg and save as combine.cfg.\n')
+        return
+
     server = config.get('Winnower', 'dnsdb_server')
     api = config.get('Winnower', 'dnsdb_api')
     enrich_ip = config.get('Winnower', 'enrich_ip')