Massive commit of decoupling from mongo and addinng a PanFileDB type (#…

…414)

* PanDB is a factory class that will load either PanMongoDB or PanFileDB
* PanFileDB merely writes the json to a flat file
	* One file per "collection" along with a `current_<collection` for current
	entries. Note this means each "current" collection is a separate file.
	* Use `bson.json_util` for serializing. Changes a few things with time
	because it _can_ store timezone aware info, which we don't use properly
	in `current_time`.
* Clear both mongo and file types of their "current" entries upon POCS init
* Change `include_collection` to `store_permanently` for calls to `insert_current`
* Change `use_mongo` to `store_result` where appropriate
* Private POCS method `_check_environment` changed to public class method, i.e.
	`POCS.check_environment`
* Tests that use the `db` now parameterize it across db types. Now testing
	takes almost twice as long! (should deal with this somehow)

bin/peas_shell

@@ -15,7 +15,7 @@ from peas.webcam import Webcam
 
 from pocs.utils import current_time
 from pocs.utils.config import load_config
-from pocs.utils.database import PanMongo
+from pocs.utils.database import PanDB
 
 
 class PanSensorShell(cmd.Cmd):
@@ -27,7 +27,7 @@ class PanSensorShell(cmd.Cmd):
     environment = None
     weather = None
     active_sensors = dict()
-    db = PanMongo()
+    db = PanDB()
     _keep_looping = False
     _loop_delay = 60
     _timer = None
@@ -190,7 +190,7 @@ class PanSensorShell(cmd.Cmd):
             port = '/dev/ttyUSB0'
 
         print("Loading AAG Cloud Sensor on {}".format(port))
-        self.weather = AAGCloudSensor(serial_address=port, use_mongo=True)
+        self.weather = AAGCloudSensor(serial_address=port, store_result=True)
         self.do_enable_sensor('weather')
 
 ##################################################################################################
@@ -353,7 +353,7 @@ class PanSensorShell(cmd.Cmd):
         if sensor_name in self.active_sensors:
             sensor = getattr(self, sensor_name)
             try:
-                sensor.capture(use_mongo=True, send_message=True)
+                sensor.capture(store_result=True, send_message=True)
             except Exception as e:
                 pass
 

bin/pocs_shell

@@ -26,7 +26,7 @@ from pocs.utils.images import fits as fits_utils
 from pocs.utils.images import cr2 as cr2_utils
 from pocs.utils.images import polar_alignment as polar_alignment_utils
 from pocs.utils import listify
-from pocs.utils.database import PanMongo
+from pocs.utils.database import PanDB
 from pocs.utils.messaging import PanMessaging
 
 
@@ -220,7 +220,7 @@ Hardware names: {}   (or all for all hardware)'''.format(
                     status = self.pocs.status()
                     pprint(status)
                     print_info('*' * 80)
-                    self.pocs.db.insert_current('system', status, include_collection=False)
+                    self.pocs.db.insert_current('system', status, store_permanently=False)
                     time.sleep(2)
                 except KeyboardInterrupt:
                     print_warning('Stopping collection')
@@ -813,7 +813,7 @@ def process_img(fn, start_time, remove_after=True):
         except Exception:
             pass
 
-        db = PanMongo()
+        db = PanDB()
 
         # Add to DB
         db.drift_align.insert_one({

conf_files/pocs.yaml

@@ -22,6 +22,7 @@ panoptes_network:
     image_storage: True
 db: 
     name: panoptes
+    type: mongo
 scheduler:
     type: dispatch
     fields_file: simple.yaml

peas/sensors.py

@@ -1,14 +1,11 @@
-import json
-import os
 
 # Note: list_comports is modified by test_sensors.py, so if changing
 # this import, the test will also need to be updated.
 from serial.tools.list_ports import comports as list_comports
 
 import sys
-import yaml
 
-from pocs.utils.database import PanMongo
+from pocs.utils.database import PanDB
 from pocs.utils.config import load_config
 from pocs.utils.logger import get_root_logger
 from pocs.utils.messaging import PanMessaging
@@ -82,7 +79,7 @@ def send_message(self, msg, channel='environment'):
 
         self.messaging.send_message(channel, msg)
 
-    def capture(self, use_mongo=True, send_message=True):
+    def capture(self, store_result=True, send_message=True):
         """
         Helper function to return serial sensor info.
 
@@ -121,10 +118,9 @@ def capture(self, use_mongo=True, send_message=True):
             except Exception as e:
                 self.logger.warning('Exception while reading from sensor {}: {}', sensor_name, e)
 
-        if use_mongo and len(sensor_data) > 0:
+        if store_result and len(sensor_data) > 0:
             if self.db is None:
-                self.db = PanMongo()
-                self.logger.info('Connected to PanMongo')
+                self.db = PanDB()
             self.db.insert_current('environment', sensor_data)
 
         return sensor_data

peas/weather.py

@@ -19,8 +19,8 @@
 
 
 def get_mongodb():
-    from pocs.utils.database import PanMongo
-    return PanMongo()
+    from pocs.utils.database import PanDB
+    return PanDB()
 
 
 def movingaverage(interval, window_size):
@@ -101,7 +101,7 @@ class AAGCloudSensor(object):
 
     """
 
-    def __init__(self, serial_address=None, use_mongo=True):
+    def __init__(self, serial_address=None, store_result=True):
         self.config = load_config(config_files='peas')
         self.logger = get_root_logger()
 
@@ -111,7 +111,7 @@ def __init__(self, serial_address=None, use_mongo=True):
         self.safety_delay = self.cfg.get('safety_delay', 15.)
 
         self.db = None
-        if use_mongo:
+        if store_result:
             self.db = get_mongodb()
 
         self.messaging = None
@@ -610,7 +610,7 @@ def send_message(self, msg, channel='weather'):
 
         self.messaging.send_message(channel, msg)
 
-    def capture(self, use_mongo=False, send_message=False, **kwargs):
+    def capture(self, store_result=False, send_message=False, **kwargs):
         """ Query the CloudWatcher """
 
         self.logger.debug("Updating weather")
@@ -662,7 +662,7 @@ def capture(self, use_mongo=False, send_message=False, **kwargs):
         if send_message:
             self.send_message({'data': data}, channel='weather')
 
-        if use_mongo:
+        if store_result:
             self.db.insert_current('weather', data)
 
         return data

pocs/base.py

@@ -3,7 +3,7 @@
 from pocs import hardware
 from pocs import __version__
 from pocs.utils import config
-from pocs.utils.database import PanMongo
+from pocs.utils.database import PanDB
 from pocs.utils.logger import get_root_logger
 
 # Global vars
@@ -49,9 +49,22 @@ def __init__(self, *args, **kwargs):
 
         self.config['simulator'] = hardware.get_simulator_names(config=self.config, kwargs=kwargs)
 
-        # Set up connection to database
-        db = kwargs.get('db', self.config['db']['name'])
-        _db = PanMongo(db=db, logger=self.logger)
+        # Get passed DB or set up new connection
+        _db = kwargs.get('db', None)
+        if _db is None:
+            # If the user requests a db_type then update runtime config
+            db_type = kwargs.get('db_type', None)
+            db_name = kwargs.get('db_name', None)
+
+            if db_type is not None:
+                self.config['db']['type'] = db_type
+            if db_name is not None:
+                self.config['db']['name'] = db_name
+
+            db_type = self.config['db']['type']
+            db_name = self.config['db']['name']
+
+            _db = PanDB(db_type=db_type, db_name=db_name, logger=self.logger).db
 
         self.db = _db
 

pocs/camera/canon_gphoto2.py

@@ -7,8 +7,6 @@
 
 from pocs.utils import current_time
 from pocs.utils import error
-from pocs.utils import images as img_utils
-from pocs.utils.images import fits as fits_utils
 from pocs.utils.images import cr2 as cr2_utils
 from pocs.camera import AbstractGPhotoCamera
 

pocs/camera/sbig.py

@@ -2,7 +2,6 @@
 from warnings import warn
 
 from astropy import units as u
-from astropy.io import fits
 
 from pocs.camera import AbstractCamera
 from pocs.camera.sbigudrv import INVALID_HANDLE_VALUE

pocs/camera/simulator.py

@@ -8,7 +8,6 @@
 
 from astropy import units as u
 from astropy.io import fits
-from astropy.time import Time
 
 from pocs.camera import AbstractCamera
 from pocs.utils.images import fits as fits_utils
@@ -76,8 +75,9 @@ def take_exposure(self, seconds=1.0 * u.second, filename=None, dark=False, block
         # Build FITS header
         header = self._fits_header(seconds, dark)
 
-        # Set up a Timer that will wait for the duration of the exposure then copy a dummy FITS file
-        # to the specified path and adjust the headers according to the exposure time, type.
+        # Set up a Timer that will wait for the duration of the exposure then
+        # copy a dummy FITS file to the specified path and adjust the headers
+        # according to the exposure time, type.
         exposure_event = Event()
         exposure_thread = Timer(interval=seconds,
                                 function=self._fake_exposure,

pocs/core.py

@@ -222,8 +222,7 @@ def power_down(self):
         if self.connected:
             self.say("I'm powering down")
             self.logger.info(
-                "Shutting down {}, please be patient and allow for exit.".format(
-                    self.name))
+                "Shutting down {}, please be patient and allow for exit.", self.name)
 
             if not self.observatory.close_dome():
                 self.logger.critical('Unable to close dome!')
@@ -347,10 +346,9 @@ def is_weather_safe(self, stale=180):
             bool: Conditions are safe (True) or unsafe (False)
 
         """
-        assert self.db.current, self.logger.warning(
-            "No connection to sensors, can't check weather safety")
 
         # Always assume False
+        self.logger.debug("Checking weather safety")
         is_safe = False
         record = {'safe': False}
 
@@ -366,15 +364,16 @@ def is_weather_safe(self, stale=180):
             record = self.db.get_current('weather')
 
             is_safe = record['data'].get('safe', False)
-            timestamp = record['date']
+            timestamp = record['date'].replace(tzinfo=None)  # current_time is timezone naive
             age = (current_time().datetime - timestamp).total_seconds()
 
             self.logger.debug(
                 "Weather Safety: {} [{:.0f} sec old - {}]".format(is_safe, age, timestamp))
 
-        except TypeError as e:
-            self.logger.warning("No record found in Mongo DB")
-            self.logger.debug('DB: {}'.format(self.db.current))
+        except (TypeError, KeyError) as e:
+            self.logger.warning("No record found in DB: {}", e)
+        except BaseException as e:
+            self.logger.error("Error checking weather: {}", e)
         else:
             if age > stale:
                 self.logger.warning("Weather record looks stale, marking unsafe.")
@@ -445,10 +444,11 @@ def wait_until_safe(self):
 
 
 ##################################################################################################
-# Private Methods
+# Class Methods
 ##################################################################################################
 
-    def _check_environment(self):
+    @classmethod
+    def check_environment(cls):
         """ Checks to see if environment is set up correctly
 
         There are a number of environmental variables that are expected
@@ -476,6 +476,10 @@ def _check_environment(self):
             print("Creating log dir at {}/logs".format(pandir))
             os.makedirs("{}/logs".format(pandir))
 
+##################################################################################################
+# Private Methods
+##################################################################################################
+
     def _check_messages(self, queue_type, q):
         cmd_dispatch = {
             'command': {

pocs/observatory.py

@@ -304,16 +304,13 @@ def analyze_recent(self):
             self.logger.debug('Offset Info: {}'.format(
                 self.current_offset_info))
 
-            # Update the observation info with the offsets
-            self.db.observations.update({'data.image_id': image_id}, {
-                '$set': {
-                    'offset_info': {
-                        'd_ra': self.current_offset_info.delta_ra.value,
-                        'd_dec': self.current_offset_info.delta_dec.value,
-                        'magnitude': self.current_offset_info.magnitude.value,
-                        'unit': 'arcsec'
-                    }
-                },
+            # Store the offset information
+            self.db.insert('offset_info', {
+                'image_id': image_id,
+                'd_ra': self.current_offset_info.delta_ra.value,
+                'd_dec': self.current_offset_info.delta_dec.value,
+                'magnitude': self.current_offset_info.magnitude.value,
+                'unit': 'arcsec',
             })
 
         except error.SolveError:

pocs/tests/conftest.py

@@ -8,7 +8,8 @@
 
 import pocs.base
 from pocs.utils.config import load_config
-from pocs.utils.database import PanMongo
+from pocs.utils.database import PanDB
+from pocs.utils.logger import get_root_logger
 
 # Global variable with the default config; we read it once, copy it each time it is needed.
 _one_time_config = None
@@ -37,9 +38,36 @@ def config_with_simulated_dome(config):
     return config
 
 
-@pytest.fixture
-def db():
-    return PanMongo(db='panoptes_testing')
+@pytest.fixture(scope='function', params=['mongo', 'file'])
+def db(request):
+    try:
+        _db = PanDB(
+            db_type=request.param,
+            db_name='panoptes_testing',
+            logger=get_root_logger(),
+            connect=True
+        )
+    except Exception:
+        pytest.skip("Can't connect to {} DB, skipping".format(request.param))
+
+    return _db
+
+
+@pytest.fixture(scope='function', params=['mongo', 'file'])
+def db_type(request):
+    # If testing mongo, make sure we can connect, otherwise skip
+    if request.param == 'mongo':
+        try:
+            PanDB(
+                db_type='mongo',
+                db_name='panoptes_testing',
+                logger=get_root_logger(),
+                connect=True
+            )
+        except Exception:
+            pytest.skip("Can't connect to {} DB, skipping".format(request.param))
+
+    return request.param
 
 
 @pytest.fixture

pocs/tests/test_messaging.py

@@ -66,13 +66,12 @@ def test_send_datetime(forwarder, sub, pub):
     assert msg_obj['date'] == '2017-01-01T00:00:00'
 
 
-def test_mongo_objectid(forwarder, sub, pub, config, db):
+def test_storage_id(forwarder, sub, pub, config, db):
 
-    db.insert_current('config', {'foo': 'bar'})
+    id0 = db.insert_current('config', {'foo': 'bar'}, store_permanently=False)
 
     pub.send_message('TEST-CHANNEL', db.get_current('config'))
     msg_type, msg_obj = sub.receive_message()
     assert '_id' in msg_obj
     assert isinstance(msg_obj['_id'], str)
-
-    db.current.remove({'type': 'config'})
+    assert id0 == msg_obj['_id']