Refactor of models.py; integration tests

.gitignore

@@ -24,6 +24,8 @@ pip-log.txt
 # Unit test / coverage reports
 .coverage
 .tox
+.noseids
+test.log
 nosetests.xml
 
 # Translations

data/query_waterpoints.py

@@ -11,10 +11,10 @@ def get_all_reports():
     print response.url
     print response.ok
 
-    data = json.loads(response.text)
-    reports = data['result']
-    print reports[0]
-    print len(reports)
+    reports = json.loads(response.text)
+    for r in reports:
+        print r
+    print 'Total %s reports' % len(reports)
 
 if __name__ == '__main__':
     get_all_reports()

data/upload_waterpoints.py

@@ -49,22 +49,22 @@ def parse_csv(csv_fn):
 
 def create_wp_service():
     headers = {'content-type': 'application/json'}
-    data = {'name': 'WaterpointService',
-            'fields': {'waterpoint_id': {'type': 'StringField', 'required': True},
-                       'region': {'type': 'StringField', 'required': True},
-                       'lga_name': {'type': 'StringField', 'required': True},
-                       'ward': {'type': 'StringField', 'required': True},
-                       'village': {'type': 'StringField', 'required': True},
-                       'technology_in_use': {'type': 'StringField', 'required': True},
-                       'waterpoint': {'type': 'StringField', 'required': True},
-                       'status': {'type': 'StringField', 'required': True},
-                       'latitude': {'type': 'FloatField', 'required': True},
-                       'longitude': {'type': 'FloatField', 'required': True},
-                       },
+    data = {'classname': 'WaterpointService',
+            'fields': [{'name': 'waterpoint_id', 'fieldtype': 'StringField', 'required': True},
+                       {'name': 'region', 'fieldtype': 'StringField', 'required': True},
+                       {'name': 'lga_name', 'fieldtype': 'StringField', 'required': True},
+                       {'name': 'ward', 'fieldtype': 'StringField', 'required': True},
+                       {'name': 'village', 'fieldtype': 'StringField', 'required': True},
+                       {'name': 'technology_in_use', 'fieldtype': 'StringField', 'required': True},
+                       {'name': 'waterpoint', 'fieldtype': 'StringField', 'required': True},
+                       {'name': 'status', 'fieldtype': 'StringField', 'required': True},
+                       {'name': 'latitude', 'fieldtype': 'FloatField', 'required': True},
+                       {'name': 'longitude', 'fieldtype': 'FloatField', 'required': True},
+                       ],
             'group': 'location based reports',
             'keywords': ['waterpoints'],
             'protocol_type': '',
-            'service_name': '',
+            'service_name': 'Waterpoint',
             'service_code': 'wp1'
             }
     requests.post(SERVICES_URL, data=json.dumps(data), headers=headers)

taarifa_backend/__init__.py

@@ -1,11 +1,12 @@
-import logging
-from os import environ
 import urlparse
+import logging
 
+from os import environ
 from flask import Flask
 from flask.ext.mongoengine import MongoEngine
 from flask.ext.security import Security, MongoEngineUserDatastore
 
+
 # configure the logging
 logging.basicConfig(level='DEBUG',
                     format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
@@ -25,14 +26,12 @@
 db = MongoEngine(app)
 
 import models
+#TODO: where is this currently used? --nweinert
 user_datastore = MongoEngineUserDatastore(db, models.User, models.Role)
 security = Security(app, user_datastore)
 
 from taarifa_backend.api import api
 app.register_blueprint(api)
 
-app.logger.debug('Registered views are: \n' +
-                 app.view_functions.keys().__repr__())
-
 if __name__ == '__main__':
     app.run()

taarifa_backend/api.py

@@ -1,35 +1,38 @@
-import logging
-import json
-
-from flask import Blueprint, request, jsonify, render_template, make_response, redirect, flash
+from flask import Blueprint, request, jsonify, render_template, redirect, flash
 from flask.ext.security import http_auth_required
 from flask.ext.mongoengine.wtf import model_form
 import mongoengine
 
 import models
-from taarifa_backend import user_datastore
-from utils import crossdomain, jsonp, db_type_to_string, mongo_to_dict
+from taarifa_backend import user_datastore, app
+from utils import crossdomain, jsonp, mongoengine_class_to_string
 
-logger = logging.getLogger(__name__)
+logger = app.logger
 
 api = Blueprint("api", __name__, template_folder='templates')
 
 
+def __json_response(document):
+    """creates an response object with the document included as json"""
+    return app.response_class(document.to_json(), mimetype='application/json')
+
+
 def get_services():
     response = {}
 
     services = models.get_available_services()
     for service in services:
         res = dict((key, getattr(service, key, None))
-                   for key in ['protocol_type', 'keywords', 'service_name',
+                   for key in ['protocol_type', 'service_name',
                                'service_code', 'group', 'description'])
+        res['keywords'] = getattr(service, 'keywords', [])
         fields = {}
         for name, f in service._fields.iteritems():
             if name in ['id', 'created_at']:
                 continue
             fields[name] = {
                 'required': f.required,
-                'type': db_type_to_string(f.__class__)
+                'type': mongoengine_class_to_string(f.__class__)
             }
         res['fields'] = fields
         response[service.__name__] = res
@@ -61,7 +64,7 @@ def receive_report():
 
     # TODO: Handle errors if the report field is not available
     data = request.json['data']
-    data.update(dict(service_code=service_code))
+    data['service_code'] = service_code
     db_obj = service_class(**data)
 
     try:
@@ -70,9 +73,9 @@ def receive_report():
         logger.debug(e)
         # TODO: Send specification of the service used and a better error
         # description
-        return jsonify({'Error': 'Validation Error'})
+        return jsonify(Error='Validation Error')
 
-    return jsonify(mongo_to_dict(doc))
+    return __json_response(doc)
 
 
 @api.route("/reports/add", methods=['GET', 'POST'])
@@ -95,22 +98,22 @@ def add_report():
 @jsonp
 def get_all_reports():
     service_code = request.args.get('service_code', None)
+    logger.debug('Received request for service_code=%s', service_code)
 
     service_class = models.get_service_class(service_code)
 
     all_reports = service_class.objects.all()
 
-    return make_response(json.dumps(map(mongo_to_dict, all_reports)))
+    return __json_response(all_reports)
 
 
 @api.route("/reports/<string:id>", methods=['GET'])
 @crossdomain(origin='*')
 @jsonp
 def get_report(id=False):
-    # TODO: This is still using BasicReport, should be moved to service based
-    # world
+    logger.debug('Received request for report: %s' % id)
     report = models.Report.objects.get(id=id)
-    return jsonify(mongo_to_dict(report))
+    return __json_response(report)
 
 
 @api.route("/services", methods=['POST'])
@@ -119,24 +122,26 @@ def create_service():
     logger.debug('Service post received')
     logger.debug('JSON: %r' % request.json)
 
-    db_obj = models.Service(**request.json)
+    fields = request.json.pop('fields')
+    report_fields = [models.ReportField(**field) for field in fields]
+
+    db_obj = models.Service(fields=report_fields, **request.json)
 
     try:
         doc = db_obj.save()
     except mongoengine.ValidationError as e:
         logger.debug(e)
         # TODO: Send specification of the service used and a better error
         # description
-        return jsonify({'Error': 'Validation Error'})
-
-    return jsonify(mongo_to_dict(doc))
+        return jsonify(Error='Validation Error')
+    return __json_response(doc)
 
 
 @api.route("/services", methods=['GET'])
 @crossdomain(origin='*')
 @jsonp
 def get_list_of_all_services():
-    # TODO: factor out the transformation from a service to json
+    logger.debug('Request for service list received')
     return jsonify(**get_services())
 
 
@@ -155,15 +160,15 @@ def create_admin():
     required = ["email", "password"]
     missing = list(set(required) - set(request.json.keys()))
     if missing:
-        response = jsonify({"Error": "Validation Error",
-                            "Missing": missing})
+        response = jsonify(Error="Validation Error",
+                           Missing=missing)
         response.status_code = 422
         return response
 
     try:
         user = user_datastore.create_user(email=request.json["email"],
                                           password=request.json["password"])
     except mongoengine.ValidationError:
-        return jsonify({'Error': 'Validation Error'})
+        return jsonify(Error='Validation Error')
 
-    return jsonify(mongo_to_dict(user))
+    return __json_response(user)

taarifa_backend/manage.py

@@ -6,7 +6,7 @@
 from flask.ext.script import Manager, Server
 
 from taarifa_backend import app
-from taarifa_backend.models import clear_database, Role, User, Service
+from taarifa_backend.models import clear_database, Role, User, Service, ReportField
 
 manager = Manager(app)
 
@@ -38,21 +38,21 @@ def setup(email, clean):
 @manager.command
 def create_services():
     """Create default service types BasicReport and Waterpoint."""
-    Service(name="Generic",
-            fields={
-                "title": {"type": "StringField", "max_length": 255, "required": True},
-                "desc": {"type": "StringField", "required": True}
-            },
+    Service(classname="Generic",
+            fields=[ReportField(name="title", fieldtype="StringField",
+                                max_length=255, required=True),
+                    ReportField(name="desc", fieldtype="StringField",
+                                required=True)],
             description="Generic location based report",
             keywords=["location", "report"],
             group="location",
             service_name="basic report",
             service_code="0001").save()
-    Service(name="Waterpoint",
-            fields={
-                "waterpoint_id": {"type": "StringField", "required": True},
-                "functional": {"type": "BooleanField", "required": True}
-            },
+    Service(classname="Waterpoint",
+            fields=[ReportField(name="waterpoint_id", fieldtype="StringField",
+                                required=True),
+                    ReportField(name="functional", fieldtype="BooleanField",
+                                required=True)],
             description="Location, description and functionality of a waterpoint",
             keywords=["location", "report", "water"],
             group="water",

taarifa_backend/models.py

@@ -4,38 +4,15 @@
 from flask.ext.mongoengine.wtf import model_form
 
 from taarifa_backend import db
+from taarifa_backend.utils import get_mongoengine_class, fieldmap
 
-fieldmap = {
-    'BinaryField': db.BinaryField,
-    'BooleanField': db.BooleanField,
-    'ComplexDateTimeField': db.ComplexDateTimeField,
-    'DateTimeField': db.DateTimeField,
-    'DecimalField': db.DecimalField,
-    'DictField': db.DictField,
-    'DynamicField': db.DynamicField,
-    'EmailField': db.EmailField,
-    'EmbeddedDocumentField': db.EmbeddedDocumentField,
-    'FileField': db.FileField,
-    'FloatField': db.FloatField,
-    'GenericEmbeddedDocumentField': db.GenericEmbeddedDocumentField,
-    'GenericReferenceField': db.GenericReferenceField,
-    'GeoPointField': db.GeoPointField,
-    'ImageField': db.ImageField,
-    'IntField': db.IntField,
-    'ListField': db.ListField,
-    'MapField': db.MapField,
-    'ObjectIdField': db.ObjectIdField,
-    'ReferenceField': db.ReferenceField,
-    'SequenceField': db.SequenceField,
-    'SortedListField': db.SortedListField,
-    'StringField': db.StringField,
-    'URLField': db.URLField,
-    'UUIDField': db.UUIDField,
-}
-
-
-class Field(db.EmbeddedDocument):
-    """Field in a :class:`Service`."""
+
+class ReportField(db.EmbeddedDocument):
+    """Description of a field in a report"""
+    # TODO: must be a python field name, check or always correct this
+    name = db.StringField(required=True)
+    fieldtype = db.StringField(required=True, choices=fieldmap.keys())
+    # mongoengine properties of a Field
     db_field = db.StringField(default=None)
     required = db.BooleanField(default=False)
     default = db.DynamicField(default=None)
@@ -48,49 +25,39 @@ class Field(db.EmbeddedDocument):
 
 
 class Service(db.Document):
-    """A service schema served by the API."""
-    name = db.StringField(required=True)
-    fields = db.DictField(required=True)
+    """A service schema served by the API.
+
+    Describes the fields and validations of a certain type of report
+    """
+    # Must be a valid python class name
+    # TODO: Have to check or correct this
+    classname = db.StringField(required=True)
+    service_name = db.StringField(required=True)
+    service_code = db.StringField(required=True, unique=True)
+    fields = db.ListField(field=db.EmbeddedDocumentField(ReportField))
     description = db.StringField()
     group = db.StringField()
     keywords = db.ListField(db.StringField())
     protocol_type = db.StringField()
-    service_name = db.StringField(required=True)
-    service_code = db.StringField(required=True, unique=True)
-
-
-def build_schema(service):
-    build_field = lambda d: fieldmap[d.pop('type')](**d)
-    return type(str(service.name), (Report,),
-                dict(description=service.description,
-                     group=service.group,
-                     keywords=service.keywords,
-                     protocol_type=service.protocol_type,
-                     service_name=service.service_name,
-                     service_code=service.service_code,
-                     meta={'allow_inheritance': True},
-                     **dict((k, build_field(v)) for k, v in service.fields.items()))
-                )
 
 
-class Metadata(object):
+def create_db_field(field):
+    """creates a mongoengine field from the description contained in a :class:`ReportField`"""
+    _class = get_mongoengine_class(field.fieldtype)
+    return _class(db_field=field.db_field, required=field.required, default=field.default,
+                  unique=field.unique, unique_with=field.unique_with,
+                  primary_key=field.primary_key, choices=field.choices,
+                  help_text=field.help_text, verbose_name=field.verbose_name)
 
-    """
-    Description of a service
-    """
 
-    def __init__(self, service_code, service_name, description, group=None):
-        self.service_code = service_code
-        self.service_name = service_name
-        self.description = description
-        self.group = group
-
-    def __repr__(self):
-        args = [self.service_code, self.service_name, self.description, self.group]
-        return 'Metadata(%s)' % ', '.join(map(str, args))
+def build_schema(service):
+    """dynamically creates the :class:`Report` for the given :class:`Service`"""
+    fields = dict([(f.name, create_db_field(f)) for f in service.fields])
+    return type(str(service.classname), (Report, ), fields)
 
 
 class Report(db.Document):
+    """base class used for all created Reports"""
     created_at = db.DateTimeField(default=datetime.datetime.now, required=True)
 
     latitude = db.FloatField(required=True)