diff --git a/.gitignore b/.gitignore index d2d6f36..411c987 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,8 @@ pip-log.txt # Unit test / coverage reports .coverage .tox +.noseids +test.log nosetests.xml # Translations diff --git a/data/query_waterpoints.py b/data/query_waterpoints.py index 178944a..117075c 100755 --- a/data/query_waterpoints.py +++ b/data/query_waterpoints.py @@ -11,10 +11,10 @@ def get_all_reports(): print response.url print response.ok - data = json.loads(response.text) - reports = data['result'] - print reports[0] - print len(reports) + reports = json.loads(response.text) + for r in reports: + print r + print 'Total %s reports' % len(reports) if __name__ == '__main__': get_all_reports() diff --git a/data/upload_waterpoints.py b/data/upload_waterpoints.py index bb0b1b7..a1f8ef9 100755 --- a/data/upload_waterpoints.py +++ b/data/upload_waterpoints.py @@ -49,22 +49,22 @@ def parse_csv(csv_fn): def create_wp_service(): headers = {'content-type': 'application/json'} - data = {'name': 'WaterpointService', - 'fields': {'waterpoint_id': {'type': 'StringField', 'required': True}, - 'region': {'type': 'StringField', 'required': True}, - 'lga_name': {'type': 'StringField', 'required': True}, - 'ward': {'type': 'StringField', 'required': True}, - 'village': {'type': 'StringField', 'required': True}, - 'technology_in_use': {'type': 'StringField', 'required': True}, - 'waterpoint': {'type': 'StringField', 'required': True}, - 'status': {'type': 'StringField', 'required': True}, - 'latitude': {'type': 'FloatField', 'required': True}, - 'longitude': {'type': 'FloatField', 'required': True}, - }, + data = {'classname': 'WaterpointService', + 'fields': [{'name': 'waterpoint_id', 'fieldtype': 'StringField', 'required': True}, + {'name': 'region', 'fieldtype': 'StringField', 'required': True}, + {'name': 'lga_name', 'fieldtype': 'StringField', 'required': True}, + {'name': 'ward', 'fieldtype': 'StringField', 'required': True}, + {'name': 'village', 'fieldtype': 'StringField', 'required': True}, + {'name': 'technology_in_use', 'fieldtype': 'StringField', 'required': True}, + {'name': 'waterpoint', 'fieldtype': 'StringField', 'required': True}, + {'name': 'status', 'fieldtype': 'StringField', 'required': True}, + {'name': 'latitude', 'fieldtype': 'FloatField', 'required': True}, + {'name': 'longitude', 'fieldtype': 'FloatField', 'required': True}, + ], 'group': 'location based reports', 'keywords': ['waterpoints'], 'protocol_type': '', - 'service_name': '', + 'service_name': 'Waterpoint', 'service_code': 'wp1' } requests.post(SERVICES_URL, data=json.dumps(data), headers=headers) diff --git a/taarifa_backend/__init__.py b/taarifa_backend/__init__.py index 3d0ecd2..8feef90 100644 --- a/taarifa_backend/__init__.py +++ b/taarifa_backend/__init__.py @@ -1,11 +1,12 @@ -import logging -from os import environ import urlparse +import logging +from os import environ from flask import Flask from flask.ext.mongoengine import MongoEngine from flask.ext.security import Security, MongoEngineUserDatastore + # configure the logging logging.basicConfig(level='DEBUG', format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') @@ -25,14 +26,12 @@ db = MongoEngine(app) import models +#TODO: where is this currently used? --nweinert user_datastore = MongoEngineUserDatastore(db, models.User, models.Role) security = Security(app, user_datastore) from taarifa_backend.api import api app.register_blueprint(api) -app.logger.debug('Registered views are: \n' + - app.view_functions.keys().__repr__()) - if __name__ == '__main__': app.run() diff --git a/taarifa_backend/api.py b/taarifa_backend/api.py index 16018e4..4bc8d4a 100644 --- a/taarifa_backend/api.py +++ b/taarifa_backend/api.py @@ -1,35 +1,38 @@ -import logging -import json - -from flask import Blueprint, request, jsonify, render_template, make_response, redirect, flash +from flask import Blueprint, request, jsonify, render_template, redirect, flash from flask.ext.security import http_auth_required from flask.ext.mongoengine.wtf import model_form import mongoengine import models -from taarifa_backend import user_datastore -from utils import crossdomain, jsonp, db_type_to_string, mongo_to_dict +from taarifa_backend import user_datastore, app +from utils import crossdomain, jsonp, mongoengine_class_to_string -logger = logging.getLogger(__name__) +logger = app.logger api = Blueprint("api", __name__, template_folder='templates') +def __json_response(document): + """creates an response object with the document included as json""" + return app.response_class(document.to_json(), mimetype='application/json') + + def get_services(): response = {} services = models.get_available_services() for service in services: res = dict((key, getattr(service, key, None)) - for key in ['protocol_type', 'keywords', 'service_name', + for key in ['protocol_type', 'service_name', 'service_code', 'group', 'description']) + res['keywords'] = getattr(service, 'keywords', []) fields = {} for name, f in service._fields.iteritems(): if name in ['id', 'created_at']: continue fields[name] = { 'required': f.required, - 'type': db_type_to_string(f.__class__) + 'type': mongoengine_class_to_string(f.__class__) } res['fields'] = fields response[service.__name__] = res @@ -61,7 +64,7 @@ def receive_report(): # TODO: Handle errors if the report field is not available data = request.json['data'] - data.update(dict(service_code=service_code)) + data['service_code'] = service_code db_obj = service_class(**data) try: @@ -70,9 +73,9 @@ def receive_report(): logger.debug(e) # TODO: Send specification of the service used and a better error # description - return jsonify({'Error': 'Validation Error'}) + return jsonify(Error='Validation Error') - return jsonify(mongo_to_dict(doc)) + return __json_response(doc) @api.route("/reports/add", methods=['GET', 'POST']) @@ -95,22 +98,22 @@ def add_report(): @jsonp def get_all_reports(): service_code = request.args.get('service_code', None) + logger.debug('Received request for service_code=%s', service_code) service_class = models.get_service_class(service_code) all_reports = service_class.objects.all() - return make_response(json.dumps(map(mongo_to_dict, all_reports))) + return __json_response(all_reports) @api.route("/reports/", methods=['GET']) @crossdomain(origin='*') @jsonp def get_report(id=False): - # TODO: This is still using BasicReport, should be moved to service based - # world + logger.debug('Received request for report: %s' % id) report = models.Report.objects.get(id=id) - return jsonify(mongo_to_dict(report)) + return __json_response(report) @api.route("/services", methods=['POST']) @@ -119,7 +122,10 @@ def create_service(): logger.debug('Service post received') logger.debug('JSON: %r' % request.json) - db_obj = models.Service(**request.json) + fields = request.json.pop('fields') + report_fields = [models.ReportField(**field) for field in fields] + + db_obj = models.Service(fields=report_fields, **request.json) try: doc = db_obj.save() @@ -127,16 +133,15 @@ def create_service(): logger.debug(e) # TODO: Send specification of the service used and a better error # description - return jsonify({'Error': 'Validation Error'}) - - return jsonify(mongo_to_dict(doc)) + return jsonify(Error='Validation Error') + return __json_response(doc) @api.route("/services", methods=['GET']) @crossdomain(origin='*') @jsonp def get_list_of_all_services(): - # TODO: factor out the transformation from a service to json + logger.debug('Request for service list received') return jsonify(**get_services()) @@ -155,8 +160,8 @@ def create_admin(): required = ["email", "password"] missing = list(set(required) - set(request.json.keys())) if missing: - response = jsonify({"Error": "Validation Error", - "Missing": missing}) + response = jsonify(Error="Validation Error", + Missing=missing) response.status_code = 422 return response @@ -164,6 +169,6 @@ def create_admin(): user = user_datastore.create_user(email=request.json["email"], password=request.json["password"]) except mongoengine.ValidationError: - return jsonify({'Error': 'Validation Error'}) + return jsonify(Error='Validation Error') - return jsonify(mongo_to_dict(user)) + return __json_response(user) diff --git a/taarifa_backend/manage.py b/taarifa_backend/manage.py index 72c3a88..fbcef55 100644 --- a/taarifa_backend/manage.py +++ b/taarifa_backend/manage.py @@ -6,7 +6,7 @@ from flask.ext.script import Manager, Server from taarifa_backend import app -from taarifa_backend.models import clear_database, Role, User, Service +from taarifa_backend.models import clear_database, Role, User, Service, ReportField manager = Manager(app) @@ -38,21 +38,21 @@ def setup(email, clean): @manager.command def create_services(): """Create default service types BasicReport and Waterpoint.""" - Service(name="Generic", - fields={ - "title": {"type": "StringField", "max_length": 255, "required": True}, - "desc": {"type": "StringField", "required": True} - }, + Service(classname="Generic", + fields=[ReportField(name="title", fieldtype="StringField", + max_length=255, required=True), + ReportField(name="desc", fieldtype="StringField", + required=True)], description="Generic location based report", keywords=["location", "report"], group="location", service_name="basic report", service_code="0001").save() - Service(name="Waterpoint", - fields={ - "waterpoint_id": {"type": "StringField", "required": True}, - "functional": {"type": "BooleanField", "required": True} - }, + Service(classname="Waterpoint", + fields=[ReportField(name="waterpoint_id", fieldtype="StringField", + required=True), + ReportField(name="functional", fieldtype="BooleanField", + required=True)], description="Location, description and functionality of a waterpoint", keywords=["location", "report", "water"], group="water", diff --git a/taarifa_backend/models.py b/taarifa_backend/models.py index 37835de..962a2fe 100644 --- a/taarifa_backend/models.py +++ b/taarifa_backend/models.py @@ -4,38 +4,15 @@ from flask.ext.mongoengine.wtf import model_form from taarifa_backend import db +from taarifa_backend.utils import get_mongoengine_class, fieldmap -fieldmap = { - 'BinaryField': db.BinaryField, - 'BooleanField': db.BooleanField, - 'ComplexDateTimeField': db.ComplexDateTimeField, - 'DateTimeField': db.DateTimeField, - 'DecimalField': db.DecimalField, - 'DictField': db.DictField, - 'DynamicField': db.DynamicField, - 'EmailField': db.EmailField, - 'EmbeddedDocumentField': db.EmbeddedDocumentField, - 'FileField': db.FileField, - 'FloatField': db.FloatField, - 'GenericEmbeddedDocumentField': db.GenericEmbeddedDocumentField, - 'GenericReferenceField': db.GenericReferenceField, - 'GeoPointField': db.GeoPointField, - 'ImageField': db.ImageField, - 'IntField': db.IntField, - 'ListField': db.ListField, - 'MapField': db.MapField, - 'ObjectIdField': db.ObjectIdField, - 'ReferenceField': db.ReferenceField, - 'SequenceField': db.SequenceField, - 'SortedListField': db.SortedListField, - 'StringField': db.StringField, - 'URLField': db.URLField, - 'UUIDField': db.UUIDField, -} - - -class Field(db.EmbeddedDocument): - """Field in a :class:`Service`.""" + +class ReportField(db.EmbeddedDocument): + """Description of a field in a report""" + # TODO: must be a python field name, check or always correct this + name = db.StringField(required=True) + fieldtype = db.StringField(required=True, choices=fieldmap.keys()) + # mongoengine properties of a Field db_field = db.StringField(default=None) required = db.BooleanField(default=False) default = db.DynamicField(default=None) @@ -48,49 +25,39 @@ class Field(db.EmbeddedDocument): class Service(db.Document): - """A service schema served by the API.""" - name = db.StringField(required=True) - fields = db.DictField(required=True) + """A service schema served by the API. + + Describes the fields and validations of a certain type of report + """ + # Must be a valid python class name + # TODO: Have to check or correct this + classname = db.StringField(required=True) + service_name = db.StringField(required=True) + service_code = db.StringField(required=True, unique=True) + fields = db.ListField(field=db.EmbeddedDocumentField(ReportField)) description = db.StringField() group = db.StringField() keywords = db.ListField(db.StringField()) protocol_type = db.StringField() - service_name = db.StringField(required=True) - service_code = db.StringField(required=True, unique=True) - - -def build_schema(service): - build_field = lambda d: fieldmap[d.pop('type')](**d) - return type(str(service.name), (Report,), - dict(description=service.description, - group=service.group, - keywords=service.keywords, - protocol_type=service.protocol_type, - service_name=service.service_name, - service_code=service.service_code, - meta={'allow_inheritance': True}, - **dict((k, build_field(v)) for k, v in service.fields.items())) - ) -class Metadata(object): +def create_db_field(field): + """creates a mongoengine field from the description contained in a :class:`ReportField`""" + _class = get_mongoengine_class(field.fieldtype) + return _class(db_field=field.db_field, required=field.required, default=field.default, + unique=field.unique, unique_with=field.unique_with, + primary_key=field.primary_key, choices=field.choices, + help_text=field.help_text, verbose_name=field.verbose_name) - """ - Description of a service - """ - def __init__(self, service_code, service_name, description, group=None): - self.service_code = service_code - self.service_name = service_name - self.description = description - self.group = group - - def __repr__(self): - args = [self.service_code, self.service_name, self.description, self.group] - return 'Metadata(%s)' % ', '.join(map(str, args)) +def build_schema(service): + """dynamically creates the :class:`Report` for the given :class:`Service`""" + fields = dict([(f.name, create_db_field(f)) for f in service.fields]) + return type(str(service.classname), (Report, ), fields) class Report(db.Document): + """base class used for all created Reports""" created_at = db.DateTimeField(default=datetime.datetime.now, required=True) latitude = db.FloatField(required=True) diff --git a/taarifa_backend/utils.py b/taarifa_backend/utils.py index ab2277d..866869b 100644 --- a/taarifa_backend/utils.py +++ b/taarifa_backend/utils.py @@ -5,46 +5,45 @@ from taarifa_backend import db -db_type_to_name = { - db.DateTimeField: 'DateTime', - db.StringField: 'String', - db.FloatField: 'Float' +fieldmap = { + 'BinaryField': db.BinaryField, + 'BooleanField': db.BooleanField, + 'ComplexDateTimeField': db.ComplexDateTimeField, + 'DateTimeField': db.DateTimeField, + 'DecimalField': db.DecimalField, + 'DictField': db.DictField, + 'DynamicField': db.DynamicField, + 'EmailField': db.EmailField, + 'EmbeddedDocumentField': db.EmbeddedDocumentField, + 'FileField': db.FileField, + 'FloatField': db.FloatField, + 'GenericEmbeddedDocumentField': db.GenericEmbeddedDocumentField, + 'GenericReferenceField': db.GenericReferenceField, + 'GeoPointField': db.GeoPointField, + 'ImageField': db.ImageField, + 'IntField': db.IntField, + 'ListField': db.ListField, + 'MapField': db.MapField, + 'ObjectIdField': db.ObjectIdField, + 'ReferenceField': db.ReferenceField, + 'SequenceField': db.SequenceField, + 'SortedListField': db.SortedListField, + 'StringField': db.StringField, + 'URLField': db.URLField, + 'UUIDField': db.UUIDField, } -def db_type_to_string(db_type): - return db_type_to_name.get(db_type, 'Unknown') +def get_mongoengine_class(fieldtype): + """gets the associated class reference for the string""" + return fieldmap[fieldtype] -# After http://stackoverflow.com/a/14025561/396967 -def mongo_to_dict(obj): - return_data = [] - - if isinstance(obj, db.Document): - return_data.append(("id", str(obj.id))) - - for field_name in obj._fields: - - if field_name in ("id",): - continue - - data = obj._data[field_name] - - if data: - if isinstance(obj._fields[field_name], db.DateTimeField): - return_data.append((field_name, str(data.isoformat()))) - elif isinstance(obj._fields[field_name], db.StringField): - return_data.append((field_name, str(data))) - elif isinstance(obj._fields[field_name], db.FloatField): - return_data.append((field_name, float(data))) - elif isinstance(obj._fields[field_name], db.IntField): - return_data.append((field_name, int(data))) - elif isinstance(obj._fields[field_name], db.ListField): - return_data.append((field_name, data)) - elif isinstance(obj._fields[field_name], db.EmbeddedDocumentField): - return_data.append((field_name, mongo_to_dict(data))) - - return dict(return_data) +def mongoengine_class_to_string(_class): + for k, v in fieldmap.iteritems(): + if v == _class: + return k + return 'Unknown' def crossdomain(origin=None, methods=None, headers=None, diff --git a/tests/curl_create_service b/tests/curl_create_service deleted file mode 100755 index 5b0bd64..0000000 --- a/tests/curl_create_service +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh -curl -v -X POST -d '{ - "name": "BasicReport", - "fields": { - "title": {"type": "StringField", "max_length": 255, "required": true}, - "desc": {"type": "StringField", "required": true} - }, - "description": "Basic location based report", - "keywords": ["location", "report"], - "group": "location based reports", - "service_name": "basic report", - "service_code": "0001" -}' -H "Content-Type: application/json" localhost:5000/services diff --git a/tests/curl_create_waterpoint b/tests/curl_create_waterpoint deleted file mode 100755 index b18f2b5..0000000 --- a/tests/curl_create_waterpoint +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -curl -v -X POST -d '{"service_code": "wp001", "data": {"longitude": 3.42, "latitude": 3.12121, "waterpoint_id": "WP_LOC_CODE_001", "functional": true}}' -H "Content-Type: application/json" localhost:5000/reports diff --git a/tests/curl_create_report b/tests/curl_scripts/curl_create_report similarity index 100% rename from tests/curl_create_report rename to tests/curl_scripts/curl_create_report diff --git a/tests/curl_scripts/curl_create_service b/tests/curl_scripts/curl_create_service new file mode 100755 index 0000000..6642af9 --- /dev/null +++ b/tests/curl_scripts/curl_create_service @@ -0,0 +1,11 @@ +#!/bin/sh +curl -v -X POST -d '{ + "classname": "BasicReportCreatedViaCurl", + "fields": [{"name": "title", "fieldtype": "StringField", "max_length": 255, "required": true}, + {"name": "desc", "fieldtype": "StringField", "required": true}], + "description": "Basic location based report created from curl script", + "keywords": ["location", "report"], + "group": "location based reports", + "service_name": "basic report", + "service_code": "curl0001" +}' -H "Content-Type: application/json" localhost:5000/services diff --git a/tests/curl_scripts/curl_reports b/tests/curl_scripts/curl_reports new file mode 100755 index 0000000..6db9d19 --- /dev/null +++ b/tests/curl_scripts/curl_reports @@ -0,0 +1,2 @@ +#!/bin/sh +curl -v -X GET localhost:5000/reports diff --git a/tests/curl_send_wrong_report b/tests/curl_scripts/curl_send_wrong_report similarity index 100% rename from tests/curl_send_wrong_report rename to tests/curl_scripts/curl_send_wrong_report diff --git a/tests/curl_services b/tests/curl_scripts/curl_services similarity index 100% rename from tests/curl_services rename to tests/curl_scripts/curl_services diff --git a/tests/test.py b/tests/test.py index 4763d9c..998f0f3 100644 --- a/tests/test.py +++ b/tests/test.py @@ -7,10 +7,14 @@ from taarifa_backend.models import User from taarifa_backend.api import user_datastore +TEST_DB = 'taarifa_backend_test' + class TaarifaTest(unittest.TestCase): def setUp(self): + configured_db = app.config['MONGODB_SETTINGS']['db'] + assert configured_db == TEST_DB, 'Expected to use %s as a db, but was configured to use %s' % (TEST_DB, configured_db) app.config['TESTING'] = True self.app = app.test_client() self.data = {"email": "taarifa_test@example.com", @@ -18,15 +22,16 @@ def setUp(self): user_datastore.create_user(email="username", password="password") def tearDown(self): - db.connection.drop_database(app.config['MONGODB_SETTINGS']['db']) + db.connection.drop_database(TEST_DB) def _create_admin(self, data, expected_response_code, expected_increment, auth=("username", "password")): user_count = User.objects().count() - headers = {"content-type": "application/json"} + headers = None if auth: - headers["Authorization"] = "Basic " + b64encode(':'.join(auth)) - r = self.app.post("/admin", data=json.dumps(data), headers=headers) + headers = {"Authorization": "Basic " + b64encode(':'.join(auth))} + r = self.app.post("/admin", data=json.dumps(data), headers=headers, + content_type="application/json") self.assertEquals(expected_response_code, r.status_code) self.assertEquals(user_count + expected_increment, User.objects().count()) diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 0000000..9111717 --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,90 @@ +TEST_DB = 'taarifa_backend_test' +if __name__ == '__main__': + # setup the environment variable to use a test database when run from the command line + import os + os.environ['DBNAME'] = TEST_DB + +import json +from taarifa_backend import app, db +import unittest + +REPORTS_URL = '/reports' +SERVICE_URL = '/services' +SERVICE_CODE = "test0001" +SUCESS = '200 OK' + +SERVICE_DESC = {"classname": "TestReport", + "fields": [{"name": "title", "fieldtype": "StringField", "max_length": 255, "required": True}, + {"name": "desc", "fieldtype": "StringField", "required": True}], + "description": "report create from create_service test", + "keywords": ["location", "report"], + "group": "location based reports", + "service_name": "test report", + "service_code": SERVICE_CODE, + } + +REPORT_DATA = {'service_code': SERVICE_CODE, + 'data': {'title': 'Test report', + 'latitude': 70, + 'longitude': 20, + 'desc': 'report send from test_api.py', + } + } + +logger = app.logger + + +class ApiTest(unittest.TestCase): + + def setUp(self): + # Somehow it is not so easy to find out which database is used + # this assert at least makes sure the app config is set to the test_db + configured_db = app.config['MONGODB_SETTINGS']['db'] + assert configured_db == TEST_DB, 'Expected to use %s as a db, but was configured to use %s' % (TEST_DB, configured_db) + app.testing = True + self.app = app.test_client() + + def tearDown(self): + db.connection.drop_database(TEST_DB) + + def test_call_landing_page(self): + self.app.post('/') + + def test_create_service_and_post_report_and_query_reports(self): + self._create_service() + self._query_services() + report_id = self._post_report() + logger.debug('Posted report with report id: ' + report_id) + self._get_report(report_id) + self._get_all_reports() + + def _create_service(self): + result = self.app.post(SERVICE_URL, data=json.dumps(SERVICE_DESC), content_type='application/json') + self._check_status_and_log(result) + + def _query_services(self): + result = self.app.get(SERVICE_URL) + self._check_status_and_log(result) + + def _post_report(self): + result = self.app.post(REPORTS_URL, data=json.dumps(REPORT_DATA), content_type='application/json') + self._check_status_and_log(result) + return json.loads(result.data)['_id']['$oid'] + + def _get_report(self, report_id): + result = self.app.get(REPORTS_URL + '/%s' % report_id) + self._check_status_and_log(result) + + def _get_all_reports(self): + result = self.app.get(REPORTS_URL + '?service_code=%s' % SERVICE_CODE) + self._check_status_and_log(result) + + def _check_status_and_log(self, result): + self.assertEqual(result._status, SUCESS) + #TODO: should put some validation of the data here + _json = json.loads(result.data) + logger.debug(_json) + + +if __name__ == '__main__': + unittest.main()