Merge pull request #6 from hackforla/dev

Dev

server/src/app.py

@@ -8,6 +8,8 @@
 from services.reporting import reports
 from configparser import ConfigParser
 from json import loads
+from threading import Timer
+from multiprocessing import cpu_count
 
 
 app = Sanic(__name__)
@@ -32,23 +34,23 @@ async def index(request):
 @app.route('/timetoclose')
 async def timetoclose(request):
     ttc_worker = time_to_close(app.config['Settings'])
-    data = []
 
-    time_diff = loads(ttc_worker.ttc_time_diff(serviced=True, allRequests=False, requestType="'Bulky Items'"))
-    summary = loads(ttc_worker.ttc_summary())
+    # data = loads(ttc_worker.ttc_view_data())
+    # dates = loads(ttc_worker.ttc_view_dates())
+    summary = ttc_worker.ttc_summary(allData=True, serviced=False, allRequests=False, requestType="'Bulky Items'")
 
-    data.append(time_diff)
-    data.append(summary)
-    return json(data)
+    # return json(data_arr)
+    # return json(dates)
+    return json(summary)
 
 
 @app.route('/requestfrequency')
 async def requestfrequency(request):
-    freq_worker = frequency()
-    # Insert frequency calculation here
-    return_data = freq_worker.freq_query()
+    freq_worker = frequency(app.config['Settings'])
+
+    summary = loads(freq_worker.freq_view_all())
 
-    return json(return_data)
+    return json(summary)
 
 
 @app.route('/sample-data')
@@ -105,7 +107,13 @@ async def biggestOffender(request):
     return await response.file(fileOutput)
 
 
+@app.route('/test_multiple_workers')
+async def test_multiple_workers(request):
+    Timer(10.0, print, ["Timer Test."]).start()
+    return json("Done")
+
 
 if __name__ == '__main__':
     configure_app()
-    app.run(host=app.config['Settings']['Server']['HOST'], port=app.config['Settings']['Server']['PORT'], debug=app.config['Settings']['Server']['DEBUG'])
+    app.run(host=app.config['Settings']['Server']['HOST'], port=int(app.config['Settings']['Server']['PORT']),
+            workers=cpu_count()//2, debug=app.config['Settings']['Server']['DEBUG'])

server/src/services/frequency.py

@@ -1,28 +1,53 @@
 from configparser import ConfigParser
-from sqlalchemy.types import Integer, Text, String, DateTime, Float
-from sqlalchemy import create_engine
+import sqlalchemy as db
+import pandas as pd
+from datetime import datetime as dt
+import numpy as np
+import json
+
 
 class frequency(object):
-    def __init__(self, config=None):
+    def __init__(self, config=None, tableName="ingest_staging_table"):
         self.config = config
         self.dbString = None if not self.config else self.config['Database']['DB_CONNECTION_STRING']
+        self.table = tableName
+        self.data = None
         pass
 
-    def freq_query(self):
-        engine = create_engine(self.dbString)
+    def freq_view_all(self, serviced=False):
+        """
+        Returns the request type and associated dates for all data
+        Sorted by request type, followed by created date, service date (if applicable), and then closed date
+        """
+        # Todo: implement condition for serviced date
+        engine = db.create_engine(self.dbString)
+
+        query = "SELECT requesttype, createddate, closeddate FROM %s" % self.table
+        df = pd.read_sql_query(query, con=engine)
+
+        df['closeddate'] = pd.to_datetime(df['closeddate'])
+        df = df.sort_values(by=['requesttype', 'createddate', 'closeddate'])
+
+        return df.to_json(orient="records")
+
+    def freq_aggregate(self):
+        engine = db.create_engine(self.dbString)
+
+        query = "SELECT requesttype FROM %s" % (self.table)
+        df = pd.read_sql_query(query, con=engine)
 
-        connection = engine.connect()
-        query = "SELECT row_to_json(row(status)) \
-            FROM ingest_staging_table"
-        result = connection.execute(query)
-        connection.close()
+        request_counts = df['requesttype'].value_counts()
+        print(np.dtype(request_counts))
+
+        # for request in request_types:
+        #     print(df[request].value_counts())
+        return df['requesttype'].value_counts()
 
-        return result
 
 if __name__ == "__main__":
-    frequer = frequency()
+    freq = frequency()
     config = ConfigParser()
     config.read("../setting.cfg")
-    frequer.config   = config
-    frequer.dbString = config['Database']['DB_CONNECTION_STRING']
-    frequer.freq_query()
+    freq.config = config
+    freq.dbString = config['Database']['DB_CONNECTION_STRING']
+    freq.freq_aggregate()

server/src/services/time_to_close.py

@@ -37,8 +37,6 @@ def ttc_view_data(self, onlyClosed=False):
         else:
             df = pd.read_sql_query("SELECT * FROM %s" % self.table, con=engine)
 
-        # df = pd.read_sql_query("SELECT * FROM %s" % self.table, con=engine)
-
         return df.to_json(orient='index')
 
     def ttc_view_dates(self, serviced=False):
@@ -62,6 +60,9 @@ def ttc_view_dates(self, serviced=False):
         return df.to_json(orient='index')
 
     def ttc_to_days(self, dt):
+        """
+        Converts Unix time to days
+        """
         num_days = pd.Timedelta.total_seconds(dt)/(24.*3600)
         if num_days <= .000001:
             return 0
@@ -72,10 +73,10 @@ def ttc_to_days(self, dt):
     def ttc_days_to_string(self, day):
         return str(day) + " Days"
 
-    def ttc_time_diff(self, serviced=False, allRequests=True, requestType=""):
+    def ttc_time_diff(self, alldata, serviced, allRequests, requestType):
         """
-        Returns the amount of time for a request to close in days
-        If serviced is set to True, only requests that have been serviced are included
+        Sets self.data to a dataframe catalogging the time it takes a request to close
+        Parameters are inherited from ttc_summary()
         """
 
         engine = db.create_engine(self.dbString)
@@ -112,28 +113,44 @@ def ttc_time_diff(self, serviced=False, allRequests=True, requestType=""):
         for column in diff_df:
             diff_df[column] = diff_df[column].apply(self.ttc_to_days)
 
-        diff_df_str = diff_df.copy()
-
-        for column in diff_df_str:
-            diff_df_str[column] = diff_df_str[column].apply(
-                self.ttc_days_to_string)
-
         self.data = diff_df
-        return diff_df_str.to_json(orient='index')
 
-    def ttc_summary(self):
+    def ttc_summary(self, allData=False, serviced=False, allRequests=True, requestType=""):
+        """
+        Returns summary data of the amount of time it takes for a request to close as a dataframe
+        If serviced is set to True, returns summary data of time_to_service as well
+        If allData is set to True, returns the data of every entry as well
+        If allRequests are set to False, queries data of the value of requestType only
+        """
+        self.ttc_time_diff(allData, serviced, allRequests, requestType)
         data = self.data
 
-        summary_obj = {}
+        summary_arr = []
 
         for column in data:
             summary = data[column].describe()
-            summary_obj.update({column: summary.to_json()})
+            df_desc = pd.DataFrame({column: summary})
+            df_json = json.loads(df_desc.to_json())
+            summary_arr.append(df_json)
+
+        if allData:
+            days_df = data.copy()
+
+            for column in days_df:
+                days_df[column] = days_df[column].apply(
+                    self.ttc_days_to_string)
 
-        return json.dumps(summary_obj)
+            days_df_json = json.loads(days_df.to_json())
+            summary_arr.append(days_df_json)
 
-    # Todo: Implement functionality for only open status data
+        return summary_arr
 
+    # Todo: Change service/closed summary dfs into columns
+    # Todo: Stringify summary
+    # Todo: Change the meaning of service
+    # Todo: Add view_dates to summary option
+    # Todo: RequestType to self?
+    # Todo: Implement functionality for only open status data?
 
 if __name__ == "__main__":
     ttc = time_to_close()
@@ -142,3 +159,4 @@ def ttc_summary(self):
     ttc.config = config
     ttc.dbString = config['Database']['DB_CONNECTION_STRING']
     ttc.ttc_view_data()
+    ttc.ttc_summary()