From b93fddf5e95f8914fdf93460e6e8e8025de40131 Mon Sep 17 00:00:00 2001 From: rgao Date: Mon, 6 Jan 2020 02:43:31 -0800 Subject: [PATCH 1/2] updated timetoclose to add viewing all entries as an option to summary, began frequency data parsing --- server/src/app.py | 16 ++++----- server/src/services/frequency.py | 30 +++++++++++----- server/src/services/time_to_close.py | 52 +++++++++++++++++++--------- 3 files changed, 64 insertions(+), 34 deletions(-) diff --git a/server/src/app.py b/server/src/app.py index 4a607304c..6e58134f7 100644 --- a/server/src/app.py +++ b/server/src/app.py @@ -32,21 +32,19 @@ async def index(request): @app.route('/timetoclose') async def timetoclose(request): ttc_worker = time_to_close(app.config['Settings']) - data = [] - time_diff = loads(ttc_worker.ttc_time_diff(serviced=True, allRequests=False, requestType="'Bulky Items'")) - summary = loads(ttc_worker.ttc_summary()) + # dates = loads(ttc_worker.ttc_view_dates()) + summary = ttc_worker.ttc_summary(allData=True, serviced=False, allRequests=False, requestType="'Bulky Items'") - data.append(time_diff) - data.append(summary) - return json(data) + # return json(dates) + return json(summary) @app.route('/requestfrequency') async def requestfrequency(request): - freq_worker = frequency() - # Insert frequency calculation here - return_data = freq_worker.freq_query() + freq_worker = frequency(app.config['Settings']) + + return_data = loads(freq_worker.freq_summary()) return json(return_data) diff --git a/server/src/services/frequency.py b/server/src/services/frequency.py index 411ab85f9..2c8f2e07a 100644 --- a/server/src/services/frequency.py +++ b/server/src/services/frequency.py @@ -1,15 +1,20 @@ from configparser import ConfigParser -from sqlalchemy.types import Integer, Text, String, DateTime, Float -from sqlalchemy import create_engine +import sqlalchemy as db +import pandas as pd +from datetime import datetime as dt +import numpy as np +import json + class frequency(object): - def __init__(self, config=None): + def __init__(self, config=None, tableName="ingest_staging_table"): self.config = config self.dbString = None if not self.config else self.config['Database']['DB_CONNECTION_STRING'] + self.table = tableName pass def freq_query(self): - engine = create_engine(self.dbString) + engine = db.create_engine(self.dbString) connection = engine.connect() query = "SELECT row_to_json(row(status)) \ @@ -19,10 +24,19 @@ def freq_query(self): return result + def freq_summary(self): + engine = db.create_engine(self.dbString) + + query = "SELECT * FROM %s" % (self.table) + df = pd.read_sql_query(query, con=engine) + + return df.describe().to_json() + + if __name__ == "__main__": - frequer = frequency() + freq = frequency() config = ConfigParser() config.read("../setting.cfg") - frequer.config = config - frequer.dbString = config['Database']['DB_CONNECTION_STRING'] - frequer.freq_query() + freq.config = config + freq.dbString = config['Database']['DB_CONNECTION_STRING'] + freq.freq_summary() diff --git a/server/src/services/time_to_close.py b/server/src/services/time_to_close.py index edefbb9af..12905a9c6 100644 --- a/server/src/services/time_to_close.py +++ b/server/src/services/time_to_close.py @@ -37,8 +37,6 @@ def ttc_view_data(self, onlyClosed=False): else: df = pd.read_sql_query("SELECT * FROM %s" % self.table, con=engine) - # df = pd.read_sql_query("SELECT * FROM %s" % self.table, con=engine) - return df.to_json(orient='index') def ttc_view_dates(self, serviced=False): @@ -62,6 +60,9 @@ def ttc_view_dates(self, serviced=False): return df.to_json(orient='index') def ttc_to_days(self, dt): + """ + Converts Unix time to days + """ num_days = pd.Timedelta.total_seconds(dt)/(24.*3600) if num_days <= .000001: return 0 @@ -72,10 +73,10 @@ def ttc_to_days(self, dt): def ttc_days_to_string(self, day): return str(day) + " Days" - def ttc_time_diff(self, serviced=False, allRequests=True, requestType=""): + def ttc_time_diff(self, alldata, serviced, allRequests, requestType): """ - Returns the amount of time for a request to close in days - If serviced is set to True, only requests that have been serviced are included + Sets self.data to a dataframe catalogging the time it takes a request to close + Parameters are inherited from ttc_summary() """ engine = db.create_engine(self.dbString) @@ -112,28 +113,44 @@ def ttc_time_diff(self, serviced=False, allRequests=True, requestType=""): for column in diff_df: diff_df[column] = diff_df[column].apply(self.ttc_to_days) - diff_df_str = diff_df.copy() - - for column in diff_df_str: - diff_df_str[column] = diff_df_str[column].apply( - self.ttc_days_to_string) - self.data = diff_df - return diff_df_str.to_json(orient='index') - def ttc_summary(self): + def ttc_summary(self, allData=False, serviced=False, allRequests=True, requestType=""): + """ + Returns summary data of the amount of time it takes for a request to close as a dataframe + If serviced is set to True, returns summary data of time_to_service as well + If allData is set to True, returns the data of every entry as well + If allRequests are set to False, queries data of the value of requestType only + """ + self.ttc_time_diff(allData, serviced, allRequests, requestType) data = self.data - summary_obj = {} + summary_arr = [] for column in data: summary = data[column].describe() - summary_obj.update({column: summary.to_json()}) + df_desc = pd.DataFrame({column: summary}) + df_json = json.loads(df_desc.to_json()) + summary_arr.append(df_json) + + if allData: + days_df = data.copy() + + for column in days_df: + days_df[column] = days_df[column].apply( + self.ttc_days_to_string) - return json.dumps(summary_obj) + days_df_json = json.loads(days_df.to_json()) + summary_arr.append(days_df_json) - # Todo: Implement functionality for only open status data + return summary_arr + # Todo: Change service/closed summary dfs into columns + # Todo: Stringify summary + # Todo: Change the meaning of service + # Todo: Add view_dates to summary option + # Todo: RequestType to self? + # Todo: Implement functionality for only open status data? if __name__ == "__main__": ttc = time_to_close() @@ -142,3 +159,4 @@ def ttc_summary(self): ttc.config = config ttc.dbString = config['Database']['DB_CONNECTION_STRING'] ttc.ttc_view_data() + ttc.ttc_summary() From c8b7ec4832e95720557da23be3d03ee89559d46f Mon Sep 17 00:00:00 2001 From: rgao Date: Mon, 13 Jan 2020 12:35:42 -0800 Subject: [PATCH 2/2] implemented basic functionality for frequency request module --- server/src/app.py | 6 ++++-- server/src/services/frequency.py | 33 +++++++++++++++++++++----------- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/server/src/app.py b/server/src/app.py index 6e58134f7..d5713e881 100644 --- a/server/src/app.py +++ b/server/src/app.py @@ -33,9 +33,11 @@ async def index(request): async def timetoclose(request): ttc_worker = time_to_close(app.config['Settings']) + # data = loads(ttc_worker.ttc_view_data()) # dates = loads(ttc_worker.ttc_view_dates()) summary = ttc_worker.ttc_summary(allData=True, serviced=False, allRequests=False, requestType="'Bulky Items'") + # return json(data_arr) # return json(dates) return json(summary) @@ -44,9 +46,9 @@ async def timetoclose(request): async def requestfrequency(request): freq_worker = frequency(app.config['Settings']) - return_data = loads(freq_worker.freq_summary()) + summary = loads(freq_worker.freq_view_all()) - return json(return_data) + return json(summary) @app.route('/sample-data') diff --git a/server/src/services/frequency.py b/server/src/services/frequency.py index 2c8f2e07a..97135265d 100644 --- a/server/src/services/frequency.py +++ b/server/src/services/frequency.py @@ -11,26 +11,37 @@ def __init__(self, config=None, tableName="ingest_staging_table"): self.config = config self.dbString = None if not self.config else self.config['Database']['DB_CONNECTION_STRING'] self.table = tableName + self.data = None pass - def freq_query(self): + def freq_view_all(self, serviced=False): + """ + Returns the request type and associated dates for all data + Sorted by request type, followed by created date, service date (if applicable), and then closed date + """ + # Todo: implement condition for serviced date engine = db.create_engine(self.dbString) - connection = engine.connect() - query = "SELECT row_to_json(row(status)) \ - FROM ingest_staging_table" - result = connection.execute(query) - connection.close() + query = "SELECT requesttype, createddate, closeddate FROM %s" % self.table + df = pd.read_sql_query(query, con=engine) + + df['closeddate'] = pd.to_datetime(df['closeddate']) + df = df.sort_values(by=['requesttype', 'createddate', 'closeddate']) - return result + return df.to_json(orient="records") - def freq_summary(self): + def freq_aggregate(self): engine = db.create_engine(self.dbString) - query = "SELECT * FROM %s" % (self.table) + query = "SELECT requesttype FROM %s" % (self.table) df = pd.read_sql_query(query, con=engine) - return df.describe().to_json() + request_counts = df['requesttype'].value_counts() + print(np.dtype(request_counts)) + + # for request in request_types: + # print(df[request].value_counts()) + return df['requesttype'].value_counts() if __name__ == "__main__": @@ -39,4 +50,4 @@ def freq_summary(self): config.read("../setting.cfg") freq.config = config freq.dbString = config['Database']['DB_CONNECTION_STRING'] - freq.freq_summary() + freq.freq_aggregate()