diff --git a/db.py b/db.py index ae2ed19..05850a0 100644 --- a/db.py +++ b/db.py @@ -31,6 +31,10 @@ def find_one(self, table, cond): ''' return self.table(table).find_one(cond) + def find_sections(self, table, cond, sections, key): + + return self.table(table).find(cond, sections).sort(key) + def find(self, table, cond): ''' Find records. diff --git a/web/api.py b/web/api.py new file mode 100644 index 0000000..073c35e --- /dev/null +++ b/web/api.py @@ -0,0 +1,180 @@ +__all__ = [ + "CommitRecord", + "TaskRecord", + "KpiRecord", +] + +import sys +sys.path.append('pypage') +sys.path.append('..') +import config +import json +from db import MongoDB +from datetime import datetime, timedelta +from kpi import Kpi + +db = MongoDB(config.db_name) + + +class objdict(dict): + def __setattr__(self, key, value): + self[key] = value + + def __getattr__(self, item): + return self[item] + + +class CommitRecord: + def __init__(self, commit=''): + self.commit = commit + self.short_commit = "" + self.date = None # datetime + self.info = "" + + @staticmethod + def get_all(): + ''' Get all commit records, and sort by latest to oldest. + returns: list of CommitRecord + ''' + # sort by 'date' in ascending order + commits = db.find_sections(config.table_name, + {'type': 'kpi'}, {'commitid': 1, "_id": 0}, "date") + commit_ids = [] + for commit in commits: + if commit['commitid'] not in commit_ids: + commit_ids.append(commit['commitid']) + + records = [] + for commit in commit_ids: + commitobj = CommitRecord(commit) + tasks = commitobj.__get_db_record() + commitobj.commit = commit + commitobj.shortcommit = commit[:7] + commitobj.date = datetime.utcfromtimestamp(int(tasks[0]['date'])) + \ + timedelta(hours=8) + + commitobj.passed = tasks_success(tasks) + records.append(commitobj) + return records + + @staticmethod + def get_tasks(commit): + ''' Get the task details belong to a commit. + returns: dict of TaskRecord + keys: task name, + values: TaskRecord ''' + record = CommitRecord(commit) + tasks = record.__get_db_record() + print (tasks) + res = objdict() + for task in tasks: + taskobj = TaskRecord(commit, task['task'], task['infos'], + task['passed']) + taskobj.kpis = taskobj.get_kpis() + res[taskobj.name] = taskobj + return res + + def __get_db_record(self): + ''' get the corresponding tasks from database. + ''' + return db.finds(config.table_name, + {'type': 'kpi', + 'commitid': self.commit}) + + +class TaskRecord(objdict): + def __init__(self, commit, name, infos, passed): + self.name = name + self.task = name + # dict of KpiRecord + self.kpis = [] + self.infos = infos + self.passed = passed + self.commitid = commit + + def get_kpis(self): + '''Get kpis details belong to a task. + returns dict of KpiRecord + keys: kpi name, + values: KpiRecord''' + task_info = self.__get_db_record() + kpi_infos = {} + for kpi in task_info['kpis-keys']: + kpiobj = KpiRecord(kpi) + kpi_infos[kpi] = kpiobj.get_kpi_info(task_info) + return kpi_infos + + def __get_db_record(self): + ''' get the corresponding kpis from database''' + return db.find_one(config.table_name, {'type': 'kpi', \ + 'commitid': self.commitid, 'task': self.name}) + +class KpiRecord: + def __init__(self, name): + self.name = name + # list of list of float + self.values = [] + self.type = "" + self.avg = 0 + self.activeds = False + self.unit = "" + self.desc = "" + + def get_kpi_info(self, task_info): + '''Get the kpi infos according to the kpi name''' + for i in range(len(task_info['kpis-keys'])): + if self.name == task_info['kpis-keys'][i]: + break + def safe_get_fields(field): + if field in task_info: + return task_info[field] + return None + #To keep the kpi datas in order, we should process the data one by one. + kpi_vals = json.loads(task_info['kpis-values']) + self.values = kpi_vals[i] + self.type = task_info['kpi-types'][i] + self.avg = '%.4f' % Kpi.dic.get(self.type).cal_kpi(data=kpi_vals[i]) + infos = parse_infos(task_info['infos']) + self.info = infos[self.name] + + activeds = safe_get_fields('kpi-activeds') + self.activeds = activeds[i] if activeds else True + + unit_reprs = safe_get_fields('kpi-unit-reprs') + self.unit = "(%s)" % unit_reprs[i] if unit_reprs else "" + + descs = safe_get_fields('kpi-descs') + self.desc = descs[i] if descs else "" + + return (self.values, self.type, self.avg, self.info, self.activeds, + self.unit, self.desc) + + +class objdict(dict): + def __setattr__(self, key, value): + self[key] = value + + def __getattr__(self, item): + return self[item] + + +def parse_infos(infos): + ''' + input format: [kpi0] xxxx [kpi1] xxx + + return dic of (kpi, info) + ''' + res = {} + for info in infos: + lb = info.find('[') + 1 + rb = info.find(']', lb) + kpi = info[lb:rb] + info = info[rb + 2:] + res[kpi] = info + return res + + +def tasks_success(tasks): + for task in tasks: + if not task['passed']: return False + return True diff --git a/web/main.py b/web/main.py index 6850511..d0d725f 100644 --- a/web/main.py +++ b/web/main.py @@ -10,6 +10,7 @@ import pprint from kpi import Kpi from view import * +from api import * import pyecharts SERVER_PATH = os.path.abspath(os.path.dirname(sys.argv[0])) @@ -33,7 +34,7 @@ def index(): a list of commitids and their status(passed or not, the info) ''' page, snips = build_index_page() - commits = get_commits() + commits = CommitRecord.get_all() latest_commit = commits[-1].commit logics = merge_logics(snips[0].logic(), snips[1].logic(latest_commit)) print('commits', snips[0].logic()) @@ -41,7 +42,7 @@ def index(): @app.route('/commit/details', methods=["GET"]) -@cache.cached(timeout=120) +#@cache.cached(timeout=5) def commit_details(): commit = request.args.get('commit') @@ -52,10 +53,10 @@ def commit_details(): @app.route('/commit/compare', methods=["GET"]) -@cache.cached(timeout=120) +#@cache.cached(timeout=5) def commit_compare(): if 'cur' not in request.args: - commits = get_commits() + commits = CommitRecord.get_all() latest_commit = commits[-1] success_commits = [v for v in filter(lambda r: r.passed, commits)] latest_success_commit = success_commits[ diff --git a/web/view.py b/web/view.py index 02bf388..d5e5830 100644 --- a/web/view.py +++ b/web/view.py @@ -4,6 +4,7 @@ import config import json from db import MongoDB +from api import * from pypage import * from pypage import layout as lyt from datetime import datetime, timedelta @@ -165,7 +166,7 @@ def html(self): alert(c=VAL('kpi[3]')).set_danger() def logic(self, commitid): - task_kpis = query_commit_from_db(commitid) + task_kpis = CommitRecord.get_tasks(commitid) res = objdict(version=dict( commit=commitid, passed=tasks_success(task_kpis), @@ -204,7 +205,7 @@ def html(self): RawHtml('
') def logic(self): - records_ = get_commits() + records_ = CommitRecord.get_all() return {self.KEY('records'): records_} @@ -236,7 +237,7 @@ def html(self): Tag('span', VAL('commit.date')) def logic(self): - commits = get_commits() + commits = CommitRecord.get_all() return {self.KEY('commits'): [v for v in reversed(commits)], } @@ -291,15 +292,15 @@ def logic(self, cur_commit, base_commit): print('cur', cur_commit) print('base', base_commit) - cur_rcds = query_commit_from_db(cur_commit) - base_rcds = query_commit_from_db(base_commit) - + cur_rcds = CommitRecord.get_tasks(cur_commit) + base_rcds = CommitRecord.get_tasks(base_commit) res = [] for name in cur_rcds.keys(): cur_task = cur_rcds.get(name, None) + base_task = base_rcds.get(name, None) # if eithor do not have some task, skip it. - if not (cur_task or base_task): continue + if not (cur_task and base_task): continue record = objdict() res.append(record) @@ -308,7 +309,7 @@ def logic(self, cur_commit, base_commit): for kpi in cur_task.kpis.keys(): cur_kpi = cur_task.kpis.get(kpi, None) base_kpi = base_task.kpis.get(kpi, None) - if not (cur_kpi or base_kpi): continue + if not (cur_kpi and base_kpi): continue kpi_ = objdict() kpi_type = Kpi.dic.get(cur_kpi[1]) @@ -347,10 +348,11 @@ def html(self): def logic(self): # should be sorted by freshness - commits = get_commits() + commits = CommitRecord.get_all() kpis = {} - for commit in commits: - rcd = query_commit_from_db(commit.commit) + last_N_commit = commits[-self.N:-1] + [commits[-1]] + for commit in last_N_commit: + rcd = CommitRecord.get_tasks(commit.commit) if self.task_name not in rcd: continue for (kpi,val) in rcd[self.task_name].kpis.items(): kpis.setdefault(kpi+'--x', []).append(commit.shortcommit) @@ -369,68 +371,6 @@ def passed_commits(): pass -def get_commit_py_records(commit): - ''' Get python records belonging to commit. ''' - records = query_commit_from_db(commit) - #print('records.values', records.values()) - return [db_task_record_to_py(r) for r in records.values()] - - -def query_commit_from_db(commit): - ''' Get the task details belong to a commit from the database. ''' - tasks = db.finds(config.table_name, {'type': 'kpi', 'commitid': commit}) - res = objdict() - for task in tasks: - task = db_task_record_to_py(task) - task['task'] = task.name - res[task.name] = task - return res - - -def db_task_record_to_py(task_rcd): - ''' Transfrom a mongodb task record to python record. All the fields should be - transformed.''' - task = objdict( - name=task_rcd['task'], - passed=task_rcd['passed'], - commitid=task_rcd['commitid'], ) - - def safe_get_fields(field): - if field in task_rcd: - print('task_rcd', task_rcd[field]) - return task_rcd[field] - # return json.loads(task_rcd[field]) - return None - - kpi_vals = json.loads(task_rcd['kpis-values']) - task.kpis = {} - infos = parse_infos(task_rcd['infos']) - activeds = safe_get_fields('kpi-activeds') - unit_reprs = safe_get_fields('kpi-unit-reprs') - descs = safe_get_fields('kpi-descs') - - for i in range(len(task_rcd['kpis-keys'])): - kpi_type = Kpi.dic.get(task_rcd['kpi-types'][i]) - kpi = task_rcd['kpis-keys'][i] - task.kpis[kpi] = ( - # kpi details - kpi_vals[i], - # type - task_rcd['kpi-types'][i], - # kpi - '%.4f' % kpi_type.cal_kpi(data=kpi_vals[i]), - # info - infos[kpi], - # actived - activeds[i] if activeds else True, - # unit repr - "(%s)" % unit_reprs[i] if unit_reprs else "", - # desc - descs[i] if descs else "", ) - task.infos = task_rcd['infos'] - return task - - class objdict(dict): def __setattr__(self, key, value): self[key] = value @@ -443,49 +383,8 @@ def __getattr__(self, item): exit(1) -def parse_infos(infos): - ''' - input format: [kpi0] xxxx [kpi1] xxx - - return dic of (kpi, info) - ''' - res = {} - for info in infos: - lb = info.find('[') + 1 - rb = info.find(']', lb) - kpi = info[lb:rb] - info = info[rb + 2:] - res[kpi] = info - return res - def tasks_success(tasks): for task in tasks.values(): if not task['passed']: return False return True - - -def get_commits(cond={'type': 'kpi'}): - ''' get all the commits ''' - records = db.finds(config.table_name, cond) - - # detact whether the task is passed. - commits = {} - for task in records: - rcd = db_task_record_to_py(task) - commits.setdefault(rcd.commitid, {})[rcd.name] = rcd - - records_ = [] - commit_set = set() - for rcd in records: - if rcd['commitid'] not in commit_set: - commit_set.add(rcd['commitid']) - rcd_ = objdict() - rcd_.commit = rcd['commitid'] - rcd_.shortcommit = rcd['commitid'][:7] - rcd_.date = datetime.utcfromtimestamp(int(rcd['date'])) + \ - timedelta(hours=8) - rcd_.passed = tasks_success(commits[rcd_.commit]) - records_.append(rcd_) - - return records_