diff --git a/flask_monitoringdashboard/__init__.py b/flask_monitoringdashboard/__init__.py index 53d1b5d0f..24ad8414e 100644 --- a/flask_monitoringdashboard/__init__.py +++ b/flask_monitoringdashboard/__init__.py @@ -14,6 +14,7 @@ import os +import traceback from flask import Blueprint from flask_monitoringdashboard.core.config import Config, TelemetryConfig @@ -29,7 +30,6 @@ def loc(): telemetry_config = TelemetryConfig() blueprint = Blueprint('dashboard', __name__, template_folder=loc() + 'templates') - def bind(app, schedule=True, include_dashboard=True): """Binding the app to this object should happen before importing the routing- methods below. Thus, the importing statement is part of this function. @@ -82,6 +82,22 @@ def bind(app, schedule=True, include_dashboard=True): # register the blueprint to the app app.register_blueprint(blueprint, url_prefix='/' + config.link) + # intercepts exceptions for dashboard purposes + def rec_strace(tb): + s = f"Endpoint: {tb.tb_frame.f_code.co_name} at line number: {tb.tb_lineno} in file: {tb.tb_frame.f_code.co_filename}\n" + if tb.tb_next is None: + return s + return rec_strace(tb.tb_next)+s + + def exc_intercept(): + old_print_exception = traceback.print_exception + def exc_log(etype, value, tb, limit=None, file=None): + print("åååhh neeej ikke igen") + print(rec_strace(tb)) + old_print_exception(etype, value, tb, limit, file) + traceback.print_exception = exc_log + exc_intercept() + # flush cache to db before shutdown import atexit from flask_monitoringdashboard.core.cache import flush_cache diff --git a/flask_monitoringdashboard/core/exception_logger.py b/flask_monitoringdashboard/core/exception_logger.py new file mode 100644 index 000000000..de20490ce --- /dev/null +++ b/flask_monitoringdashboard/core/exception_logger.py @@ -0,0 +1,28 @@ +import traceback +import linecache +from types import TracebackType +from flask_monitoringdashboard.core.types import ExcInfo +from flask_monitoringdashboard.database import CodeLine +from flask_monitoringdashboard.database.exception_info import add_exception_stack_line, add_exception_info + +def create_codeline(fs: traceback.FrameSummary): + c_line = CodeLine() + c_line.filename = fs.filename + c_line.line_number = fs.lineno + c_line.function_name = fs.name + c_line.code = linecache.getline(c_line.filename, c_line.line_number).strip() + return c_line + +class ExceptionLogger(): + def __init__(self, exc_info: ExcInfo): + self.type : type[BaseException] = exc_info[0] + self.value : BaseException = exc_info[1] + self.tb : TracebackType = exc_info[2] + + def log(self, request_id: int, session): + add_exception_info(session, request_id, str(self.type.__name__), str(self.value)) + for idx, fs in enumerate(traceback.extract_tb(self.tb)[1:]): + c_line = create_codeline(fs) + add_exception_stack_line(session, request_id, idx, c_line) + + diff --git a/flask_monitoringdashboard/core/measurement.py b/flask_monitoringdashboard/core/measurement.py index ed18c75e1..dcbe0856e 100644 --- a/flask_monitoringdashboard/core/measurement.py +++ b/flask_monitoringdashboard/core/measurement.py @@ -2,8 +2,14 @@ Contains all functions that are used to track the performance of the flask-application. See init_measurement() for more detailed info. """ +import linecache +import sys import time from functools import wraps +import traceback +from typing import Any, cast, TYPE_CHECKING +from flask_monitoringdashboard.core.exception_logger import ExceptionLogger +from flask_monitoringdashboard.core.types import ExcInfo, OptExcInfo from werkzeug.exceptions import HTTPException from flask_monitoringdashboard import config @@ -17,6 +23,9 @@ from flask_monitoringdashboard.database import session_scope from flask_monitoringdashboard.database.endpoint import get_endpoint_by_name +def print_fs(lst: list[traceback.FrameSummary]): + for fs in lst: + print(f"code: {linecache.getline(fs.filename, fs.lineno)}, name {fs.name}") def init_measurement(): """ @@ -67,7 +76,7 @@ def is_valid_status_code(status_code): return type(status_code) == int and 100 <= status_code < 600 -def status_code_from_response(result): +def status_code_from_response(result) -> int: """ Extracts the status code from the result that was returned from the route handler. @@ -94,6 +103,13 @@ def status_code_from_response(result): return status_code +def ptb(tb): + fsl : list[traceback.FrameSummary] = traceback.extract_tb(tb) + sl : list[traceback.FrameSummary] = traceback.extract_stack() + print("hitttttttttttttttttttttt") + print_fs(sl) + print("hitttttttttttttttttttttt") + print_fs(fsl) def evaluate(route_handler, args, kwargs): """ @@ -110,9 +126,11 @@ def evaluate(route_handler, args, kwargs): return result, status_code, None except HTTPException as e: - return None, e.code, e - except Exception as e: - return None, 500, e + exc_info : OptExcInfo = sys.exc_info() + return None, e.code, (ExceptionLogger(exc_info) if exc_info[0] is not None else None) + except Exception as _: + exc_info = sys.exc_info() + return None, 500, (ExceptionLogger(exc_info) if exc_info[0] is not None else None) def add_wrapper1(endpoint, fun): @@ -120,13 +138,13 @@ def add_wrapper1(endpoint, fun): def wrapper(*args, **kwargs): start_time = time.time() - result, status_code, raised_exception = evaluate(fun, args, kwargs) + result, status_code, e_logger = evaluate(fun, args, kwargs) duration = time.time() - start_time - start_performance_thread(endpoint, duration, status_code) + start_performance_thread(endpoint, duration, status_code, e_logger) - if raised_exception: - raise raised_exception + if e_logger: + raise e_logger.value return result @@ -140,13 +158,13 @@ def wrapper(*args, **kwargs): outlier = start_outlier_thread(endpoint) start_time = time.time() - result, status_code, raised_exception = evaluate(fun, args, kwargs) + result, status_code, e_logger = evaluate(fun, args, kwargs) duration = time.time() - start_time outlier.stop(duration, status_code) - if raised_exception: - raise raised_exception + if e_logger: + raise e_logger.value return result @@ -160,13 +178,13 @@ def wrapper(*args, **kwargs): thread = start_profiler_and_outlier_thread(endpoint) start_time = time.time() - result, status_code, raised_exception = evaluate(fun, args, kwargs) + result, status_code, e_logger = evaluate(fun, args, kwargs) duration = time.time() - start_time thread.stop(duration, status_code) - if raised_exception: - raise raised_exception + #if raised_exception: + # raise raised_exception return result diff --git a/flask_monitoringdashboard/core/profiler/__init__.py b/flask_monitoringdashboard/core/profiler/__init__.py index 8cec57ce8..8ec034eff 100644 --- a/flask_monitoringdashboard/core/profiler/__init__.py +++ b/flask_monitoringdashboard/core/profiler/__init__.py @@ -16,7 +16,7 @@ def start_thread_last_requested(endpoint): BaseProfiler(endpoint).start() -def start_performance_thread(endpoint, duration, status_code): +def start_performance_thread(endpoint, duration, status_code, e_logger): """ Starts a thread that updates performance, utilization and last_requested in the database. :param endpoint: Endpoint object @@ -24,7 +24,7 @@ def start_performance_thread(endpoint, duration, status_code): :param status_code: HTTP status code of the request """ group_by = get_group_by() - PerformanceProfiler(endpoint, get_ip(), duration, group_by, status_code).start() + PerformanceProfiler(endpoint, get_ip(), duration, group_by, e_logger, status_code).start() def start_profiler_thread(endpoint): diff --git a/flask_monitoringdashboard/core/profiler/outlier_profiler.py b/flask_monitoringdashboard/core/profiler/outlier_profiler.py index a203fd963..c9c7af971 100644 --- a/flask_monitoringdashboard/core/profiler/outlier_profiler.py +++ b/flask_monitoringdashboard/core/profiler/outlier_profiler.py @@ -7,6 +7,7 @@ from flask_monitoringdashboard import config from flask_monitoringdashboard.core.cache import update_duration_cache, get_avg_endpoint +from flask_monitoringdashboard.core.exception_logger import ExceptionLogger from flask_monitoringdashboard.core.logger import log from flask_monitoringdashboard.database import session_scope from flask_monitoringdashboard.database.outlier import add_outlier @@ -57,7 +58,7 @@ def run(self): self._cpu_percent = str(psutil.cpu_percent(interval=None, percpu=True)) self._memory = str(psutil.virtual_memory()) - def stop(self, duration, status_code): + def stop(self, duration, status_code, e_logger : ExceptionLogger): self._exit.set() update_duration_cache(endpoint_name=self._endpoint.name, duration=duration * 1000) with session_scope() as session: @@ -69,6 +70,7 @@ def stop(self, duration, status_code): group_by=self._group_by, status_code=status_code, ) + e_logger.log(request_id, session) if self._memory: add_outlier( session, diff --git a/flask_monitoringdashboard/core/profiler/performance_profiler.py b/flask_monitoringdashboard/core/profiler/performance_profiler.py index 5144751ea..68c316556 100644 --- a/flask_monitoringdashboard/core/profiler/performance_profiler.py +++ b/flask_monitoringdashboard/core/profiler/performance_profiler.py @@ -2,6 +2,7 @@ from flask_monitoringdashboard.core.profiler.base_profiler import BaseProfiler from flask_monitoringdashboard.database import session_scope from flask_monitoringdashboard.database.request import add_request +from flask_monitoringdashboard.core.exception_logger import ExceptionLogger class PerformanceProfiler(BaseProfiler): @@ -10,18 +11,19 @@ class PerformanceProfiler(BaseProfiler): Used when monitoring-level == 1 """ - def __init__(self, endpoint, ip, duration, group_by, status_code=200): + def __init__(self, endpoint, ip, duration, group_by, e_logger, status_code=200): super(PerformanceProfiler, self).__init__(endpoint) self._ip = ip self._duration = duration * 1000 # Conversion from sec to ms self._endpoint = endpoint self._group_by = group_by self._status_code = status_code + self.e_logger = e_logger def run(self): update_duration_cache(endpoint_name=self._endpoint.name, duration=self._duration) with session_scope() as session: - add_request( + rid = add_request( session, duration=self._duration, endpoint_id=self._endpoint.id, @@ -29,3 +31,5 @@ def run(self): group_by=self._group_by, status_code=self._status_code, ) + if self.e_logger is not None: + self.e_logger.log(rid, session) diff --git a/flask_monitoringdashboard/core/types.py b/flask_monitoringdashboard/core/types.py new file mode 100644 index 000000000..3ae5ed777 --- /dev/null +++ b/flask_monitoringdashboard/core/types.py @@ -0,0 +1,7 @@ + +from types import TracebackType +from typing import TypeAlias + +ExcInfo: TypeAlias = tuple[type[BaseException], BaseException, TracebackType] +OptExcInfo: TypeAlias = ExcInfo | tuple[None, None, None] + diff --git a/flask_monitoringdashboard/database/__init__.py b/flask_monitoringdashboard/database/__init__.py index 8bf545af3..a00eb4226 100644 --- a/flask_monitoringdashboard/database/__init__.py +++ b/flask_monitoringdashboard/database/__init__.py @@ -51,7 +51,7 @@ class User(Base): """False for guest permissions (only view access). True for admin permissions.""" def set_password(self, password): - self.password_hash = generate_password_hash(password) + self.password_hash = generate_password_hash(password, method='pbkdf2') def check_password(self, password): return check_password_hash(self.password_hash, password) @@ -238,6 +238,36 @@ class CustomGraphData(Base): value = Column(Float) """Actual value that is measured.""" + +class ExceptionInfo(Base): + """Table for storing exception id together with request id.""" + + __tablename__ = '{}ExceptionInfo'.format(config.table_prefix) + + request_id = Column(Integer, ForeignKey(Request.id), primary_key=True) + request = relationship(Request) + + exception_type = Column(String(1500), nullable=False) + exception_msg = Column(String(1500), nullable=False) + + stack_lines = relationship('ExceptionStackLine', backref='exception_info') + + +class ExceptionStackLine(Base): + """Table for storing exception id together with request id.""" + + __tablename__ = '{}ExceptionStackLine'.format(config.table_prefix) + + request_id = Column(Integer, ForeignKey(ExceptionInfo.request_id), primary_key=True) + """Request that belongs to this exc_stack_line.""" + + code_id = Column(Integer, ForeignKey(CodeLine.id)) + code = relationship(CodeLine) + """Corresponding codeline.""" + + position = Column(Integer, primary_key=True) + """Position in the flattened stack tree.""" + # define the database engine = create_engine(config.database_name) @@ -284,4 +314,4 @@ def row2dict(row): def get_tables(): - return [Endpoint, Request, Outlier, StackLine, CodeLine, CustomGraph, CustomGraphData] + return [Endpoint, Request, Outlier, StackLine, CodeLine, CustomGraph, CustomGraphData, ExceptionInfo, ExceptionStackLine] diff --git a/flask_monitoringdashboard/database/exception_info.py b/flask_monitoringdashboard/database/exception_info.py new file mode 100644 index 000000000..1f9b910c4 --- /dev/null +++ b/flask_monitoringdashboard/database/exception_info.py @@ -0,0 +1,50 @@ +""" +Contains all functions that access an ExceptionInfo object. +""" + +from flask_monitoringdashboard.database import CodeLine, ExceptionInfo, ExceptionStackLine +from flask_monitoringdashboard.database.code_line import get_code_line + +def get_exception_info(session, request_id: int): + """ + Retrieve an ExceptionInfo record by request_id. + """ + return session.query(ExceptionInfo).filter_by(request_id=request_id).first() + + +def add_exception_info(session, request_id: int, exception_type: str, exception_msg: str): + """ + Add a new ExceptionInfo record. + """ + exception_info = ExceptionInfo( + request_id=request_id, + exception_type=exception_type, + exception_msg=exception_msg + ) + session.add(exception_info) + session.commit() + return exception_info + +def add_exception_stack_line(session, request_id, position, code_line: CodeLine): + """ + Adds a StackLine to the database (and possibly a CodeLine) + :param session: Session for the database + :param request_id: id of the request + :param position: position of the StackLine + :param indent: indent-value + :param duration: duration of this line (in ms) + :param code_line: quadruple that consists of: (filename, line_number, function_name, code) + """ + #fn, ln, name, code = code_line + fn = code_line.filename + ln = code_line.line_number + name = code_line.function_name + code = code_line.code + db_code_line = get_code_line(session, fn, ln, name, code) + session.add( + ExceptionStackLine( + request_id=request_id, + position=position, + code_id=db_code_line.id, + ) + ) diff --git a/flask_monitoringdashboard/main.py b/flask_monitoringdashboard/main.py index 2ee0f728e..704246dbd 100644 --- a/flask_monitoringdashboard/main.py +++ b/flask_monitoringdashboard/main.py @@ -82,6 +82,25 @@ def endpoint5(): time.sleep(0.2) return 'Ok' +def a(): + raise Exception("åhhh nej") + +def b(): + return a() + +def c(): + return b() + +def d(): + return c + +@app.route('/throws') +def throws(): + time.sleep(0.2) + d()() + return 'Ok' + + def my_func(): # here should be something actually useful @@ -90,4 +109,4 @@ def my_func(): if __name__ == "__main__": dashboard.bind(app) - app.run() + app.run(port=4200)