From f9066bf42296bcb04810cdbf31c2f3c5c4793ae2 Mon Sep 17 00:00:00 2001 From: Jordan Padams Date: Fri, 2 Nov 2018 15:40:27 -0700 Subject: [PATCH] Issue #92: Update monitoring with notitication throttling --- ait/gui/__init__.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/ait/gui/__init__.py b/ait/gui/__init__.py index ece8c1c8..f0dc97c7 100644 --- a/ait/gui/__init__.py +++ b/ait/gui/__init__.py @@ -391,6 +391,9 @@ def telem_handler(session): log.info('Starting telemetry limit monitoring') try: + limit_trip_repeat = 0 + prev_v = None + prev_defn = None while True: if len(session.telemetry) > 0: p = session.telemetry.popleft() @@ -407,11 +410,28 @@ def telem_handler(session): if defn.error(v): msg = 'Field {} error out of limit with value {}'.format(field, v) log.error(msg) - notify.trigger_notification('limit-error', msg) + + if limit_trip_repeat < 30 and v == prev_v and defn == prev_defn: + limit_trip_repeat += 1 + else: + limit_trip_repeat = 0 + notify.trigger_notification('limit-error', msg) + + prev_v = v + prev_defn = defn + elif defn.warn(v): msg = 'Field {} warning out of limit with value {}'.format(field, v) log.warn(msg) - notify.trigger_notification('limit-warn', msg) + + if limit_trip_repeat < 30 and v == prev_v and defn == prev_defn: + limit_trip_repeat += 1 + else: + limit_trip_repeat = 0 + notify.trigger_notification('limit-warn', msg) + + prev_v = v + prev_defn = defn gevent.sleep(0) finally: