From 8952115dab92a3e9df5688a67067b0418246166e Mon Sep 17 00:00:00 2001 From: whelan Date: Tue, 3 Dec 2019 18:52:57 +0800 Subject: [PATCH 1/4] Support application insights(prototype). --- requirements.txt | 4 ++++ setup.py | 1 + superset/utils/log.py | 11 ++++++++++- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c9d0d4c529cce..d0cac1aaefd38 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,7 @@ alembic==1.0.11 # via flask-migrate amqp==2.5.0 # via kombu apispec[yaml]==1.3.3 # via flask-appbuilder +applicationinsights==0.11.9 asn1crypto==0.24.0 # via cryptography attrs==19.1.0 # via jsonschema babel==2.7.0 # via flask-babel @@ -82,3 +83,6 @@ webencodings==0.5.1 # via bleach werkzeug==0.15.5 # via flask, flask-jwt-extended wtforms-json==0.3.3 wtforms==2.2.1 # via flask-wtf, wtforms-json + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/setup.py b/setup.py index 5f616a215ae12..60588b80905ac 100644 --- a/setup.py +++ b/setup.py @@ -106,6 +106,7 @@ def get_git_sha(): "sqlalchemy-utils>=0.33.2", "sqlparse>=0.3.0,<0.4", "wtforms-json", + "applicationinsights", ], extras_require={ "bigquery": ["pybigquery>=0.4.10", "pandas_gbq>=0.10.0"], diff --git a/superset/utils/log.py b/superset/utils/log.py index 85d41398c9287..72cb43d503f0a 100644 --- a/superset/utils/log.py +++ b/superset/utils/log.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. # pylint: disable=C,R,W +import os import functools import inspect import json @@ -26,6 +27,8 @@ from flask import current_app, g, request +from applicationinsights import TelemetryClient + class AbstractEventLogger(ABC): @abstractmethod @@ -125,8 +128,13 @@ def get_event_logger_from_cfg_value(cfg_value: object) -> AbstractEventLogger: logging.info(f"Configured event logger of type {type(result)}") return cast(AbstractEventLogger, result) - +INSTRUMENTATION_KEY=os.environ.get('INSTRUMENTATION_KEY') +tc = TelemetryClient(INSTRUMENTATION_KEY) class DBEventLogger(AbstractEventLogger): + def appinsights(self, data): + tc.track_event('superset event', { 'data': data }) + tc.flush() + def log(self, user_id, action, *args, **kwargs): from superset.models.core import Log @@ -152,6 +160,7 @@ def log(self, user_id, action, *args, **kwargs): user_id=user_id, ) logs.append(log) + self.appinsights(data=log) sesh = current_app.appbuilder.get_session sesh.bulk_save_objects(logs) From d97f721c1b2cc25038290c8f2b962bba474ee754 Mon Sep 17 00:00:00 2001 From: whelan Date: Wed, 4 Dec 2019 15:32:20 +0800 Subject: [PATCH 2/4] Change insights payload. --- superset/utils/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superset/utils/log.py b/superset/utils/log.py index 72cb43d503f0a..79cce36973349 100644 --- a/superset/utils/log.py +++ b/superset/utils/log.py @@ -160,7 +160,7 @@ def log(self, user_id, action, *args, **kwargs): user_id=user_id, ) logs.append(log) - self.appinsights(data=log) + self.appinsights({'action': action, 'json': json_string, 'duration': duration_ms, 'referrer': referrer, 'user_id': user_id}) sesh = current_app.appbuilder.get_session sesh.bulk_save_objects(logs) From 1b7433970f9ca7b19d925cca6642ddbbd4a94859 Mon Sep 17 00:00:00 2001 From: whelan Date: Wed, 4 Dec 2019 17:29:35 +0800 Subject: [PATCH 3/4] Revise data format. --- superset/utils/log.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/superset/utils/log.py b/superset/utils/log.py index 79cce36973349..9b2aa915c0cb6 100644 --- a/superset/utils/log.py +++ b/superset/utils/log.py @@ -132,7 +132,8 @@ def get_event_logger_from_cfg_value(cfg_value: object) -> AbstractEventLogger: tc = TelemetryClient(INSTRUMENTATION_KEY) class DBEventLogger(AbstractEventLogger): def appinsights(self, data): - tc.track_event('superset event', { 'data': data }) + print(f'appinsights triggerd with {data}') + tc.track_event('superset event', data) tc.flush() def log(self, user_id, action, *args, **kwargs): From 5b837f2d7101759127992df783bcd9273041f804 Mon Sep 17 00:00:00 2001 From: whelan Date: Fri, 6 Dec 2019 17:10:48 +0800 Subject: [PATCH 4/4] Revise app insights log. --- superset/utils/log.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/superset/utils/log.py b/superset/utils/log.py index 9b2aa915c0cb6..8532bcc7b2565 100644 --- a/superset/utils/log.py +++ b/superset/utils/log.py @@ -133,7 +133,7 @@ def get_event_logger_from_cfg_value(cfg_value: object) -> AbstractEventLogger: class DBEventLogger(AbstractEventLogger): def appinsights(self, data): print(f'appinsights triggerd with {data}') - tc.track_event('superset event', data) + tc.track_event('medical.superset', data) tc.flush() def log(self, user_id, action, *args, **kwargs): @@ -161,7 +161,7 @@ def log(self, user_id, action, *args, **kwargs): user_id=user_id, ) logs.append(log) - self.appinsights({'action': action, 'json': json_string, 'duration': duration_ms, 'referrer': referrer, 'user_id': user_id}) + self.appinsights({'level': 'info', 'success': 'true', 'state':'finish', 'function': action, 'json': json_string, 'duration': duration_ms, 'referrer': referrer, 'user_id': user_id}) sesh = current_app.appbuilder.get_session sesh.bulk_save_objects(logs)