diff --git a/.gitignore b/.gitignore index 7449a91d059b0..57c945dd2bc3f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.pyc +*.db tmp diff --git a/app.db b/app.db index 42d893593db15..16e9d6aa8fde4 100644 Binary files a/app.db and b/app.db differ diff --git a/app.py b/app.py deleted file mode 100644 index 1145b5dc4c699..0000000000000 --- a/app.py +++ /dev/null @@ -1,132 +0,0 @@ -from pydruid import client -from pydruid.utils.filters import Dimension -from dateutil.parser import parse -from datetime import datetime, timedelta -from flask import Flask, render_template, request -from flask_bootstrap import Bootstrap -import json -from wtforms import Form, SelectMultipleField, SelectField, TextField -import pandas as pd -pd.set_option('display.max_colwidth', -1) - -ROW_LIMIT = 10000 -PORT = 8088 -query = client.PyDruid("http://10.181.47.80:8080", 'druid/v2') - -app = Flask(__name__) -Bootstrap(app) - -def latest_metadata(datasource): - max_time = query.time_boundary(datasource=datasource)[0]['result']['maxTime'] - max_time = parse(max_time) - intervals = (max_time - timedelta(seconds=1)).isoformat() + '/' - intervals += max_time.isoformat() - return query.segment_metadata( - datasource=datasource, - intervals=intervals)[-1]['columns'] - -@app.route("/datasource//") -def datasource(datasource): - - metadata = latest_metadata(datasource) - grain = ['all', 'none', 'minute', 'hour', 'day'] - since_l = { - '1hour': timedelta(hours=1), - '1day': timedelta(days=1), - '7days': timedelta(days=7), - '28days': timedelta(days=28), - 'all': timedelta(days=365*100) - } - limits = [0, 5, 10, 25, 50, 100, 500] - limit = request.args.get("limit") - try: - limit = int(limit) - if limit not in limits: - limits.append(limit) - limits = sorted(limits) - except: - pass - class QueryForm(Form): - groupby = SelectMultipleField( - 'Group by', choices=[(m, m) for m in sorted(metadata.keys())]) - granularity = SelectField( - 'Granularity', choices=[(g, g) for g in grain]) - since = SelectField( - 'Since', choices=[(s, s) for s in since_l.keys()]) - limit = SelectField( - 'Limit', choices=[(s, s) for s in limits]) - flt_col_1 = SelectField( - 'Filter 1', choices=[(m, m) for m in sorted(metadata.keys())]) - flt_op_1 = SelectField( - 'Filter 1', choices=[(m, m) for m in ['==', 'in', '<', '>']]) - flt_eq_1 = TextField("Super") - - groupby = request.args.getlist("groupby") or [] - granularity = request.args.get("granularity") - limit = int(request.args.get("limit", ROW_LIMIT)) or ROW_LIMIT - since = request.args.get("since", "all") - from_dttm = (datetime.now() - since_l[since]).isoformat() - - # Building filters - i = 1 - filters = [] - while True: - col = request.args.get("flt_col_" + str(i)) - op = request.args.get("flt_op_" + str(i)) - eq = request.args.get("flt_eq_" + str(i)) - print (col,op,eq) - if col and op and eq: - filters.append(Dimension(col)==eq) - filters = Dimension(col)==eq - else: - break - i += 1 - print filters - - results=[] - results = query.groupby( - datasource=datasource, - granularity=granularity or 'all', - intervals=from_dttm + '/' + datetime.now().isoformat(), - dimensions=groupby, - aggregations={"count": client.doublesum("count")}, - filter=filters, - limit_spec={ - "type": "default", - "limit": limit, - "columns": [{ - "dimension" : "count", - "direction" : "descending", - },], - }, - ) - - df = query.export_pandas() - if df is not None and not df.empty: - df = df.sort(df.columns[0], ascending=False) - if granularity == 'all': - del df['timestamp'] - - table = df.to_html( - classes=["table", "table-striped", 'table-bordered'], index=False) - else: - table = None - - return render_template( - 'panoramix/datasource.html', - table=table, - datasource=datasource, - latest_metadata=json.dumps( - metadata, - sort_keys=True, - indent=2), - results=json.dumps( - results, - sort_keys=True, - indent=2), - form=QueryForm(request.args), - ) - -if __name__ == '__main__': - app.debug = True - app.run(host='0.0.0.0', port=PORT) diff --git a/app/__init__.py b/app/__init__.py index ccc0a50e31c30..8c8ef8829e968 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1,6 +1,6 @@ import logging from flask import Flask -from flask.ext.appbuilder import SQLA, AppBuilder +from flask.ext.appbuilder import SQLA, AppBuilder, IndexView """ Logging configuration @@ -12,8 +12,13 @@ app = Flask(__name__) app.config.from_object('config') db = SQLA(app) + +class MyIndexView(IndexView): + index_template = 'index.html' + appbuilder = AppBuilder( - app, db.session, base_template='panoramix/base.html') + app, db.session, base_template='panoramix/base.html', + indexview=MyIndexView) #appbuilder.app_name = 'Panoramix' @@ -31,4 +36,3 @@ def set_sqlite_pragma(dbapi_connection, connection_record): """ from app import views - diff --git a/app/models.py b/app/models.py index 39259323ac9cb..81c51050e3540 100644 --- a/app/models.py +++ b/app/models.py @@ -43,7 +43,7 @@ def latest_metadata(cls, name): print "---" * 100 print name print results - max_time = results[0]['result']['maxTime'] + max_time = results[0]['result']['minTime'] max_time = parse(max_time) intervals = (max_time - timedelta(seconds=1)).isoformat() + '/' intervals += (max_time + timedelta(seconds=1)).isoformat() diff --git a/app/templates/index.html b/app/templates/index.html index 2970ab1592018..626b725243eee 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -1,2 +1,13 @@ -{% extends "appbuilder/baselayout.html" %} +{% extends "appbuilder/base.html" %} +{% block content %} +
+
+

Panoramix

+

Panoramix is an interactive visualization platform built on top of Druid.io

+
+
+
+ +
+{% endblock %} diff --git a/app/utils.py b/app/utils.py index b92ebfe224cee..48e91e7709bbc 100644 --- a/app/utils.py +++ b/app/utils.py @@ -1,5 +1,6 @@ import config -from datetime import timedelta +from datetime import timedelta, datetime +import parsedatetime since_l = { '1hour': timedelta(hours=1), @@ -16,3 +17,15 @@ def get_pydruid_client(): config.DRUID_BASE_ENDPOINT) +def parse_human_datetime(s): + """ + Use the parsedatetime lib to return ``datetime.datetime`` from human + generated strings + + >>> parse_human_datetime("now") <= datetime.now() + True + """ + cal = parsedatetime.Calendar() + d = cal.parse(s)[0] + return datetime( + d.tm_year, d.tm_mon, d.tm_mday, d.tm_hour, d.tm_min, d.tm_sec) diff --git a/app/views.py b/app/views.py index 90a3eaa80c3fb..a24d3f828775a 100644 --- a/app/views.py +++ b/app/views.py @@ -1,3 +1,6 @@ +from datetime import timedelta +import logging + from flask import request, redirect, flash from flask.ext.appbuilder.models.sqla.interface import SQLAInterface from flask.ext.appbuilder import ModelView, CompactCRUDMixin, BaseView, expose @@ -5,7 +8,6 @@ import config from wtforms import Form, SelectMultipleField, SelectField, TextField from wtforms.fields import Field -from datetime import timedelta class OmgWtForm(Form): field_order = ( @@ -131,7 +133,11 @@ def refresh_datasources(self): ).format(**config.__dict__) datasources = json.loads(requests.get(endpoint).text) for datasource in datasources: - models.Datasource.sync_to_db(datasource) + try: + models.Datasource.sync_to_db(datasource) + except Exception as e: + logging.exception(e) + logging.error("Failed at syncing " + datasource) flash("Refreshed metadata from Druid!", 'info') return redirect("/datasourcemodelview/list/") diff --git a/app/viz.py b/app/viz.py index f1957aa6ee64f..543c831b66da5 100644 --- a/app/viz.py +++ b/app/viz.py @@ -200,6 +200,7 @@ class TimeSeriesBarViz(TimeSeriesViz): verbose_name = "Time Series - Bar Chart" chart_kind = "bar" + class TimeSeriesStackedBarViz(TimeSeriesViz): verbose_name = "Time Series - Stacked Bar Chart" chart_kind = "bar" diff --git a/config.py b/config.py index 333efa26c41a3..65e596a920b5e 100644 --- a/config.py +++ b/config.py @@ -76,14 +76,17 @@ BABEL_DEFAULT_FOLDER = 'translations' # The allowed translation for you app LANGUAGES = { - 'en': {'flag':'gb', 'name':'English'}, - 'pt': {'flag':'pt', 'name':'Portuguese'}, - 'pt_BR': {'flag':'br', 'name': 'Pt Brazil'}, - 'es': {'flag':'es', 'name':'Spanish'}, - 'de': {'flag':'de', 'name':'German'}, - 'zh': {'flag':'cn', 'name':'Chinese'}, - 'ru': {'flag':'ru', 'name':'Russian'} + 'en': {'flag':'us', 'name':'English'}, + 'fr': {'flag':'fr', 'name':'French'}, } +""" +'pt': {'flag':'pt', 'name':'Portuguese'}, +'pt_BR': {'flag':'br', 'name': 'Pt Brazil'}, +'es': {'flag':'es', 'name':'Spanish'}, +'de': {'flag':'de', 'name':'German'}, +'zh': {'flag':'cn', 'name':'Chinese'}, +'ru': {'flag':'ru', 'name':'Russian'} +""" #--------------------------------------------------- # Image and file configuration #--------------------------------------------------- diff --git a/requirements.txt b/requirements.txt index a5717712317aa..c7c6de1f0c0d8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ pydruid +parsedatetime python-dateutil flask flask-appbuilder