From f1e757a8266caff9256c33144d67f79ff6e5f99a Mon Sep 17 00:00:00 2001 From: cccs-jc Date: Fri, 21 May 2021 15:45:28 -0400 Subject: [PATCH 1/6] add support for filters in sqlLab --- superset/views/utils.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/superset/views/utils.py b/superset/views/utils.py index 767490ca7317c..2052660bbf7c6 100644 --- a/superset/views/utils.py +++ b/superset/views/utils.py @@ -127,13 +127,16 @@ def loads_request_json(request_json_data: str) -> Dict[Any, Any]: def get_form_data( # pylint: disable=too-many-locals slice_id: Optional[int] = None, use_slice_data: bool = False ) -> Tuple[Dict[str, Any], Optional[Slice]]: - form_data = {} + form_data: Dict[Any, Any] = {} # chart data API requests are JSON request_json_data = ( request.json["queries"][0] if request.is_json and "queries" in request.json else None ) + + add_sqllab_custom_filters(form_data) + request_form_data = request.form.get("form_data") request_args_data = request.args.get("form_data") if request_json_data: @@ -196,6 +199,26 @@ def get_form_data( # pylint: disable=too-many-locals return form_data, slc +def add_sqllab_custom_filters(form_data: Dict[Any, Any]) -> Any: + """ + SQLLab can include a "filters" attribute in the templateParams. + The filters attribute is a list of filters to include in the + request. Useful for testing templates in SQLLab. + """ + try: + data = json.loads(request.data) + if isinstance(data, dict): + params_str = data.get("templateParams") + if isinstance(params_str, str): + params = json.loads(params_str) + if isinstance(params, dict): + filters = params.get("filters") + if filters: + form_data.update({"filters": filters}) + except (TypeError, json.JSONDecodeError): + data = {} + + def get_datasource_info( datasource_id: Optional[int], datasource_type: Optional[str], form_data: FormData ) -> Tuple[int, Optional[str]]: From a3f1f11b4259ef9e763d8432c4ea17b9ec3beb01 Mon Sep 17 00:00:00 2001 From: cccs-jc Date: Sun, 30 May 2021 13:56:58 -0400 Subject: [PATCH 2/6] code review #14507 don't save filters attribute --- .../src/SqlLab/components/ResultSet.tsx | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/superset-frontend/src/SqlLab/components/ResultSet.tsx b/superset-frontend/src/SqlLab/components/ResultSet.tsx index 398e2f2caa51e..cd7a457d67dea 100644 --- a/superset-frontend/src/SqlLab/components/ResultSet.tsx +++ b/superset-frontend/src/SqlLab/components/ResultSet.tsx @@ -271,9 +271,21 @@ export default class ResultSet extends React.PureComponent< return; } - const { schema, sql, dbId, templateParams } = this.props.query; + const { schema, sql, dbId } = this.props.query; + let { templateParams } = this.props.query; const selectedColumns = this.props.query?.results?.selected_columns || []; + // The filters param is only used to test jinja templates. + // Remove the special filters entry from the templateParams + // before saving the dataset. + if (templateParams) { + const p = JSON.parse(templateParams) + if (p.filters) { + delete p['filters'] + templateParams = JSON.stringify(p) + } + } + this.props.actions .createDatasource({ schema, From 6dc13943f3e92b2e62fdb8666b8b67b5f88e6849 Mon Sep 17 00:00:00 2001 From: cccs-jc Date: Sun, 30 May 2021 14:16:27 -0400 Subject: [PATCH 3/6] typing #14507 --- superset/views/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superset/views/utils.py b/superset/views/utils.py index 2052660bbf7c6..1d1603e41d786 100644 --- a/superset/views/utils.py +++ b/superset/views/utils.py @@ -127,7 +127,7 @@ def loads_request_json(request_json_data: str) -> Dict[Any, Any]: def get_form_data( # pylint: disable=too-many-locals slice_id: Optional[int] = None, use_slice_data: bool = False ) -> Tuple[Dict[str, Any], Optional[Slice]]: - form_data: Dict[Any, Any] = {} + form_data: Dict[str, Any] = {} # chart data API requests are JSON request_json_data = ( request.json["queries"][0] From 7523b493980d31a4b9bb518342902038192c8914 Mon Sep 17 00:00:00 2001 From: cccs-jc Date: Mon, 31 May 2021 09:34:53 -0400 Subject: [PATCH 4/6] _filters indicates it's a temporary construct --- docs/installation.rst | 31 +++++++++++++++++++ .../src/SqlLab/components/ResultSet.tsx | 6 ++-- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index 8d28bd0b811f7..a267cce2f76a4 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -1256,6 +1256,37 @@ in this dictionary are made available for users to use in their SQL. 'my_crazy_macro': lambda x: x*2, } +Default values for jinja templates can be specified via ``Parameters`` menu in the SQL Lab user interface. +In the UI you can assign a set of parameters as JSON + +.. code-block:: JSON + { + "my_table": "foo" + } + +The parameters become available in your SQL (example:SELECT * FROM {{ my_table }} ) by using Jinja templating syntax. +SQL Lab template parameters are stored with the dataset as TEMPLATE PARAMETERS. + +There is a special ``_filters`` parameter which can be used to test filters used in the jinja template. + +.. code-block:: JSON + { + "_filters": { + "col": "action_type", + "op": "IN", + "val": ["sell", "buy"] + } + +.. code-block:: python + SELECT action, count(*) as times + FROM logs + WHERE + action in ({{ "'" + "','".join(filter_values('action_type')) + "'" }}) + GROUP BY action + +Note ``_filters`` is not stored with the dataset. It's only used within the SQL Lab UI. + + Besides default Jinja templating, SQL lab also supports self-defined template processor by setting the ``CUSTOM_TEMPLATE_PROCESSORS`` in your superset configuration. The values in this dictionary overwrite the default Jinja template processors of the diff --git a/superset-frontend/src/SqlLab/components/ResultSet.tsx b/superset-frontend/src/SqlLab/components/ResultSet.tsx index cd7a457d67dea..53aa61a3945b9 100644 --- a/superset-frontend/src/SqlLab/components/ResultSet.tsx +++ b/superset-frontend/src/SqlLab/components/ResultSet.tsx @@ -279,10 +279,10 @@ export default class ResultSet extends React.PureComponent< // Remove the special filters entry from the templateParams // before saving the dataset. if (templateParams) { - const p = JSON.parse(templateParams) + const p = JSON.parse(templateParams); if (p.filters) { - delete p['filters'] - templateParams = JSON.stringify(p) + delete p._filters; + templateParams = JSON.stringify(p); } } From 7574c5312ce5ef1e64a2198e50027f7c904470df Mon Sep 17 00:00:00 2001 From: cccs-jc Date: Mon, 31 May 2021 12:09:13 -0400 Subject: [PATCH 5/6] fix underscore dangle --- superset-frontend/src/SqlLab/components/ResultSet.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/superset-frontend/src/SqlLab/components/ResultSet.tsx b/superset-frontend/src/SqlLab/components/ResultSet.tsx index 53aa61a3945b9..4b230e28f40a3 100644 --- a/superset-frontend/src/SqlLab/components/ResultSet.tsx +++ b/superset-frontend/src/SqlLab/components/ResultSet.tsx @@ -281,6 +281,7 @@ export default class ResultSet extends React.PureComponent< if (templateParams) { const p = JSON.parse(templateParams); if (p.filters) { + /* eslint-disable-next-line no-underscore-dangle */ delete p._filters; templateParams = JSON.stringify(p); } From ed3843450ae427e35e1f635a1c06eab4ad49f854 Mon Sep 17 00:00:00 2001 From: cccs-jc Date: Tue, 1 Jun 2021 11:43:56 -0400 Subject: [PATCH 6/6] removed trailing whitespace --- docs/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index a267cce2f76a4..6c3cc1efedb60 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -1274,7 +1274,7 @@ There is a special ``_filters`` parameter which can be used to test filters used "_filters": { "col": "action_type", "op": "IN", - "val": ["sell", "buy"] + "val": ["sell", "buy"] } .. code-block:: python