From c1141e5d37ef601f22834c45017b7eb035e0c770 Mon Sep 17 00:00:00 2001 From: sebastianliebscher Date: Wed, 10 May 2023 11:58:18 +0200 Subject: [PATCH 1/9] Remove deprecated distutils --- requirements/base.txt | 8 ++++---- requirements/integration.txt | 4 +--- setup.py | 1 + superset/db_engine_specs/elasticsearch.py | 6 ++---- superset/db_engine_specs/presto.py | 7 +++---- superset/utils/core.py | 22 ++++++++++++++++++++-- tests/unit_tests/utils/test_core.py | 17 ++++++++++++++++- 7 files changed, 47 insertions(+), 18 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 54c5c43cf5119..86b535d2c1f8d 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -193,8 +193,9 @@ numpy==1.23.5 # pyarrow ordered-set==4.1.0 # via flask-limiter -packaging==21.3 +packaging==23.1 # via + # apache-superset # deprecation # limits pandas==1.5.3 @@ -227,9 +228,7 @@ pymeeus==0.5.11 pynacl==1.5.0 # via paramiko pyparsing==3.0.6 - # via - # apache-superset - # packaging + # via apache-superset pyrsistent==0.16.1 # via jsonschema python-dateutil==2.8.2 @@ -301,6 +300,7 @@ typing-extensions==4.4.0 # apache-superset # flask-limiter # limits + # rich urllib3==1.26.6 # via selenium vine==5.0.0 diff --git a/requirements/integration.txt b/requirements/integration.txt index 29c43279f9ebd..661dffdcf43bf 100644 --- a/requirements/integration.txt +++ b/requirements/integration.txt @@ -23,7 +23,7 @@ identify==2.2.13 # via pre-commit nodeenv==1.6.0 # via pre-commit -packaging==21.3 +packaging==23.1 # via # build # tox @@ -41,8 +41,6 @@ pre-commit==3.3.1 # via -r requirements/integration.in py==1.10.0 # via tox -pyparsing==3.0.6 - # via packaging pyyaml==5.4.1 # via pre-commit six==1.16.0 diff --git a/setup.py b/setup.py index eb396b8ccef06..f3fd9ab0ee536 100644 --- a/setup.py +++ b/setup.py @@ -102,6 +102,7 @@ def get_git_sha() -> str: "msgpack>=1.0.0, <1.1", "nh3>=0.2.11, <0.3", "numpy==1.23.5", + "packaging", "pandas>=1.5.3, <1.6", "parsedatetime", "pgsanity", diff --git a/superset/db_engine_specs/elasticsearch.py b/superset/db_engine_specs/elasticsearch.py index c96d0b36a7e87..934aa0bb03cf6 100644 --- a/superset/db_engine_specs/elasticsearch.py +++ b/superset/db_engine_specs/elasticsearch.py @@ -16,9 +16,9 @@ # under the License. import logging from datetime import datetime -from distutils.version import StrictVersion from typing import Any, Dict, Optional, Type +from packaging.version import Version from sqlalchemy import types from superset.db_engine_specs.base import BaseEngineSpec @@ -79,9 +79,7 @@ def convert_dttm( supports_dttm_parse = False try: if es_version: - supports_dttm_parse = StrictVersion(es_version) >= StrictVersion( - "7.8" - ) + supports_dttm_parse = Version(es_version) >= Version("7.8") except Exception as ex: # pylint: disable=broad-except logger.error("Unexpected error while convert es_version", exc_info=True) logger.exception(ex) diff --git a/superset/db_engine_specs/presto.py b/superset/db_engine_specs/presto.py index 0889dd653b8ee..8e2116ccdff3d 100644 --- a/superset/db_engine_specs/presto.py +++ b/superset/db_engine_specs/presto.py @@ -23,7 +23,6 @@ from abc import ABCMeta from collections import defaultdict, deque from datetime import datetime -from distutils.version import StrictVersion from textwrap import dedent from typing import ( Any, @@ -43,6 +42,7 @@ import simplejson as json from flask import current_app from flask_babel import gettext as __, lazy_gettext as _ +from packaging.version import Version from sqlalchemy import Column, literal_column, types from sqlalchemy.engine.base import Engine from sqlalchemy.engine.reflection import Inspector @@ -470,8 +470,7 @@ def _partition_query( # pylint: disable=too-many-arguments,too-many-locals,unus # Default to the new syntax if version is unset. partition_select_clause = ( f'SELECT * FROM "{table_name}$partitions"' - if not presto_version - or StrictVersion(presto_version) >= StrictVersion("0.199") + if not presto_version or Version(presto_version) >= Version("0.199") else f"SHOW PARTITIONS FROM {table_name}" ) @@ -705,7 +704,7 @@ class PrestoEngineSpec(PrestoBaseEngineSpec): @classmethod def get_allow_cost_estimate(cls, extra: Dict[str, Any]) -> bool: version = extra.get("version") - return version is not None and StrictVersion(version) >= StrictVersion("0.319") + return version is not None and Version(version) >= Version("0.319") @classmethod def update_impersonation_config( diff --git a/superset/utils/core.py b/superset/utils/core.py index 4569031f3f5b0..adb905fb7a6ef 100644 --- a/superset/utils/core.py +++ b/superset/utils/core.py @@ -38,7 +38,6 @@ from contextlib import contextmanager from dataclasses import dataclass from datetime import date, datetime, time, timedelta -from distutils.util import strtobool from email.mime.application import MIMEApplication from email.mime.image import MIMEImage from email.mime.multipart import MIMEMultipart @@ -1191,6 +1190,7 @@ def merge_extra_filters(form_data: Dict[str, Any]) -> None: "__time_grain": "time_grain_sqla", "__granularity": "granularity", } + # Grab list of existing filters 'keyed' on the column and operator def get_filter_key(f: Dict[str, Any]) -> str: @@ -1787,8 +1787,26 @@ def indexed( return idx +def strtobool(val: str) -> bool: + """Convert a string representation of truth to true (1) or false (0). + + True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values + are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if + 'val' is anything else. + + Original implementation from deprecated distutils. + """ + val = val.lower() + if val in {"y", "yes", "t", "true", "on", "1"}: + return True + elif val in {"n", "no", "f", "false", "off", "0"}: + return False + else: + raise ValueError("invalid truth value %r" % (val,)) + + def is_test() -> bool: - return strtobool(os.environ.get("SUPERSET_TESTENV", "false")) # type: ignore + return strtobool(os.environ.get("SUPERSET_TESTENV", "false")) def get_time_filter_status( diff --git a/tests/unit_tests/utils/test_core.py b/tests/unit_tests/utils/test_core.py index 6845bb2fc1545..0c2d7bab8d8df 100644 --- a/tests/unit_tests/utils/test_core.py +++ b/tests/unit_tests/utils/test_core.py @@ -19,7 +19,11 @@ import pytest -from superset.utils.core import QueryObjectFilterClause, remove_extra_adhoc_filters +from superset.utils.core import ( + parse_boolean_string, + QueryObjectFilterClause, + remove_extra_adhoc_filters, +) ADHOC_FILTER: QueryObjectFilterClause = { "col": "foo", @@ -84,3 +88,14 @@ def test_remove_extra_adhoc_filters( ) -> None: remove_extra_adhoc_filters(original) assert expected == original + + +def test_parse_boolean_string(): + true = ("y", "Y", "yes", "True", "t", "true", "True", "On", "on", "1") + false = ("n", "no", "f", "false", "off", "0", "Off", "No", "N", "foo") + + for y in true: + assert parse_boolean_string(y) + + for n in false: + assert not parse_boolean_string(n) From 384bb2dfbc85612a2d7c9dd998e5caa3f74566c6 Mon Sep 17 00:00:00 2001 From: sebastianliebscher Date: Wed, 10 May 2023 12:14:42 +0200 Subject: [PATCH 2/9] pin packaging for google-cloud-bigquery --- requirements/base.txt | 2 +- requirements/integration.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 86b535d2c1f8d..81968fed34ed8 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -193,7 +193,7 @@ numpy==1.23.5 # pyarrow ordered-set==4.1.0 # via flask-limiter -packaging==23.1 +packaging==21.3 # via # apache-superset # deprecation diff --git a/requirements/integration.txt b/requirements/integration.txt index 661dffdcf43bf..b6a8578c70d68 100644 --- a/requirements/integration.txt +++ b/requirements/integration.txt @@ -23,7 +23,7 @@ identify==2.2.13 # via pre-commit nodeenv==1.6.0 # via pre-commit -packaging==23.1 +packaging==21.3 # via # build # tox From be6576b55a4a263bc888a1d07362a8afbb8f104b Mon Sep 17 00:00:00 2001 From: sebastianliebscher Date: Wed, 10 May 2023 12:34:04 +0200 Subject: [PATCH 3/9] pylint --- superset/utils/core.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/superset/utils/core.py b/superset/utils/core.py index adb905fb7a6ef..fe86a9d00d31f 100644 --- a/superset/utils/core.py +++ b/superset/utils/core.py @@ -1799,10 +1799,9 @@ def strtobool(val: str) -> bool: val = val.lower() if val in {"y", "yes", "t", "true", "on", "1"}: return True - elif val in {"n", "no", "f", "false", "off", "0"}: + if val in {"n", "no", "f", "false", "off", "0"}: return False - else: - raise ValueError("invalid truth value %r" % (val,)) + raise ValueError("invalid truth value %r" % (val,)) def is_test() -> bool: From 31adc510c5a248c1c3489d71635c3c538774d1ef Mon Sep 17 00:00:00 2001 From: sebastianliebscher Date: Wed, 10 May 2023 13:18:14 +0200 Subject: [PATCH 4/9] update .pylintrc --- .pylintrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pylintrc b/.pylintrc index 848767fe5dcb1..b39335c56b003 100644 --- a/.pylintrc +++ b/.pylintrc @@ -300,7 +300,7 @@ ignore-mixin-members=yes # (useful for modules/projects where namespaces are manipulated during runtime # and thus existing member attributes cannot be deduced by static analysis. It # supports qualified module names, as well as Unix pattern matching. -ignored-modules=numpy,pandas,alembic.op,sqlalchemy,alembic.context,flask_appbuilder.security.sqla.PermissionView.role,flask_appbuilder.Model.metadata,flask_appbuilder.Base.metadata,distutils +ignored-modules=numpy,pandas,alembic.op,sqlalchemy,alembic.context,flask_appbuilder.security.sqla.PermissionView.role,flask_appbuilder.Model.metadata,flask_appbuilder.Base.metadata # List of class names for which member attributes should not be checked (useful # for classes with dynamically set attributes). This supports the use of From b7e80e0b61d4cbc7b50df15263c1d3e45f390376 Mon Sep 17 00:00:00 2001 From: EugeneTorap Date: Wed, 10 May 2023 21:42:02 +0300 Subject: [PATCH 5/9] Remove 'numpy.distutils' import and use own is_sequence func --- superset/utils/pandas_postprocessing/flatten.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/superset/utils/pandas_postprocessing/flatten.py b/superset/utils/pandas_postprocessing/flatten.py index 1026164e454ee..616a88567d6ae 100644 --- a/superset/utils/pandas_postprocessing/flatten.py +++ b/superset/utils/pandas_postprocessing/flatten.py @@ -15,10 +15,10 @@ # specific language governing permissions and limitations # under the License. -from typing import Sequence, Union +from collections.abc import Iterable +from typing import Sequence, Union, Any import pandas as pd -from numpy.distutils.misc_util import is_sequence from superset.utils.pandas_postprocessing.utils import ( _is_multi_index_on_columns, @@ -27,6 +27,13 @@ ) +def is_sequence(seq: Any) -> bool: + if isinstance(seq, str): + return False + + return isinstance(seq, Iterable) + + def flatten( df: pd.DataFrame, reset_index: bool = True, From 46f71e6c9f80f4a3fc8dd8b7eb1da37a039c083c Mon Sep 17 00:00:00 2001 From: EugeneTorap Date: Wed, 10 May 2023 21:52:55 +0300 Subject: [PATCH 6/9] Fix isort & mypy errors --- superset/utils/pandas_postprocessing/flatten.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/superset/utils/pandas_postprocessing/flatten.py b/superset/utils/pandas_postprocessing/flatten.py index 616a88567d6ae..da9954ef111f6 100644 --- a/superset/utils/pandas_postprocessing/flatten.py +++ b/superset/utils/pandas_postprocessing/flatten.py @@ -16,7 +16,7 @@ # under the License. from collections.abc import Iterable -from typing import Sequence, Union, Any +from typing import Any, Sequence, Union import pandas as pd @@ -92,7 +92,7 @@ def flatten( _columns = [] for series in df.columns.to_flat_index(): _cells = [] - for cell in series if is_sequence(series) else [series]: # type: ignore + for cell in series if is_sequence(series) else [series]: if pd.notnull(cell): # every cell should be converted to string and escape comma _cells.append(escape_separator(str(cell))) From febb7bf7e14e79e8aebf836ecf236833e24a50c7 Mon Sep 17 00:00:00 2001 From: EugeneTorap Date: Thu, 11 May 2023 07:55:32 +0300 Subject: [PATCH 7/9] refactor requirements/base.txt --- requirements/base.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index 86325100ff121..34478f1052ce5 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -296,7 +296,6 @@ typing-extensions==4.4.0 # apache-superset # flask-limiter # limits - # rich urllib3==1.26.6 # via selenium vine==5.0.0 From b72954b1e32cdff124ad0e3796c0d840b3b9b99c Mon Sep 17 00:00:00 2001 From: sebastianliebscher Date: Thu, 11 May 2023 11:28:05 +0200 Subject: [PATCH 8/9] consolidate --- superset/utils/core.py | 24 ++---------------------- tests/unit_tests/utils/test_core.py | 28 +++++++++++++++++++++------- 2 files changed, 23 insertions(+), 29 deletions(-) diff --git a/superset/utils/core.py b/superset/utils/core.py index fe86a9d00d31f..8451eaaa6f457 100644 --- a/superset/utils/core.py +++ b/superset/utils/core.py @@ -1787,25 +1787,8 @@ def indexed( return idx -def strtobool(val: str) -> bool: - """Convert a string representation of truth to true (1) or false (0). - - True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values - are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if - 'val' is anything else. - - Original implementation from deprecated distutils. - """ - val = val.lower() - if val in {"y", "yes", "t", "true", "on", "1"}: - return True - if val in {"n", "no", "f", "false", "off", "0"}: - return False - raise ValueError("invalid truth value %r" % (val,)) - - def is_test() -> bool: - return strtobool(os.environ.get("SUPERSET_TESTENV", "false")) + return parse_boolean_string(os.environ.get("SUPERSET_TESTENV", "false")) def get_time_filter_status( @@ -1969,10 +1952,7 @@ def parse_boolean_string(bool_str: Optional[str]) -> bool: """ if bool_str is None: return False - try: - return bool(strtobool(bool_str.lower())) - except ValueError: - return False + return bool_str.lower() in ("y", "Y", "yes", "True", "t", "true", "On", "on", "1") def apply_max_row_limit( diff --git a/tests/unit_tests/utils/test_core.py b/tests/unit_tests/utils/test_core.py index 0c2d7bab8d8df..83c22ad281bb9 100644 --- a/tests/unit_tests/utils/test_core.py +++ b/tests/unit_tests/utils/test_core.py @@ -15,11 +15,13 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +import os from typing import Any, Dict import pytest from superset.utils.core import ( + is_test, parse_boolean_string, QueryObjectFilterClause, remove_extra_adhoc_filters, @@ -90,12 +92,24 @@ def test_remove_extra_adhoc_filters( assert expected == original -def test_parse_boolean_string(): - true = ("y", "Y", "yes", "True", "t", "true", "True", "On", "on", "1") - false = ("n", "no", "f", "false", "off", "0", "Off", "No", "N", "foo") +def test_is_test(): + orig_value = os.getenv("SUPERSET_TESTENV") + + os.environ["SUPERSET_TESTENV"] = "true" + assert is_test() + os.environ["SUPERSET_TESTENV"] = "false" + assert not is_test() + os.environ["SUPERSET_TESTENV"] = "" + assert not is_test() - for y in true: - assert parse_boolean_string(y) + if orig_value is not None: + os.environ["SUPERSET_TESTENV"] = orig_value - for n in false: - assert not parse_boolean_string(n) + +def test_parse_boolean_string(): + true = ("y", "Y", "yes", "True", "t", "true", "On", "on", "1") + false = ("n", "N", "no", "False", "f", "false", "Off", "off", "0", "foo", "", None) + for val in true: + assert parse_boolean_string(val) + for val in false: + assert not parse_boolean_string(val) From 8d1366f6489d37babb3e1a351c4863408dd104a1 Mon Sep 17 00:00:00 2001 From: sebastianliebscher Date: Thu, 11 May 2023 15:48:20 +0200 Subject: [PATCH 9/9] parameterize test --- tests/unit_tests/utils/test_core.py | 36 ++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/tests/unit_tests/utils/test_core.py b/tests/unit_tests/utils/test_core.py index 83c22ad281bb9..3636983156fdb 100644 --- a/tests/unit_tests/utils/test_core.py +++ b/tests/unit_tests/utils/test_core.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. import os -from typing import Any, Dict +from typing import Any, Dict, Optional import pytest @@ -106,10 +106,30 @@ def test_is_test(): os.environ["SUPERSET_TESTENV"] = orig_value -def test_parse_boolean_string(): - true = ("y", "Y", "yes", "True", "t", "true", "On", "on", "1") - false = ("n", "N", "no", "False", "f", "false", "Off", "off", "0", "foo", "", None) - for val in true: - assert parse_boolean_string(val) - for val in false: - assert not parse_boolean_string(val) +@pytest.mark.parametrize( + "test_input,expected", + [ + ("y", True), + ("Y", True), + ("yes", True), + ("True", True), + ("t", True), + ("true", True), + ("On", True), + ("on", True), + ("1", True), + ("n", False), + ("N", False), + ("no", False), + ("False", False), + ("f", False), + ("false", False), + ("Off", False), + ("off", False), + ("0", False), + ("foo", False), + (None, False), + ], +) +def test_parse_boolean_string(test_input: Optional[str], expected: bool): + assert parse_boolean_string(test_input) == expected