From 66c5f5c20ac033b9955c8fb8d6f676e21ca9e7ac Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Thu, 12 Aug 2021 11:16:48 +0300 Subject: [PATCH 1/3] chore: upgrade mypy and add type guards --- .pre-commit-config.yaml | 3 ++- requirements/base.in | 12 ++++++++++++ requirements/base.txt | 24 ++++++++++++++++++++++-- setup.py | 6 +++--- superset/tasks/schedules.py | 2 +- superset/typing.py | 4 ++-- superset/utils/async_query_manager.py | 4 ++-- superset/utils/core.py | 7 +++---- tests/unit_tests/core_tests.py | 7 +++++++ 9 files changed, 54 insertions(+), 15 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bca0923e8068a..80202bebfab9a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,9 +24,10 @@ repos: hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.790 + rev: v0.910 hooks: - id: mypy + additional_dependencies: [types-all] - repo: https://github.com/peterdemin/pip-compile-multi rev: v2.4.1 hooks: diff --git a/requirements/base.in b/requirements/base.in index 294583801824c..669e9dd65ea1c 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -18,3 +18,15 @@ -e file:. pyrsistent>=0.16.1,<0.17 zipp==3.4.1 + +# stubs +types-Markdown +types-PyYAML +types-bleach +types-croniter +types-python-dateutil +types-pytz +types-redis +types-requests +types-setuptools +types-simplejson diff --git a/requirements/base.txt b/requirements/base.txt index 0125e97608990..d18ec7cc3fe2f 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,4 +1,4 @@ -# SHA1:0862095245a068ae2fc00217da78331e1e7ae505 +# SHA1:6ada23ace0591867da0f02fa77840f4df5239e65 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -266,7 +266,27 @@ sqlparse==0.3.0 # via apache-superset tabulate==0.8.9 # via apache-superset -typing-extensions==3.7.4.3 +types-bleach==4.0.0 + # via -r requirements/base.in +types-croniter==1.0.0 + # via -r requirements/base.in +types-markdown==3.3.2 + # via -r requirements/base.in +types-python-dateutil==0.1.6 + # via -r requirements/base.in +types-pytz==2021.1.2 + # via -r requirements/base.in +types-pyyaml==5.4.6 + # via -r requirements/base.in +types-redis==3.5.6 + # via -r requirements/base.in +types-requests==2.25.6 + # via -r requirements/base.in +types-setuptools==57.0.2 + # via -r requirements/base.in +types-simplejson==0.1.5 + # via -r requirements/base.in +typing-extensions==3.10.0.0 # via # aiohttp # apache-superset diff --git a/setup.py b/setup.py index a38c7d2a2d8ca..d082733fa38d4 100644 --- a/setup.py +++ b/setup.py @@ -106,14 +106,14 @@ def get_git_sha() -> str: "simplejson>=3.15.0", "slackclient==2.5.0", # PINNED! slack changes file upload api in the future versions "sqlalchemy>=1.3.16, <1.4, !=1.3.21", - "sqlalchemy-utils>=0.36.6,<0.37", + "sqlalchemy-utils>=0.36.6, <0.37", "sqlparse==0.3.0", # PINNED! see https://github.com/andialbrecht/sqlparse/issues/562 "tabulate==0.8.9", - "typing-extensions>=3.7.4.3,<4", # needed to support typing.Literal on py37 + "typing-extensions>=3.10, <4", # needed to support Literal (3.8) and TypeGuard (3.10) "wtforms-json", ], extras_require={ - "athena": ["pyathena>=1.10.8,<1.11"], + "athena": ["pyathena>=1.10.8, <1.11"], "bigquery": [ "pandas_gbq>=0.10.0", "pybigquery>=0.4.10", diff --git a/superset/tasks/schedules.py b/superset/tasks/schedules.py index 8d85ded4c555b..ddf060bc7869f 100644 --- a/superset/tasks/schedules.py +++ b/superset/tasks/schedules.py @@ -89,7 +89,7 @@ WEBDRIVER_BASEURL_USER_FRIENDLY = config["WEBDRIVER_BASEURL_USER_FRIENDLY"] ReportContent = namedtuple( - "EmailContent", + "ReportContent", [ "body", # email body "data", # attachments diff --git a/superset/typing.py b/superset/typing.py index 4273444fe4e54..d076402df0ac9 100644 --- a/superset/typing.py +++ b/superset/typing.py @@ -29,7 +29,7 @@ from flask import Flask from flask_caching import Cache -from typing_extensions import TypedDict +from typing_extensions import Literal, TypedDict from werkzeug.wrappers import Response if TYPE_CHECKING: @@ -57,7 +57,7 @@ class AdhocMetricColumn(TypedDict, total=False): class AdhocMetric(TypedDict, total=False): aggregate: str column: Optional[AdhocMetricColumn] - expressionType: str + expressionType: Literal["SIMPLE", "SQL"] label: Optional[str] sqlExpression: Optional[str] diff --git a/superset/utils/async_query_manager.py b/superset/utils/async_query_manager.py index 90428a11ee3de..258024f7f3f6d 100644 --- a/superset/utils/async_query_manager.py +++ b/superset/utils/async_query_manager.py @@ -73,7 +73,7 @@ class AsyncQueryManager: def __init__(self) -> None: super().__init__() - self._redis: redis.Redis + self._redis: redis.Redis # type: ignore self._stream_prefix: str = "" self._stream_limit: Optional[int] self._stream_limit_firehose: Optional[int] @@ -100,7 +100,7 @@ def init_app(self, app: Flask) -> None: "Please provide a JWT secret at least 32 bytes long" ) - self._redis = redis.Redis( # type: ignore + self._redis = redis.Redis( **config["GLOBAL_ASYNC_QUERIES_REDIS_CONFIG"], decode_responses=True ) self._stream_prefix = config["GLOBAL_ASYNC_QUERIES_REDIS_STREAM_PREFIX"] diff --git a/superset/utils/core.py b/superset/utils/core.py index d48606ec0de74..33c7fda6d0b8e 100644 --- a/superset/utils/core.py +++ b/superset/utils/core.py @@ -81,7 +81,7 @@ from sqlalchemy.engine.reflection import Inspector from sqlalchemy.sql.type_api import Variant from sqlalchemy.types import TEXT, TypeDecorator, TypeEngine -from typing_extensions import TypedDict +from typing_extensions import TypedDict, TypeGuard import _thread # pylint: disable=C0411 from superset.constants import ( @@ -1275,7 +1275,7 @@ def backend() -> str: return get_example_database().backend -def is_adhoc_metric(metric: Metric) -> bool: +def is_adhoc_metric(metric: Metric) -> TypeGuard[AdhocMetric]: return isinstance(metric, dict) and "expressionType" in metric @@ -1288,7 +1288,6 @@ def get_metric_name(metric: Metric) -> str: :raises ValueError: if metric object is invalid """ if is_adhoc_metric(metric): - metric = cast(AdhocMetric, metric) label = metric.get("label") if label: return label @@ -1306,7 +1305,7 @@ def get_metric_name(metric: Metric) -> str: if column_name: return column_name raise ValueError(__("Invalid metric object")) - return cast(str, metric) + return metric # type: ignore def get_metric_names(metrics: Sequence[Metric]) -> List[str]: diff --git a/tests/unit_tests/core_tests.py b/tests/unit_tests/core_tests.py index bb3e50f518b3d..51d1d0993aff4 100644 --- a/tests/unit_tests/core_tests.py +++ b/tests/unit_tests/core_tests.py @@ -23,6 +23,7 @@ GenericDataType, get_metric_name, get_metric_names, + is_adhoc_metric, ) STR_METRIC = "my_metric" @@ -91,3 +92,9 @@ def test_get_metric_names(): assert get_metric_names( [STR_METRIC, SIMPLE_SUM_ADHOC_METRIC, SQL_ADHOC_METRIC] ) == ["my_metric", "my SUM", "my_sql"] + + +def test_is_adhoc_metric(): + assert is_adhoc_metric(STR_METRIC) is False + assert is_adhoc_metric(SIMPLE_SUM_ADHOC_METRIC) is True + assert is_adhoc_metric(SQL_ADHOC_METRIC) is True From 140ed9b5134046b0095f7a1d86cff3e00d8fda17 Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Thu, 12 Aug 2021 11:51:16 +0300 Subject: [PATCH 2/3] lint --- RELEASING/changelog.py | 4 ++-- scripts/benchmark_migration.py | 10 +++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/RELEASING/changelog.py b/RELEASING/changelog.py index d6a1842fd878a..e9ff2de041a23 100644 --- a/RELEASING/changelog.py +++ b/RELEASING/changelog.py @@ -384,12 +384,12 @@ def change_log( with open(csv, "w") as csv_file: log_items = list(logs) field_names = log_items[0].keys() - writer = lib_csv.DictWriter( + writer = lib_csv.DictWriter( # type: ignore csv_file, delimiter=",", quotechar='"', quoting=lib_csv.QUOTE_ALL, - fieldnames=field_names, + fieldnames=field_names, # type: ignore ) writer.writeheader() for log in logs: diff --git a/scripts/benchmark_migration.py b/scripts/benchmark_migration.py index e4e49906d7da9..27670b5d4d729 100644 --- a/scripts/benchmark_migration.py +++ b/scripts/benchmark_migration.py @@ -44,9 +44,13 @@ def import_migration_script(filepath: Path) -> ModuleType: Import migration script as if it were a module. """ spec = importlib.util.spec_from_file_location(filepath.stem, filepath) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) # type: ignore - return module + if spec: + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) # type: ignore + return module + raise Exception( + "No module spec found in location: `{path}`".format(path=str(filepath)) + ) def extract_modified_tables(module: ModuleType) -> Set[str]: From 09ca27f951fd7fdc206368e4ec87f25c83977967 Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Fri, 13 Aug 2021 08:23:08 +0300 Subject: [PATCH 3/3] remove unnecessary stubs reqs --- requirements/base.in | 12 ------------ requirements/base.txt | 22 +--------------------- 2 files changed, 1 insertion(+), 33 deletions(-) diff --git a/requirements/base.in b/requirements/base.in index 669e9dd65ea1c..294583801824c 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -18,15 +18,3 @@ -e file:. pyrsistent>=0.16.1,<0.17 zipp==3.4.1 - -# stubs -types-Markdown -types-PyYAML -types-bleach -types-croniter -types-python-dateutil -types-pytz -types-redis -types-requests -types-setuptools -types-simplejson diff --git a/requirements/base.txt b/requirements/base.txt index d18ec7cc3fe2f..c8b1a00511956 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,4 +1,4 @@ -# SHA1:6ada23ace0591867da0f02fa77840f4df5239e65 +# SHA1:0862095245a068ae2fc00217da78331e1e7ae505 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -266,26 +266,6 @@ sqlparse==0.3.0 # via apache-superset tabulate==0.8.9 # via apache-superset -types-bleach==4.0.0 - # via -r requirements/base.in -types-croniter==1.0.0 - # via -r requirements/base.in -types-markdown==3.3.2 - # via -r requirements/base.in -types-python-dateutil==0.1.6 - # via -r requirements/base.in -types-pytz==2021.1.2 - # via -r requirements/base.in -types-pyyaml==5.4.6 - # via -r requirements/base.in -types-redis==3.5.6 - # via -r requirements/base.in -types-requests==2.25.6 - # via -r requirements/base.in -types-setuptools==57.0.2 - # via -r requirements/base.in -types-simplejson==0.1.5 - # via -r requirements/base.in typing-extensions==3.10.0.0 # via # aiohttp