Upgrade Python / linter versions for development

Pipfile

@@ -4,14 +4,15 @@ url = "https://pypi.org/simple"
 verify_ssl = true
 
 [packages]
-pysen = {editable = true,extras = ["lint"],path = "."}
+pysen = {editable = true, path = "."}
 pytest = ">=5.4.0,<6.0.0"
 tox = ">=3.15.0,<4.0.0"
-black = "==20.8b1"
+black = "==22.8.0"
 flake8 = "==3.8.4"
 flake8-bugbear = "==20.11.1"
 isort = "==5.1.4"
-mypy = "==0.782"
+mypy = "==0.971"
+types-setuptools = "==65.3.0"
 
 [requires]
-python_version = "3.7"
+python_version = "3.8"

examples/advanced_example/lint.py

@@ -20,7 +20,7 @@ def build(
             ),
         },
         mypy_targets=[
-            pysen.MypyTarget([pathlib.Path("."), pathlib.Path("tests")]),
+            pysen.MypyTarget([pathlib.Path(".")]),
             pysen.MypyTarget([pathlib.Path("tools")]),
         ],
     )

examples/sync_cmdclass_pyproject/setup.py

@@ -5,7 +5,7 @@
 except ImportError:
     import setuptools
 
-    setup = setuptools.setup
+    setup = setuptools.setup  # type: ignore
 
 
 setup(

pyproject.toml

@@ -10,10 +10,22 @@ enable_flake8 = true
 enable_isort = true
 enable_mypy = true
 mypy_preset = "very_strict"
-py_version = "py37"
+py_version = "py38"
 isort_known_first_party = ["fakes", "pysen"]
 [[tool.pysen.lint.mypy_targets]]
-  paths = [".", "tests/"]
+  paths = [
+    "pysen/",
+    "setup.py",
+    "tests/",
+  ]
+[[tool.pysen.lint.mypy_targets]]
+  paths = ["examples/advanced_example"]
+[[tool.pysen.lint.mypy_targets]]
+  paths = ["examples/simple_package"]
+[[tool.pysen.lint.mypy_targets]]
+  paths = ["examples/plugin_example"]
+[[tool.pysen.lint.mypy_targets]]
+  paths = ["examples/sync_cmdclass_pyproject"]
 
 [tool.pysen.lint.source]
   excludes = ["build/", "dist/", "examples/", ".tox"]
@@ -24,7 +36,7 @@ markers = ["examples"]
 [tool.black] # automatically generated by pysen
 # pysen ignores and overwrites any modifications
 line-length = 88
-target-version = ["py37"]
+target-version = ["py38"]
 
 [tool.isort] # automatically generated by pysen
 # pysen ignores and overwrites any modifications

pysen/cli_config.py

@@ -6,7 +6,7 @@
 import tomlkit
 
 from .exceptions import InvalidConfigurationError
-from .pyproject_model import _workaround_tomlkit_unmarshal
+from .pyproject_model import _get_descendant, _workaround_tomlkit_unmarshal
 
 
 @dataclasses.dataclass
@@ -27,11 +27,10 @@ def _load_cli_section(path: pathlib.Path) -> Optional[Dict[str, Any]]:
     with path.open("r") as f:
         pyproject = tomlkit.loads(f.read())
 
-    if "tool" not in pyproject or "pysen-cli" not in pyproject["tool"]:
+    section = _get_descendant(pyproject, ["tool", "pysen-cli"])
+    if section is None:
         return None
 
-    section = pyproject["tool"]["pysen-cli"]
-
     data = _workaround_tomlkit_unmarshal(section)
     assert isinstance(data, dict)
     return data

pysen/dumper.py

@@ -38,8 +38,8 @@ def _repr_cfg(data: Any) -> Optional[str]:
     elif isinstance(data, PRIMITIVE_TYPES):
         return str(data)
     elif isinstance(data, SEQUENCE_TYPES):
-        items = sorted(_repr_cfg(x) for x in data)
-        return ",".join(x for x in items if x is not None)
+        items = [_repr_cfg(x) for x in data]
+        return ",".join(sorted(x for x in items if x is not None))
     else:
         raise RuntimeError(f"{type(data)} is not supported in cfg")
 

pysen/error_lines.py

@@ -29,8 +29,8 @@ def parse_error_lines(
     """
     number = r"(?:0|[1-9]\d*)"
     _file_path = r"^(?P<file_path>.*?)"
-    _line = fr":(?P<line>{number})"
-    _column = fr"(:(?P<column>{number}))?"
+    _line = rf":(?P<line>{number})"
+    _column = rf"(:(?P<column>{number}))?"
     _message = r": (?P<message>.*$)"
     pattern = _file_path + _line + _column + _message
     invalid_lines = []

pysen/exceptions.py

@@ -1,4 +1,6 @@
-import pathlib
+from typing import Optional
+
+from .path import PathLikeType
 
 
 class PysenError(Exception):
@@ -23,12 +25,12 @@ class InvalidConfigurationError(PysenError):
 
 
 class InvalidPluginError(PysenError):
-    def __init__(self, module_path: str, error: str) -> None:
+    def __init__(self, module_path: Optional[PathLikeType], error: str) -> None:
         super().__init__(f"invalid plugin: {module_path}, {error}")
 
 
 class InvalidManifestBuilderError(PysenError):
-    def __init__(self, path: pathlib.Path, error: str) -> None:
+    def __init__(self, path: PathLikeType, error: str) -> None:
         super().__init__(f"invalid manifest builder: {path}, {error}")
 
 

pysen/git_utils.py

@@ -10,6 +10,9 @@
 
 try:
     import git
+    from git import Blob  # type: ignore
+
+    # NOTE: Upstream issue: https://github.com/gitpython-developers/GitPython/issues/1349
 
     _git_available = True
 except ImportError:
@@ -51,7 +54,7 @@ def _list_indexed_files(target_dir: pathlib.Path) -> Sequence[pathlib.Path]:
     # We avoid pathlib.Path because the loop calling predicate is performance critical.
     abs_target_dir = os.path.join(str(target_dir.resolve()), "")
 
-    def predicate(item: Tuple[int, git.Blob]) -> bool:
+    def predicate(item: Tuple[int, Blob]) -> bool:
         blob = item[1]
         ret: bool = blob.abspath.startswith(abs_target_dir)
         return ret
@@ -82,7 +85,7 @@ def _check_tracked(path: pathlib.Path) -> bool:
     # TODO(igarashi) use git command directly for better performance
     abspath = str(path.expanduser().resolve())
 
-    def predicate(item: Tuple[int, git.Blob]) -> bool:
+    def predicate(item: Tuple[int, Blob]) -> bool:
         blob = item[1]
         return cast(bool, blob.abspath == abspath)
 

pysen/logging_utils.py

@@ -67,7 +67,7 @@ def emit(self, record: logging.LogRecord) -> None:
     def shouldFlush(self, record: logging.LogRecord) -> bool:
         return False
 
-    def setFormatter(self, fmt: logging.Formatter) -> None:
+    def setFormatter(self, fmt: Optional[logging.Formatter]) -> None:
         self.target: Optional[logging.Handler]
         assert self.target is not None
         self.target.setFormatter(fmt)

pysen/process_utils.py

@@ -8,7 +8,10 @@
 from .reporter import Reporter
 
 
-def _read_stream(stream: IO[str], reporter: Reporter, loglevel: int) -> str:
+def _read_stream(stream: Optional[IO[str]], reporter: Reporter, loglevel: int) -> str:
+    if stream is None:
+        return ""
+
     ret: List[str] = []
     for line in stream:
         ret.append(line)

pysen/py_version.py

@@ -28,10 +28,10 @@ def version(self) -> str:
     @classmethod
     def from_str(cls, s: str) -> "VersionRepresentation":
         number = r"(?:0|[1-9]\d*)"
-        major = fr"^(?P<major>{number})"
-        minor = fr"\.(?P<minor>{number})"
-        patch = fr"(\.(?P<patch>{number}))?"
-        pre_release = fr"(?P<pre_release>(a|b|rc){number})?$"
+        major = rf"^(?P<major>{number})"
+        minor = rf"\.(?P<minor>{number})"
+        patch = rf"(\.(?P<patch>{number}))?"
+        pre_release = rf"(?P<pre_release>(a|b|rc){number})?$"
         pattern = major + minor + patch + pre_release
         m = re.match(pattern, s)
 

pysen/pyproject_model.py

@@ -1,7 +1,7 @@
 import dataclasses
 import logging
 import pathlib
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Sequence
 
 import dacite
 import tomlkit
@@ -317,11 +317,33 @@ def _workaround_tomlkit_unmarshal(data: Any) -> Any:
     return data
 
 
+def _get_descendant(
+    document: tomlkit.toml_document.TOMLDocument, paths: Sequence[str]
+) -> Optional[Any]:
+    assert len(paths) > 0
+
+    node: Dict[str, Any] = document
+    for p in paths[:-1]:
+        e = node.get(p)
+        if not isinstance(e, dict):
+            return None
+        node = e
+
+    return node.get(paths[-1])
+
+
+def get_tool_section(
+    tool_name: str,
+    pyproject: tomlkit.toml_document.TOMLDocument,
+) -> Optional[tomlkit.toml_document.TOMLDocument]:
+    return _get_descendant(pyproject, ["tool", tool_name])
+
+
 def has_tool_section(
     tool_name: str,
     pyproject: tomlkit.toml_document.TOMLDocument,
 ) -> bool:
-    return "tool" in pyproject and tool_name in pyproject["tool"]
+    return get_tool_section(tool_name, pyproject) is not None
 
 
 def _load_pysen_section(path: pathlib.Path) -> Dict[str, Any]:
@@ -331,14 +353,15 @@ def _load_pysen_section(path: pathlib.Path) -> Dict[str, Any]:
     with path.open("r") as f:
         pyproject = tomlkit.loads(f.read())
 
-    if has_tool_section("pysen", pyproject):
-        section = pyproject["tool"]["pysen"]
-    elif has_tool_section("jiro", pyproject):
-        _logger.warning(
-            "jiro section under a config file is deprecated. Use pysen instead."
-        )
-        section = pyproject["tool"]["jiro"]
-    else:
+    section = get_tool_section("pysen", pyproject)
+    if section is None:
+        section = get_tool_section("jiro", pyproject)
+        if section is not None:
+            _logger.warning(
+                "jiro section under a config file is deprecated. Use pysen instead."
+            )
+
+    if section is None:
         raise PysenSectionNotFoundError(str(path))
 
     data = _workaround_tomlkit_unmarshal(section)

pysen/setuptools.py

@@ -1,4 +1,5 @@
 import argparse
+import distutils.core
 import functools
 import pathlib
 import sys
@@ -14,7 +15,7 @@
 from .runner_options import RunOptions
 
 ManifestLikeType = Union[str, pathlib.Path, ManifestBase]
-CommandClassType = Type[setuptools.Command]
+CommandClassType = Type[distutils.core.Command]
 UNDEFINED = object()
 
 _PREDEFINED_COMMAND_NAMES = [
@@ -42,7 +43,7 @@ def _get_setuptool_command(name: str) -> CommandClassType:
         except BaseException:
             pass  # failover
 
-    return setuptools.Command  # type: ignore
+    return setuptools.Command
 
 
 def _get_setuptool_user_options(

setup.cfg

@@ -24,7 +24,7 @@ disallow_untyped_defs = True
 ignore_errors = False
 ignore_missing_imports = True
 no_implicit_optional = True
-python_version = 3.7
+python_version = 3.8
 show_error_codes = True
 strict_equality = True
 strict_optional = True

tests/example_tests/test_sync_cmdclass_pyproject.py

@@ -33,4 +33,4 @@ def test_setuptools_sandbox(example_dir: pathlib.Path) -> None:
     setup_py = target / "setup.py"
     assert setup_py.exists()
 
-    sandbox.run_setup(str(setup_py), ["lint"])
+    sandbox.run_setup(str(setup_py), ["lint"])  # type: ignore

tests/fakes/configs/error_builder.py

@@ -7,7 +7,7 @@
 raise BufferError()
 
 
-def build(
+def build(  # type: ignore
     components: Sequence[ComponentBase], src_path: Optional[pathlib.Path]
 ) -> ManifestBase:
     assert src_path is not None

tests/test_setting.py

@@ -1,6 +1,7 @@
 import pytest
 import tomlkit
 
+from pysen.pyproject_model import _get_descendant
 from pysen.setting import _create_dict, _traverse_toml
 
 
@@ -27,4 +28,4 @@ def test__traverse_toml() -> None:
     with pytest.raises(ValueError):
         _traverse_toml(("tool", "hoge", "answer"), document, False)
     _traverse_toml(("tool", "hoge", "answer"), document, True)
-    assert document["tool"]["hoge"]["answer"] == {}
+    assert _get_descendant(document, ["tool", "hoge", "answer"]) == {}

tests/test_setuptools.py

@@ -45,7 +45,7 @@ def test__get_setuptool_user_options() -> None:
     assert build_py_options is not None
     assert len(build_py_options) > 0
 
-    command_options = _get_setuptool_user_options(setuptools.Command)
+    command_options = _get_setuptool_user_options(setuptools.Command)  # type: ignore
     assert command_options is not None
     assert command_options == []
 

tox.ini

@@ -24,19 +24,21 @@ setenv =
     LANG = C
 
 [testenv:latest]
-basepython = python3.7
+basepython = python3.8
 deps =
-    pipenv==2020.11.15
+    pipenv==2022.9.8
 commands =
-    pipenv sync
-    pipenv run pip install -U black
-    pipenv run pytest
+    pipenv sync --system
+    pip install .
+    pip install -U black
+    pytest
 
 [testenv:development]
-basepython = python3.7
+basepython = python3.8
 deps =
-    pipenv==2020.11.15
+    pipenv==2022.9.8
 commands =
-    pipenv sync
-    pipenv run pysen run lint
-    pipenv run pytest
+    pipenv sync --system
+    pip install .
+    pysen run lint
+    pytest