Introduce mypy and enforce it on a few files

.gitignore

@@ -95,5 +95,6 @@ UNKNOWN.egg-info/
 
 /.idea
 
+/.mypy_cache
 /.pytest_cache
 /docs/html

.pre-commit-config.yaml

@@ -0,0 +1,24 @@
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v1.3.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: check-yaml
+    -   id: check-added-large-files
+    -   id: flake8
+        exclude: ^docs/conf.py$
+-   repo: https://github.com/asottile/seed-isort-config
+    rev: v1.0.1
+    hooks:
+    -   id: seed-isort-config
+        args: [--application-directories, '.:src']
+-   repo: https://github.com/pre-commit/mirrors-isort
+    rev: v4.3.4
+    hooks:
+    -   id: isort
+-   repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.610-1
+    hooks:
+    -   id: mypy
+        exclude: ^docs/conf.py$

.travis.yml

@@ -9,6 +9,8 @@ matrix:
       env: TOXENV=py37
     - python: 3.6
       env: TOXENV=doc_build
+    - python: 3.6
+      env: TOXENV=lint
 
 install:
     - pip install --upgrade pip
@@ -23,3 +25,8 @@ notifications:
   irc:
     channels:
       - "chat.freenode.net#bandersnatch"
+
+cache:
+    directories:
+        - $HOME/.cache/pip
+        - $HOME/.cache/pre-commit

docs/conf.py

@@ -246,7 +246,7 @@ class DocStub(object):
 if selected_theme not in available_themes:
     selected_theme = available_themes[0]
     print('SPHINX_THEME is not installed, using %r theme' % selected_theme)
-    
+
 html_theme = available_theme_settings[selected_theme]['theme']
 html_theme_path = available_theme_settings[selected_theme]['path']
 if available_theme_settings[selected_theme].get('options', None):
@@ -381,4 +381,4 @@ def setup(app):
             'enable_eval_rst': True,
             'enable_auto_doc_ref': True,
         }, True)
-        app.add_transform(AutoStructify)
+        app.add_transform(AutoStructify)

mypy.ini

@@ -0,0 +1,16 @@
+[mypy]
+
+[mypy-bandersnatch.configuration]
+disallow_untyped_defs = True
+
+[mypy-bandersnatch.filter]
+disallow_untyped_defs = True
+
+[mypy-bandersnatch.log]
+disallow_untyped_defs = True
+
+[mypy-bandersnatch.release]
+disallow_untyped_defs = True
+
+[mypy-bandersnatch.util]
+disallow_untyped_defs = True

setup.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
-from setuptools import setup, find_packages
+from setuptools import find_packages, setup
+
 from src.bandersnatch import __version__
 
 install_deps = [
@@ -45,7 +46,7 @@
                 update_stable_tag = bandersnatch.release:update_stable_tag
             [zest.releaser.postreleaser.after]
                 update_requirements = bandersnatch.release:update_requirements
-      """,
+      """,  # noqa
     classifiers=[
         'Programming Language :: Python :: 3 :: Only',
         'Programming Language :: Python :: 3.6',

src/bandersnatch/__init__.py

@@ -1,21 +1,25 @@
 #!/usr/bin/env python3
+from typing import NamedTuple
 
-from collections import namedtuple
 
-# a namedtuple like that given by sys.version_info
-__version_info__ = namedtuple(
-    'version_info',
-    'major minor micro releaselevel serial')(
-        major=3,
-        minor=0,
-        micro=0,
-        releaselevel='dev0',
-        serial=0  # Not currently in use below ...
-    )
+class _VersionInfo(NamedTuple):
+    major: int
+    minor: int
+    micro: int
+    releaselevel: str
+    serial: int
 
-if __version_info__.releaselevel:
-    __version__ = '{v.major}.{v.minor}.{v.micro}.{v.releaselevel}'.format(
-        v=__version_info__
-    )
-else:
-    __version__ = '{v.major}.{v.minor}.{v.micro}'.format(v=__version_info__)
+    @property
+    def version_str(self) -> str:
+        extra = f'.{self.releaselevel}' if self.releaselevel else ''
+        return f'{self.major}.{self.minor}.{self.micro}.{extra}'
+
+
+__version_info__ = _VersionInfo(
+    major=3,
+    minor=0,
+    micro=0,
+    releaselevel='dev0',
+    serial=0  # Not currently in use below ...
+)
+__version__ = __version_info__.version_str

src/bandersnatch/buildout.py

@@ -1,12 +1,13 @@
 import os.path
+
 import pkg_resources
 
 
 class Requirements(object):
     """Generate a pip compatible requirements.txt from our working set based on
     the 'bandersnatch' requirement."""
 
-    def __init__(self, buildout, name, options):
+    def __init__(self, buildout, name, options) -> None:
         self.name, self.options = name, options
         options['path'] = os.path.join(
             buildout['buildout']['directory'], 'requirements.txt')

src/bandersnatch/configuration.py

@@ -1,14 +1,16 @@
 """
 Module containing classes to access the bandersnatch configuration file
 """
-import pkg_resources
 from configparser import ConfigParser
+from typing import Any, Dict, Optional, Type
+
+import pkg_resources
 
 
 class Singleton(type):  # pragma: no cover
-    _instances = {}
+    _instances: Dict['Singleton', Type] = {}
 
-    def __call__(cls, *args, **kwargs):
+    def __call__(cls, *args: Any, **kwargs: Any) -> Type:
         if cls not in cls._instances:
             cls._instances[cls] = super(
                 Singleton, cls
@@ -17,7 +19,7 @@ def __call__(cls, *args, **kwargs):
 
 
 class BandersnatchConfig(metaclass=Singleton):
-    def __init__(self, config_file=None):
+    def __init__(self, config_file: Optional[str] = None) -> None:
         """
         Bandersnatch configuration class singleton
 
@@ -35,7 +37,7 @@ def __init__(self, config_file=None):
         self.config_file = config_file
         self.load_configuration()
 
-    def load_configuration(self):
+    def load_configuration(self) -> None:
         """
         Read the configuration from the configuration files
         """

src/bandersnatch/filter.py

@@ -1,12 +1,14 @@
 """
 Blacklist management
 """
-import pkg_resources
 from collections import defaultdict
-from .configuration import BandersnatchConfig
+from typing import Any, Dict, Iterable, List
 
+import pkg_resources
+
+from .configuration import BandersnatchConfig
 
-loaded_filter_plugins = defaultdict(list)
+loaded_filter_plugins: Dict[str, List['Filter']] = defaultdict(list)
 
 
 class Filter(object):
@@ -15,17 +17,17 @@ class Filter(object):
     """
     name = 'filter'
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
         self.configuration = BandersnatchConfig().config
         self.initialize_plugin()
 
-    def initialize_plugin(self):
+    def initialize_plugin(self) -> None:
         """
         Code to initialize the plugin
         """
         pass
 
-    def check_match(self, **kwargs):
+    def check_match(self, **kwargs: Any) -> bool:
         """
         Check if the plugin matches based on the arguments provides.
 
@@ -51,7 +53,7 @@ class FilterReleasePlugin(Filter):
     name = 'release_plugin'
 
 
-def load_filter_plugins(entrypoint_group):
+def load_filter_plugins(entrypoint_group: str) -> Iterable[Filter]:
     """
     Load all blacklist plugins that are registered with pkg_resources
 
@@ -71,7 +73,7 @@ def load_filter_plugins(entrypoint_group):
     try:
         config_blacklist_plugins = config['blacklist']['plugins']
     except KeyError:
-        config_blacklist_plugins = None
+        config_blacklist_plugins = ''
     if config_blacklist_plugins:
         config_plugins = []
         for plugin in config_blacklist_plugins.split('\n'):
@@ -98,7 +100,7 @@ def load_filter_plugins(entrypoint_group):
     return plugins
 
 
-def filter_project_plugins():
+def filter_project_plugins() -> Iterable[Filter]:
     """
     Load and return the release filtering plugin objects
 
@@ -110,7 +112,7 @@ def filter_project_plugins():
     return load_filter_plugins('bandersnatch_filter_plugins.project')
 
 
-def filter_release_plugins():
+def filter_release_plugins() -> Iterable[Filter]:
     """
     Load and return the release filtering plugin objects
 

src/bandersnatch/log.py

@@ -2,9 +2,10 @@
 # coverage analysis. Unfortunately this is really hard to test as the Python
 # logging module won't allow reasonable teardown. :(
 import logging
+from typing import Any
 
 
-def setup_logging(args):
+def setup_logging(args: Any) -> logging.StreamHandler:
     ch = logging.StreamHandler()
     formatter = logging.Formatter(
         '%(asctime)s %(levelname)s: %(message)s')

src/bandersnatch/main.py

@@ -1,16 +1,17 @@
 import argparse
-import bandersnatch.log
-import bandersnatch.master
-import bandersnatch.mirror
-import bandersnatch.utils
-import bandersnatch.verify
 import configparser
 import logging
 import logging.config
 import os.path
 import shutil
-from .configuration import BandersnatchConfig
 
+import bandersnatch.log
+import bandersnatch.master
+import bandersnatch.mirror
+import bandersnatch.utils
+import bandersnatch.verify
+
+from .configuration import BandersnatchConfig
 
 logger = logging.getLogger(__name__)
 

src/bandersnatch/master.py

@@ -1,8 +1,10 @@
-from .utils import USER_AGENT
 import logging
+
 import requests
 import xmlrpc2
 
+from .utils import USER_AGENT
+
 logger = logging.getLogger(__name__)
 
 

src/bandersnatch/mirror.py

@@ -1,15 +1,16 @@
-from .filter import filter_project_plugins
-from .package import Package
-from .utils import rewrite, USER_AGENT, update_safe
-from packaging.utils import canonicalize_name
-from threading import RLock
 import asyncio
 import concurrent.futures
 import datetime
 import fcntl
 import logging
 import os
+from threading import RLock
 
+from packaging.utils import canonicalize_name
+
+from .filter import filter_project_plugins
+from .package import Package
+from .utils import USER_AGENT, rewrite, update_safe
 
 logger = logging.getLogger(__name__)
 

src/bandersnatch/package.py

@@ -1,17 +1,18 @@
-from . import utils
-from .filter import filter_release_plugins
-from .master import StalePage
-from packaging.utils import canonicalize_name
-from urllib.parse import urlparse, unquote
 import hashlib
 import json
 import logging
 import os.path
-import pkg_resources
-import requests
 import sys
 import time
+from urllib.parse import unquote, urlparse
 
+import pkg_resources
+import requests
+from packaging.utils import canonicalize_name
+
+from . import utils
+from .filter import filter_release_plugins
+from .master import StalePage
 
 logger = logging.getLogger(__name__)
 

src/bandersnatch/release.py

@@ -1,19 +1,21 @@
-from zest.releaser.utils import execute_command
 import logging
 import os
+from typing import Dict
+
+from zest.releaser.utils import execute_command
 
 logger = logging.getLogger(__name__)
 
 
-def update_requirements(data):
+def update_requirements(data: Dict[str, str]) -> None:
     os.chdir(data['workingdir'])
     logger.info('Running buildout to update requirements.txt.')
     execute_command('bin/buildout')
     logger.info('Committing requirements.txt.')
     execute_command('hg commit -v -m "Update requirements.txt"')
 
 
-def update_stable_tag(data):
+def update_stable_tag(data: Dict[str, str]) -> None:
     os.chdir(data['workingdir'])
     logger.info('Updating stable tag.')
     execute_command('hg tag -f -r %s stable' % data['version'])

src/bandersnatch/tests/conftest.py

@@ -1,4 +1,5 @@
 import unittest.mock as mock
+
 import pytest