Add types to synapse.util. (#10601)

changelog.d/10601.misc

@@ -0,0 +1 @@
+Add type annotations to the synapse.util package.

mypy.ini

@@ -74,17 +74,7 @@ files =
   synapse/storage/util,
   synapse/streams,
   synapse/types.py,
-  synapse/util/async_helpers.py,
-  synapse/util/caches,
-  synapse/util/daemonize.py,
-  synapse/util/hash.py,
-  synapse/util/iterutils.py,
-  synapse/util/linked_list.py,
-  synapse/util/metrics.py,
-  synapse/util/macaroons.py,
-  synapse/util/module_loader.py,
-  synapse/util/msisdn.py,
-  synapse/util/stringutils.py,
+  synapse/util,
   synapse/visibility.py,
   tests/replication,
   tests/test_event_auth.py,
@@ -102,6 +92,69 @@ files =
 [mypy-synapse.rest.client.*]
 disallow_untyped_defs = True
 
+[mypy-synapse.util.batching_queue]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.caches.dictionary_cache]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.file_consumer]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.frozenutils]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.hash]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.httpresourcetree]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.iterutils]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.linked_list]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.logcontext]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.logformatter]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.macaroons]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.manhole]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.module_loader]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.msisdn]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.ratelimitutils]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.retryutils]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.rlimit]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.stringutils]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.templates]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.threepids]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.wheel_timer]
+disallow_untyped_defs = True
+
 [mypy-pymacaroons.*]
 ignore_missing_imports = True
 

stubs/txredisapi.pyi

@@ -73,4 +73,4 @@ class RedisFactory(protocol.ReconnectingClientFactory):
     def buildProtocol(self, addr) -> RedisProtocol: ...
 
 class SubscriberFactory(RedisFactory):
-    def __init__(self): ...
+    def __init__(self) -> None: ...

synapse/api/ratelimiting.py

@@ -46,7 +46,7 @@ def __init__(
         #   * How many times an action has occurred since a point in time
         #   * The point in time
         #   * The rate_hz of this particular entry. This can vary per request
-        self.actions: OrderedDict[Hashable, Tuple[float, int, float]] = OrderedDict()
+        self.actions: OrderedDict[Hashable, Tuple[float, float, float]] = OrderedDict()
 
     async def can_do_action(
         self,
@@ -56,7 +56,7 @@ async def can_do_action(
         burst_count: Optional[int] = None,
         update: bool = True,
         n_actions: int = 1,
-        _time_now_s: Optional[int] = None,
+        _time_now_s: Optional[float] = None,
     ) -> Tuple[bool, float]:
         """Can the entity (e.g. user or IP address) perform the action?
 
@@ -160,7 +160,7 @@ async def can_do_action(
 
         return allowed, time_allowed
 
-    def _prune_message_counts(self, time_now_s: int):
+    def _prune_message_counts(self, time_now_s: float):
         """Remove message count entries that have not exceeded their defined
         rate_hz limit
 
@@ -188,7 +188,7 @@ async def ratelimit(
         burst_count: Optional[int] = None,
         update: bool = True,
         n_actions: int = 1,
-        _time_now_s: Optional[int] = None,
+        _time_now_s: Optional[float] = None,
     ):
         """Checks if an action can be performed. If not, raises a LimitExceededError
 

synapse/config/ratelimiting.py

@@ -14,6 +14,8 @@
 
 from typing import Dict, Optional
 
+import attr
+
 from ._base import Config
 
 
@@ -29,18 +31,13 @@ def __init__(
         self.burst_count = int(config.get("burst_count", defaults["burst_count"]))
 
 
+@attr.s(auto_attribs=True)
 class FederationRateLimitConfig:
-    _items_and_default = {
-        "window_size": 1000,
-        "sleep_limit": 10,
-        "sleep_delay": 500,
-        "reject_limit": 50,
-        "concurrent": 3,
-    }
-
-    def __init__(self, **kwargs):
-        for i in self._items_and_default.keys():
-            setattr(self, i, kwargs.get(i) or self._items_and_default[i])
+    window_size: int = 1000
+    sleep_limit: int = 10
+    sleep_delay: int = 500
+    reject_limit: int = 50
+    concurrent: int = 3
 
 
 class RatelimitConfig(Config):
@@ -69,11 +66,15 @@ def read_config(self, config, **kwargs):
         else:
             self.rc_federation = FederationRateLimitConfig(
                 **{
-                    "window_size": config.get("federation_rc_window_size"),
-                    "sleep_limit": config.get("federation_rc_sleep_limit"),
-                    "sleep_delay": config.get("federation_rc_sleep_delay"),
-                    "reject_limit": config.get("federation_rc_reject_limit"),
-                    "concurrent": config.get("federation_rc_concurrent"),
+                    k: v
+                    for k, v in {
+                        "window_size": config.get("federation_rc_window_size"),
+                        "sleep_limit": config.get("federation_rc_sleep_limit"),
+                        "sleep_delay": config.get("federation_rc_sleep_delay"),
+                        "reject_limit": config.get("federation_rc_reject_limit"),
+                        "concurrent": config.get("federation_rc_concurrent"),
+                    }.items()
+                    if v is not None
                 }
             )
 

synapse/federation/sender/__init__.py

@@ -22,6 +22,7 @@
 from typing_extensions import Literal
 
 from twisted.internet import defer
+from twisted.internet.interfaces import IDelayedCall
 
 import synapse.metrics
 from synapse.api.presence import UserPresenceState
@@ -284,7 +285,9 @@ def __init__(self, hs: "HomeServer"):
         )
 
         # wake up destinations that have outstanding PDUs to be caught up
-        self._catchup_after_startup_timer = self.clock.call_later(
+        self._catchup_after_startup_timer: Optional[
+            IDelayedCall
+        ] = self.clock.call_later(
             CATCH_UP_STARTUP_DELAY_SEC,
             run_as_background_process,
             "wake_destinations_needing_catchup",
@@ -406,7 +409,7 @@ async def handle_event(event: EventBase) -> None:
 
                         now = self.clock.time_msec()
                         ts = await self.store.get_received_ts(event.event_id)
-
+                        assert ts is not None
                         synapse.metrics.event_processing_lag_by_event.labels(
                             "federation_sender"
                         ).observe((now - ts) / 1000)
@@ -435,6 +438,7 @@ async def handle_room_events(events: Iterable[EventBase]) -> None:
                 if events:
                     now = self.clock.time_msec()
                     ts = await self.store.get_received_ts(events[-1].event_id)
+                    assert ts is not None
 
                     synapse.metrics.event_processing_lag.labels(
                         "federation_sender"

synapse/handlers/account_validity.py

@@ -398,6 +398,7 @@ async def renew_account_for_user(
         """
         now = self.clock.time_msec()
         if expiration_ts is None:
+            assert self._account_validity_period is not None
             expiration_ts = now + self._account_validity_period
 
         await self.store.set_account_validity_for_user(

synapse/handlers/appservice.py

@@ -131,6 +131,8 @@ async def start_scheduler():
 
                         now = self.clock.time_msec()
                         ts = await self.store.get_received_ts(event.event_id)
+                        assert ts is not None
+
                         synapse.metrics.event_processing_lag_by_event.labels(
                             "appservice_sender"
                         ).observe((now - ts) / 1000)
@@ -166,6 +168,7 @@ async def handle_room_events(events):
                     if events:
                         now = self.clock.time_msec()
                         ts = await self.store.get_received_ts(events[-1].event_id)
+                        assert ts is not None
 
                         synapse.metrics.event_processing_lag.labels(
                             "appservice_sender"

synapse/handlers/presence.py

@@ -28,6 +28,7 @@
 from contextlib import contextmanager
 from typing import (
     TYPE_CHECKING,
+    Any,
     Callable,
     Collection,
     Dict,
@@ -615,7 +616,7 @@ def __init__(self, hs: "HomeServer"):
         super().__init__(hs)
         self.hs = hs
         self.server_name = hs.hostname
-        self.wheel_timer = WheelTimer()
+        self.wheel_timer: WheelTimer[str] = WheelTimer()
         self.notifier = hs.get_notifier()
         self._presence_enabled = hs.config.use_presence
 
@@ -924,7 +925,7 @@ async def bump_presence_active_time(self, user: UserID) -> None:
 
         prev_state = await self.current_state_for_user(user_id)
 
-        new_fields = {"last_active_ts": self.clock.time_msec()}
+        new_fields: Dict[str, Any] = {"last_active_ts": self.clock.time_msec()}
         if prev_state.state == PresenceState.UNAVAILABLE:
             new_fields["state"] = PresenceState.ONLINE
 

synapse/handlers/typing.py

@@ -73,7 +73,7 @@ def __init__(self, hs: "HomeServer"):
         self._room_typing: Dict[str, Set[str]] = {}
 
         self._member_last_federation_poke: Dict[RoomMember, int] = {}
-        self.wheel_timer = WheelTimer(bucket_size=5000)
+        self.wheel_timer: WheelTimer[RoomMember] = WheelTimer(bucket_size=5000)
         self._latest_room_serial = 0
 
         self.clock.looping_call(self._handle_timeouts, 5000)

synapse/rest/client/register.py

@@ -330,11 +330,11 @@ def __init__(self, hs: "HomeServer"):
                 # Artificially delay requests if rate > sleep_limit/window_size
                 sleep_limit=1,
                 # Amount of artificial delay to apply
-                sleep_msec=1000,
+                sleep_delay=1000,
                 # Error with 429 if more than reject_limit requests are queued
                 reject_limit=1,
                 # Allow 1 request at a time
-                concurrent_requests=1,
+                concurrent=1,
             ),
         )
 
@@ -763,7 +763,10 @@ async def _create_registration_details(
         Returns:
              dictionary for response from /register
         """
-        result = {"user_id": user_id, "home_server": self.hs.hostname}
+        result: JsonDict = {
+            "user_id": user_id,
+            "home_server": self.hs.hostname,
+        }
         if not params.get("inhibit_login", False):
             device_id = params.get("device_id")
             initial_display_name = params.get("initial_device_display_name")
@@ -814,7 +817,7 @@ async def _do_guest_registration(
             user_id, device_id, initial_display_name, is_guest=True
         )
 
-        result = {
+        result: JsonDict = {
             "user_id": user_id,
             "device_id": device_id,
             "access_token": access_token,

synapse/rest/synapse/client/new_user_consent.py

@@ -52,7 +52,7 @@ def template_search_dirs():
                 yield hs.config.sso.sso_template_dir
             yield hs.config.sso.default_template_dir
 
-        self._jinja_env = build_jinja_env(template_search_dirs(), hs.config)
+        self._jinja_env = build_jinja_env(list(template_search_dirs()), hs.config)
 
     async def _async_render_GET(self, request: Request) -> None:
         try:

synapse/rest/synapse/client/pick_username.py

@@ -80,7 +80,7 @@ def template_search_dirs():
                 yield hs.config.sso.sso_template_dir
             yield hs.config.sso.default_template_dir
 
-        self._jinja_env = build_jinja_env(template_search_dirs(), hs.config)
+        self._jinja_env = build_jinja_env(list(template_search_dirs()), hs.config)
 
     async def _async_render_GET(self, request: Request) -> None:
         try:

synapse/storage/databases/main/registration.py

@@ -1091,6 +1091,7 @@ def set_expiration_date_for_user_txn(self, txn, user_id, use_delta=False):
                 delta equal to 10% of the validity period.
         """
         now_ms = self._clock.time_msec()
+        assert self._account_validity_period is not None
         expiration_ts = now_ms + self._account_validity_period
 
         if use_delta:

synapse/types.py

@@ -38,6 +38,7 @@
     IReactorCore,
     IReactorPluggableNameResolver,
     IReactorTCP,
+    IReactorThreads,
     IReactorTime,
 )
 
@@ -63,7 +64,12 @@
 # Note that this seems to require inheriting *directly* from Interface in order
 # for mypy-zope to realize it is an interface.
 class ISynapseReactor(
-    IReactorTCP, IReactorPluggableNameResolver, IReactorTime, IReactorCore, Interface
+    IReactorTCP,
+    IReactorPluggableNameResolver,
+    IReactorTime,
+    IReactorCore,
+    IReactorThreads,
+    Interface,
 ):
     """The interfaces necessary for Synapse to function."""