Add types to synapse.util. (#10601)

changelog.d/10601.misc

@@ -0,0 +1 @@
+Add type annotations to the synapse.util package.

mypy.ini

@@ -74,17 +74,7 @@ files =
  synapse/storage/util,
  synapse/streams,
  synapse/types.py,
- synapse/util/async_helpers.py,
- synapse/util/caches,
- synapse/util/daemonize.py,
- synapse/util/hash.py,
- synapse/util/iterutils.py,
- synapse/util/linked_list.py,
- synapse/util/metrics.py,
- synapse/util/macaroons.py,
- synapse/util/module_loader.py,
- synapse/util/msisdn.py,
- synapse/util/stringutils.py,
+ synapse/util,
  synapse/visibility.py,
  tests/replication,
  tests/test_event_auth.py,
@@ -102,6 +92,69 @@ files =
 [mypy-synapse.rest.client.*]
 disallow_untyped_defs = True
 
+[mypy-synapse.util.batching_queue]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.caches.dictionary_cache]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.file_consumer]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.frozenutils]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.hash]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.httpresourcetree]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.iterutils]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.linked_list]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.logcontext]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.logformatter]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.macaroons]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.manhole]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.module_loader]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.msisdn]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.ratelimitutils]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.retryutils]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.rlimit]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.stringutils]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.templates]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.threepids]
+disallow_untyped_defs = True
+
+[mypy-synapse.util.wheel_timer]
+disallow_untyped_defs = True
+
 [mypy-pymacaroons.*]
 ignore_missing_imports = True
 

stubs/txredisapi.pyi

@@ -73,4 +73,4 @@ class RedisFactory(protocol.ReconnectingClientFactory):
  def buildProtocol(self, addr) -> RedisProtocol: ...
 
 class SubscriberFactory(RedisFactory):
- def __init__(self): ...
+ def __init__(self) -> None: ...

synapse/api/ratelimiting.py

@@ -46,7 +46,7 @@ def __init__(
  # * How many times an action has occurred since a point in time
  # * The point in time
  # * The rate_hz of this particular entry. This can vary per request
- self.actions: OrderedDict[Hashable, Tuple[float, int, float]] = OrderedDict()
+ self.actions: OrderedDict[Hashable, Tuple[float, float, float]] = OrderedDict()
 
  async def can_do_action(
  self,
@@ -56,7 +56,7 @@ async def can_do_action(
  burst_count: Optional[int] = None,
  update: bool = True,
  n_actions: int = 1,
- _time_now_s: Optional[int] = None,
+ _time_now_s: Optional[float] = None,
  ) -> Tuple[bool, float]:
  """Can the entity (e.g. user or IP address) perform the action?
 
@@ -160,7 +160,7 @@ async def can_do_action(
 
  return allowed, time_allowed
 
- def _prune_message_counts(self, time_now_s: int):
+ def _prune_message_counts(self, time_now_s: float):
  """Remove message count entries that have not exceeded their defined
  rate_hz limit
 
@@ -188,7 +188,7 @@ async def ratelimit(
  burst_count: Optional[int] = None,
  update: bool = True,
  n_actions: int = 1,
- _time_now_s: Optional[int] = None,
+ _time_now_s: Optional[float] = None,
  ):
  """Checks if an action can be performed. If not, raises a LimitExceededError
 

synapse/config/ratelimiting.py

@@ -14,6 +14,8 @@
 
 from typing import Dict, Optional
 
+import attr
+
 from ._base import Config
 
 
@@ -29,18 +31,13 @@ def __init__(
  self.burst_count = int(config.get("burst_count", defaults["burst_count"]))
 
 
+@attr.s(auto_attribs=True)
 class FederationRateLimitConfig:
- _items_and_default = {
- "window_size": 1000,
- "sleep_limit": 10,
- "sleep_delay": 500,
- "reject_limit": 50,
- "concurrent": 3,
- }
-
- def __init__(self, **kwargs):
- for i in self._items_and_default.keys():
- setattr(self, i, kwargs.get(i) or self._items_and_default[i])
+ window_size: int = 1000
+ sleep_limit: int = 10
+ sleep_delay: int = 500
+ reject_limit: int = 50
+ concurrent: int = 3
 
 
 class RatelimitConfig(Config):
@@ -69,11 +66,15 @@ def read_config(self, config, **kwargs):
  else:
  self.rc_federation = FederationRateLimitConfig(
  **{
- "window_size": config.get("federation_rc_window_size"),
- "sleep_limit": config.get("federation_rc_sleep_limit"),
- "sleep_delay": config.get("federation_rc_sleep_delay"),
- "reject_limit": config.get("federation_rc_reject_limit"),
- "concurrent": config.get("federation_rc_concurrent"),
+ k: v
+ for k, v in {
+ "window_size": config.get("federation_rc_window_size"),
+ "sleep_limit": config.get("federation_rc_sleep_limit"),
+ "sleep_delay": config.get("federation_rc_sleep_delay"),
+ "reject_limit": config.get("federation_rc_reject_limit"),
+ "concurrent": config.get("federation_rc_concurrent"),
+ }.items()
+ if v is not None
  }
  )
 

synapse/federation/sender/__init__.py

@@ -22,6 +22,7 @@
 from typing_extensions import Literal
 
 from twisted.internet import defer
+from twisted.internet.interfaces import IDelayedCall
 
 import synapse.metrics
 from synapse.api.presence import UserPresenceState
@@ -284,7 +285,9 @@ def __init__(self, hs: "HomeServer"):
  )
 
  # wake up destinations that have outstanding PDUs to be caught up
- self._catchup_after_startup_timer = self.clock.call_later(
+ self._catchup_after_startup_timer: Optional[
+ IDelayedCall
+ ] = self.clock.call_later(
  CATCH_UP_STARTUP_DELAY_SEC,
  run_as_background_process,
  "wake_destinations_needing_catchup",
@@ -406,7 +409,7 @@ async def handle_event(event: EventBase) -> None:
 
  now = self.clock.time_msec()
  ts = await self.store.get_received_ts(event.event_id)
-
+ assert ts is not None
  synapse.metrics.event_processing_lag_by_event.labels(
  "federation_sender"
  ).observe((now - ts) / 1000)
@@ -435,6 +438,7 @@ async def handle_room_events(events: Iterable[EventBase]) -> None:
  if events:
  now = self.clock.time_msec()
  ts = await self.store.get_received_ts(events[-1].event_id)
+ assert ts is not None
 
  synapse.metrics.event_processing_lag.labels(
  "federation_sender"

synapse/handlers/account_validity.py

@@ -398,6 +398,7 @@ async def renew_account_for_user(
  """
  now = self.clock.time_msec()
  if expiration_ts is None:
+ assert self._account_validity_period is not None
  expiration_ts = now + self._account_validity_period
 
  await self.store.set_account_validity_for_user(

synapse/handlers/appservice.py

@@ -131,6 +131,8 @@ async def start_scheduler():
 
  now = self.clock.time_msec()
  ts = await self.store.get_received_ts(event.event_id)
+ assert ts is not None
+
  synapse.metrics.event_processing_lag_by_event.labels(
  "appservice_sender"
  ).observe((now - ts) / 1000)
@@ -166,6 +168,7 @@ async def handle_room_events(events):
  if events:
  now = self.clock.time_msec()
  ts = await self.store.get_received_ts(events[-1].event_id)
+ assert ts is not None
 
  synapse.metrics.event_processing_lag.labels(
  "appservice_sender"

synapse/handlers/presence.py

@@ -28,6 +28,7 @@
 from contextlib import contextmanager
 from typing import (
  TYPE_CHECKING,
+ Any,
  Callable,
  Collection,
  Dict,
@@ -615,7 +616,7 @@ def __init__(self, hs: "HomeServer"):
  super().__init__(hs)
  self.hs = hs
  self.server_name = hs.hostname
- self.wheel_timer = WheelTimer()
+ self.wheel_timer: WheelTimer[str] = WheelTimer()
  self.notifier = hs.get_notifier()
  self._presence_enabled = hs.config.use_presence
 
@@ -924,7 +925,7 @@ async def bump_presence_active_time(self, user: UserID) -> None:
 
  prev_state = await self.current_state_for_user(user_id)
 
- new_fields = {"last_active_ts": self.clock.time_msec()}
+ new_fields: Dict[str, Any] = {"last_active_ts": self.clock.time_msec()}
  if prev_state.state == PresenceState.UNAVAILABLE:
  new_fields["state"] = PresenceState.ONLINE
 

synapse/handlers/typing.py

@@ -73,7 +73,7 @@ def __init__(self, hs: "HomeServer"):
  self._room_typing: Dict[str, Set[str]] = {}
 
  self._member_last_federation_poke: Dict[RoomMember, int] = {}
- self.wheel_timer = WheelTimer(bucket_size=5000)
+ self.wheel_timer: WheelTimer[RoomMember] = WheelTimer(bucket_size=5000)
  self._latest_room_serial = 0
 
  self.clock.looping_call(self._handle_timeouts, 5000)

synapse/rest/client/register.py

@@ -330,11 +330,11 @@ def __init__(self, hs: "HomeServer"):
  # Artificially delay requests if rate > sleep_limit/window_size
  sleep_limit=1,
  # Amount of artificial delay to apply
- sleep_msec=1000,
+ sleep_delay=1000,
  # Error with 429 if more than reject_limit requests are queued
  reject_limit=1,
  # Allow 1 request at a time
- concurrent_requests=1,
+ concurrent=1,
  ),
  )
 
@@ -763,7 +763,10 @@ async def _create_registration_details(
  Returns:
  dictionary for response from /register
  """
- result = {"user_id": user_id, "home_server": self.hs.hostname}
+ result: JsonDict = {
+ "user_id": user_id,
+ "home_server": self.hs.hostname,
+ }
  if not params.get("inhibit_login", False):
  device_id = params.get("device_id")
  initial_display_name = params.get("initial_device_display_name")
@@ -814,7 +817,7 @@ async def _do_guest_registration(
  user_id, device_id, initial_display_name, is_guest=True
  )
 
- result = {
+ result: JsonDict = {
  "user_id": user_id,
  "device_id": device_id,
  "access_token": access_token,

synapse/rest/synapse/client/new_user_consent.py

@@ -52,7 +52,7 @@ def template_search_dirs():
  yield hs.config.sso.sso_template_dir
  yield hs.config.sso.default_template_dir
 
- self._jinja_env = build_jinja_env(template_search_dirs(), hs.config)
+ self._jinja_env = build_jinja_env(list(template_search_dirs()), hs.config)
 
  async def _async_render_GET(self, request: Request) -> None:
  try:

synapse/rest/synapse/client/pick_username.py

@@ -80,7 +80,7 @@ def template_search_dirs():
  yield hs.config.sso.sso_template_dir
  yield hs.config.sso.default_template_dir
 
- self._jinja_env = build_jinja_env(template_search_dirs(), hs.config)
+ self._jinja_env = build_jinja_env(list(template_search_dirs()), hs.config)
 
  async def _async_render_GET(self, request: Request) -> None:
  try:

synapse/storage/databases/main/registration.py

@@ -1091,6 +1091,7 @@ def set_expiration_date_for_user_txn(self, txn, user_id, use_delta=False):
  delta equal to 10% of the validity period.
  """
  now_ms = self._clock.time_msec()
+ assert self._account_validity_period is not None
  expiration_ts = now_ms + self._account_validity_period
 
  if use_delta:

synapse/types.py

@@ -38,6 +38,7 @@
  IReactorCore,
  IReactorPluggableNameResolver,
  IReactorTCP,
+ IReactorThreads,
  IReactorTime,
 )
 
@@ -63,7 +64,12 @@
 # Note that this seems to require inheriting *directly* from Interface in order
 # for mypy-zope to realize it is an interface.
 class ISynapseReactor(
- IReactorTCP, IReactorPluggableNameResolver, IReactorTime, IReactorCore, Interface
+ IReactorTCP,
+ IReactorPluggableNameResolver,
+ IReactorTime,
+ IReactorCore,
+ IReactorThreads,
+ Interface,
 ):
  """The interfaces necessary for Synapse to function."""