From 9b334b3f97057ac145622d2e4d0ad036ef27b468 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Sun, 11 Mar 2018 20:01:41 +0000
Subject: [PATCH 001/830] WIP experiment in lazyloading room members

---
 synapse/handlers/sync.py | 43 ++++++++++++++++++++++++++----------
 synapse/storage/state.py | 47 +++++++++++++++++++++++++++++++++++-----
 2 files changed, 73 insertions(+), 17 deletions(-)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 0f713ce03855..809e9fece9ca 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -399,7 +399,7 @@ def _load_filtered_recents(self, room_id, sync_config, now_token,
         ))
 
     @defer.inlineCallbacks
-    def get_state_after_event(self, event):
+    def get_state_after_event(self, event, types=None):
         """
         Get the room state after the given event
 
@@ -409,14 +409,14 @@ def get_state_after_event(self, event):
         Returns:
             A Deferred map from ((type, state_key)->Event)
         """
-        state_ids = yield self.store.get_state_ids_for_event(event.event_id)
+        state_ids = yield self.store.get_state_ids_for_event(event.event_id, types)
         if event.is_state():
             state_ids = state_ids.copy()
             state_ids[(event.type, event.state_key)] = event.event_id
         defer.returnValue(state_ids)
 
     @defer.inlineCallbacks
-    def get_state_at(self, room_id, stream_position):
+    def get_state_at(self, room_id, stream_position, types=None):
         """ Get the room state at a particular stream position
 
         Args:
@@ -432,7 +432,7 @@ def get_state_at(self, room_id, stream_position):
 
         if last_events:
             last_event = last_events[-1]
-            state = yield self.get_state_after_event(last_event)
+            state = yield self.get_state_after_event(last_event, types)
 
         else:
             # no events in this room - so presumably no state
@@ -441,7 +441,7 @@ def get_state_at(self, room_id, stream_position):
 
     @defer.inlineCallbacks
     def compute_state_delta(self, room_id, batch, sync_config, since_token, now_token,
-                            full_state):
+                            full_state, filter_members):
         """ Works out the differnce in state between the start of the timeline
         and the previous sync.
 
@@ -454,6 +454,8 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                 be None.
             now_token(str): Token of the end of the current batch.
             full_state(bool): Whether to force returning the full state.
+            filter_members(bool): Whether to only return state for members
+                referenced in this timeline segment
 
         Returns:
              A deferred new event dictionary
@@ -464,18 +466,35 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
         # TODO(mjark) Check for new redactions in the state events.
 
         with Measure(self.clock, "compute_state_delta"):
+
+            types = None
+            if filter_members:
+                # We only request state for the members needed to display the
+                # timeline:
+                types = (
+                    (EventTypes.Member, state_key)
+                    for state_key in set(
+                        event.sender  # FIXME: we also care about targets etc.
+                        for event in batch.events
+                    )
+                )
+                types.append((None, None))  # don't just filter to room members
+
+                # TODO: we should opportunistically deduplicate these members too
+                # within the same sync series (based on an in-memory cache)
+
             if full_state:
                 if batch:
                     current_state_ids = yield self.store.get_state_ids_for_event(
-                        batch.events[-1].event_id
+                        batch.events[-1].event_id, types=types
                     )
 
                     state_ids = yield self.store.get_state_ids_for_event(
-                        batch.events[0].event_id
+                        batch.events[0].event_id, types=types
                     )
                 else:
                     current_state_ids = yield self.get_state_at(
-                        room_id, stream_position=now_token
+                        room_id, stream_position=now_token, types=types
                     )
 
                     state_ids = current_state_ids
@@ -493,15 +512,15 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                 )
             elif batch.limited:
                 state_at_previous_sync = yield self.get_state_at(
-                    room_id, stream_position=since_token
+                    room_id, stream_position=since_token, types=types
                 )
 
                 current_state_ids = yield self.store.get_state_ids_for_event(
-                    batch.events[-1].event_id
+                    batch.events[-1].event_id, types=types
                 )
 
                 state_at_timeline_start = yield self.store.get_state_ids_for_event(
-                    batch.events[0].event_id
+                    batch.events[0].event_id, types=types
                 )
 
                 timeline_state = {
@@ -1325,7 +1344,7 @@ def _generate_room_entry(self, sync_result_builder, ignored_users,
 
         state = yield self.compute_state_delta(
             room_id, batch, sync_config, since_token, now_token,
-            full_state=full_state
+            full_state=full_state, filter_members=True
         )
 
         if room_builder.rtype == "joined":
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index 2b325e1c1f81..da6bb685fa9e 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -198,8 +198,15 @@ def _get_state_groups_from_groups(self, groups, types):
 
     def _get_state_groups_from_groups_txn(self, txn, groups, types=None):
         results = {group: {} for group in groups}
+
+        include_other_types = False
+
         if types is not None:
-            types = list(set(types))  # deduplicate types list
+            type_set = set(types)
+            if (None, None) in type_set:
+                include_other_types = True
+                type_set.remove((None, None))
+            types = list(type_set)  # deduplicate types list
 
         if isinstance(self.database_engine, PostgresEngine):
             # Temporarily disable sequential scans in this transaction. This is
@@ -238,11 +245,21 @@ def _get_state_groups_from_groups_txn(self, txn, groups, types=None):
             if types:
                 clause_to_args = [
                     (
-                        "AND type = ? AND state_key = ?",
-                        (etype, state_key)
+                        "AND type = ? AND state_key = ?" if state_key is not None else "AND type = ?",
+                        (etype, state_key) if state_key is not None else (etype)
                     )
                     for etype, state_key in types
                 ]
+
+                if include_other_types:
+                    # XXX: check whether this slows postgres down like a list of
+                    # ORs does too?
+                    clause_to_args.append(
+                        (
+                            "AND type <> ? " * len(types),
+                            [t for (t, _) in types]
+                        )
+                    )
             else:
                 # If types is None we fetch all the state, and so just use an
                 # empty where clause with no extra args.
@@ -263,6 +280,10 @@ def _get_state_groups_from_groups_txn(self, txn, groups, types=None):
                 where_clause = "AND (%s)" % (
                     " OR ".join(["(type = ? AND state_key = ?)"] * len(types)),
                 )
+                if include_other_types:
+                    where_clause += " AND (%s)" % (
+                        " AND ".join(["type <> ?"] * len(types)),
+                    )
             else:
                 where_clause = ""
 
@@ -449,17 +470,27 @@ def _get_some_state_from_cache(self, group, types):
             group: The state group to lookup
             types (list): List of 2-tuples of the form (`type`, `state_key`),
                 where a `state_key` of `None` matches all state_keys for the
-                `type`.
+                `type`. Presence of type of `None` indicates that types not
+                in the list should not be filtered out.
         """
         is_all, known_absent, state_dict_ids = self._state_group_cache.get(group)
 
         type_to_key = {}
         missing_types = set()
 
+        include_other_types = False
+
         for typ, state_key in types:
             key = (typ, state_key)
+
+            if typ is None:
+                include_other_types = True
+                next
+
             if state_key is None:
                 type_to_key[typ] = None
+                # XXX: why do we mark the type as missing from our cache just
+                # because we weren't filtering on a specific value of state_key?
                 missing_types.add(key)
             else:
                 if type_to_key.get(typ, object()) is not None:
@@ -478,7 +509,7 @@ def include(typ, state_key):
                 return True
             if state_key in valid_state_keys:
                 return True
-            return False
+            return include_other_types
 
         got_all = is_all or not missing_types
 
@@ -507,6 +538,12 @@ def _get_state_for_groups(self, groups, types=None):
         with matching types. `types` is a list of `(type, state_key)`, where
         a `state_key` of None matches all state_keys. If `types` is None then
         all events are returned.
+
+        XXX: is it really true that `state_key` of None in `types` matches all
+        state_keys? it looks like _get-some_state_from_cache does the right thing,
+        but _get_state_groups_from_groups_txn treats ths None is turned into
+        'AND state_key = NULL' or similar (at least until i just fixed it) --Matthew
+        I've filed this as https://github.com/matrix-org/synapse/issues/2969
         """
         if types:
             types = frozenset(types)

From 87133652657c5073616419b0afc533eac6ae6750 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Sun, 11 Mar 2018 20:10:25 +0000
Subject: [PATCH 002/830] typos

---
 synapse/handlers/sync.py | 4 ++--
 synapse/storage/state.py | 2 ++
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 809e9fece9ca..fa730ca7604f 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -471,13 +471,13 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
             if filter_members:
                 # We only request state for the members needed to display the
                 # timeline:
-                types = (
+                types = [
                     (EventTypes.Member, state_key)
                     for state_key in set(
                         event.sender  # FIXME: we also care about targets etc.
                         for event in batch.events
                     )
-                )
+                ]
                 types.append((None, None))  # don't just filter to room members
 
                 # TODO: we should opportunistically deduplicate these members too
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index da6bb685fa9e..02382002864f 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -301,6 +301,8 @@ def _get_state_groups_from_groups_txn(self, txn, groups, types=None):
                     args = [next_group]
                     if types:
                         args.extend(i for typ in types for i in typ)
+                        if include_other_types:
+                            args.extend(typ for (typ, _) in types)
 
                     txn.execute(
                         "SELECT type, state_key, event_id FROM state_groups_state"

From 97c0496cfa89b037d89fccd05dd03442b80e07fc Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Mon, 12 Mar 2018 00:27:06 +0000
Subject: [PATCH 003/830] fix sqlite where clause

---
 synapse/storage/state.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index 02382002864f..b796d3c995d9 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -277,13 +277,14 @@ def _get_state_groups_from_groups_txn(self, txn, groups, types=None):
                         results[group][key] = event_id
         else:
             if types is not None:
-                where_clause = "AND (%s)" % (
+                where_clause = "AND (%s" % (
                     " OR ".join(["(type = ? AND state_key = ?)"] * len(types)),
                 )
                 if include_other_types:
-                    where_clause += " AND (%s)" % (
+                    where_clause += " OR (%s)" % (
                         " AND ".join(["type <> ?"] * len(types)),
                     )
+                where_clause += ")"
             else:
                 where_clause = ""
 

From fdedcd1f4ddeaa3ed5bfd3c05ab2977b4e8ed457 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Mon, 12 Mar 2018 01:39:06 +0000
Subject: [PATCH 004/830] correctly handle None state_keys

and fix include_other_types thinko
---
 synapse/storage/state.py | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index b796d3c995d9..405e6b6770f2 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -276,15 +276,23 @@ def _get_state_groups_from_groups_txn(self, txn, groups, types=None):
                         key = (typ, state_key)
                         results[group][key] = event_id
         else:
+            where_args = []
             if types is not None:
-                where_clause = "AND (%s" % (
-                    " OR ".join(["(type = ? AND state_key = ?)"] * len(types)),
-                )
+                where_clause = "AND ("
+                for typ in types:
+                    if typ[1] is None:
+                        where_clause += "(type = ?) OR "
+                        where_args.extend(typ[0])
+                    else:
+                        where_clause += "(type = ? AND state_key = ?) OR "
+                        where_args.extend([typ[0], typ[1]])
+
                 if include_other_types:
-                    where_clause += " OR (%s)" % (
+                    where_clause += "(%s) OR " % (
                         " AND ".join(["type <> ?"] * len(types)),
                     )
-                where_clause += ")"
+                    where_args.extend(t for (t, _) in types)
+                where_clause += "0)"  # 0 to terminate the last OR
             else:
                 where_clause = ""
 
@@ -301,9 +309,7 @@ def _get_state_groups_from_groups_txn(self, txn, groups, types=None):
                     # after we finish deduping state, which requires this func)
                     args = [next_group]
                     if types:
-                        args.extend(i for typ in types for i in typ)
-                        if include_other_types:
-                            args.extend(typ for (typ, _) in types)
+                        args.extend(where_args)
 
                     txn.execute(
                         "SELECT type, state_key, event_id FROM state_groups_state"
@@ -507,12 +513,12 @@ def _get_some_state_from_cache(self, group, types):
         def include(typ, state_key):
             valid_state_keys = type_to_key.get(typ, sentinel)
             if valid_state_keys is sentinel:
-                return False
+                return include_other_types
             if valid_state_keys is None:
                 return True
             if state_key in valid_state_keys:
                 return True
-            return include_other_types
+            return False
 
         got_all = is_all or not missing_types
 

From 1b1c13777154b5b0cf8bf8cf809381f889a2a82d Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Tue, 13 Mar 2018 17:52:52 +0000
Subject: [PATCH 005/830] fix bug #2926

---
 synapse/storage/state.py | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index 2b325e1c1f81..783cebb35184 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -240,6 +240,10 @@ def _get_state_groups_from_groups_txn(self, txn, groups, types=None):
                     (
                         "AND type = ? AND state_key = ?",
                         (etype, state_key)
+                    ) if state_key is not None else
+                    (
+                        "AND type = ?",
+                        (etype)
                     )
                     for etype, state_key in types
                 ]
@@ -259,10 +263,19 @@ def _get_state_groups_from_groups_txn(self, txn, groups, types=None):
                         key = (typ, state_key)
                         results[group][key] = event_id
         else:
+            where_args = []
             if types is not None:
-                where_clause = "AND (%s)" % (
-                    " OR ".join(["(type = ? AND state_key = ?)"] * len(types)),
-                )
+                where_clause = "AND ("
+                for typ in types:
+                    if typ[1] is None:
+                        where_clause += "(type = ?)"
+                        where_args.extend(typ[0])
+                    else:
+                        where_clause += "(type = ? AND state_key = ?)"
+                        where_args.extend([typ[0], typ[1]])
+                    if typ != types[-1]:
+                        where_clause += " OR "
+                where_clause += ")"
             else:
                 where_clause = ""
 
@@ -279,7 +292,7 @@ def _get_state_groups_from_groups_txn(self, txn, groups, types=None):
                     # after we finish deduping state, which requires this func)
                     args = [next_group]
                     if types:
-                        args.extend(i for typ in types for i in typ)
+                        args.extend(where_args)
 
                     txn.execute(
                         "SELECT type, state_key, event_id FROM state_groups_state"

From 52f7e23c7276b2848aa5291d8b78875b7c32a658 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Tue, 13 Mar 2018 18:07:55 +0000
Subject: [PATCH 006/830] PR feedbackz

---
 synapse/storage/state.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index 783cebb35184..77259a31415c 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -240,10 +240,9 @@ def _get_state_groups_from_groups_txn(self, txn, groups, types=None):
                     (
                         "AND type = ? AND state_key = ?",
                         (etype, state_key)
-                    ) if state_key is not None else
-                    (
+                    ) if state_key is not None else (
                         "AND type = ?",
-                        (etype)
+                        (etype,)
                     )
                     for etype, state_key in types
                 ]

From b2aba9e43053f9f297671fce0051bfc18a8b655a Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Tue, 13 Mar 2018 18:13:44 +0000
Subject: [PATCH 007/830] build where_clause sanely

---
 synapse/storage/state.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index 77259a31415c..82740266bfe4 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -263,18 +263,16 @@ def _get_state_groups_from_groups_txn(self, txn, groups, types=None):
                         results[group][key] = event_id
         else:
             where_args = []
+            where_clauses = []
             if types is not None:
-                where_clause = "AND ("
                 for typ in types:
                     if typ[1] is None:
-                        where_clause += "(type = ?)"
+                        where_clauses.append("(type = ?)")
                         where_args.extend(typ[0])
                     else:
-                        where_clause += "(type = ? AND state_key = ?)"
+                        where_clauses.append("(type = ? AND state_key = ?)")
                         where_args.extend([typ[0], typ[1]])
-                    if typ != types[-1]:
-                        where_clause += " OR "
-                where_clause += ")"
+                where_clause = "AND (%s)" % (" OR ".join(where_clauses))
             else:
                 where_clause = ""
 

From 865377a70d9d7db27b89348d2ebbd394f701c490 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Tue, 13 Mar 2018 19:45:36 +0000
Subject: [PATCH 008/830] disable optimisation for searching for state groups
 when type filter includes wildcards on state_key

---
 synapse/storage/state.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index 82740266bfe4..39f73afaa211 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -264,11 +264,13 @@ def _get_state_groups_from_groups_txn(self, txn, groups, types=None):
         else:
             where_args = []
             where_clauses = []
+            wildcard_types = False
             if types is not None:
                 for typ in types:
                     if typ[1] is None:
                         where_clauses.append("(type = ?)")
                         where_args.extend(typ[0])
+                        wildcard_types = True
                     else:
                         where_clauses.append("(type = ? AND state_key = ?)")
                         where_args.extend([typ[0], typ[1]])
@@ -302,9 +304,17 @@ def _get_state_groups_from_groups_txn(self, txn, groups, types=None):
                         if (typ, state_key) not in results[group]
                     )
 
-                    # If the lengths match then we must have all the types,
-                    # so no need to go walk further down the tree.
-                    if types is not None and len(results[group]) == len(types):
+                    # If the number of entries inthe (type,state_key)->event_id dict
+                    # matches the number of (type,state_keys) types we were searching
+                    # for, then we must have found them all, so no need to go walk
+                    # further down the tree... UNLESS our types filter contained
+                    # wildcards (i.e. Nones) in which case we have to do an exhaustive
+                    # search
+                    if (
+                        types is not None and
+                        not wildcard_types and
+                        len(results[group]) == len(types)
+                    ):
                         break
 
                     next_group = self._simple_select_one_onecol_txn(

From afbf4d3dccb3d18276e4b119b0267490ca522b4b Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Tue, 13 Mar 2018 19:48:04 +0000
Subject: [PATCH 009/830] typoe

---
 synapse/storage/state.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index 39f73afaa211..ffa424603188 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -304,7 +304,7 @@ def _get_state_groups_from_groups_txn(self, txn, groups, types=None):
                         if (typ, state_key) not in results[group]
                     )
 
-                    # If the number of entries inthe (type,state_key)->event_id dict
+                    # If the number of entries in the (type,state_key)->event_id dict
                     # matches the number of (type,state_keys) types we were searching
                     # for, then we must have found them all, so no need to go walk
                     # further down the tree... UNLESS our types filter contained

From 14a9d2f73d50225f190f42e270cbf9ef7447bd8c Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Tue, 13 Mar 2018 22:03:42 +0000
Subject: [PATCH 010/830] ensure we always include the members for a given
 timeline block

---
 synapse/handlers/sync.py | 36 ++++++++++++++++++++++++++++++++----
 1 file changed, 32 insertions(+), 4 deletions(-)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index fa730ca7604f..c754cfdeeb70 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -468,6 +468,8 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
         with Measure(self.clock, "compute_state_delta"):
 
             types = None
+            member_state_ids = {}
+
             if filter_members:
                 # We only request state for the members needed to display the
                 # timeline:
@@ -492,6 +494,13 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                     state_ids = yield self.store.get_state_ids_for_event(
                         batch.events[0].event_id, types=types
                     )
+
+                    if filter_members:
+                        member_state_ids = {
+                            t: state_ids[t]
+                            for t in state_ids if t[0] == EventTypes.member
+                        }
+
                 else:
                     current_state_ids = yield self.get_state_at(
                         room_id, stream_position=now_token, types=types
@@ -499,6 +508,12 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
 
                     state_ids = current_state_ids
 
+                    if filter_members:
+                        member_state_ids = {
+                            t: state_ids[t]
+                            for t in state_ids if t[0] == EventTypes.member
+                        }
+
                 timeline_state = {
                     (event.type, event.state_key): event.event_id
                     for event in batch.events if event.is_state()
@@ -507,6 +522,7 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                 state_ids = _calculate_state(
                     timeline_contains=timeline_state,
                     timeline_start=state_ids,
+                    timeline_start_members=member_state_ids,
                     previous={},
                     current=current_state_ids,
                 )
@@ -523,6 +539,12 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                     batch.events[0].event_id, types=types
                 )
 
+                if filter_members:
+                    member_state_ids = {
+                        t: state_at_timeline_start[t]
+                        for t in state_ids if t[0] == EventTypes.member
+                    }
+
                 timeline_state = {
                     (event.type, event.state_key): event.event_id
                     for event in batch.events if event.is_state()
@@ -531,6 +553,7 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                 state_ids = _calculate_state(
                     timeline_contains=timeline_state,
                     timeline_start=state_at_timeline_start,
+                    timeline_start_members=member_state_ids,
                     previous=state_at_previous_sync,
                     current=current_state_ids,
                 )
@@ -1440,12 +1463,16 @@ def _action_has_highlight(actions):
     return False
 
 
-def _calculate_state(timeline_contains, timeline_start, previous, current):
+def _calculate_state(timeline_contains, timeline_start, timeline_start_members,
+                     previous, current):
     """Works out what state to include in a sync response.
 
     Args:
         timeline_contains (dict): state in the timeline
         timeline_start (dict): state at the start of the timeline
+        timeline_start_members (dict): state at the start of the timeline
+            for room members who participate in this chunk of timeline.
+            Should always be a subset of timeline_start.
         previous (dict): state at the end of the previous sync (or empty dict
             if this is an initial sync)
         current (dict): state at the end of the timeline
@@ -1464,11 +1491,12 @@ def _calculate_state(timeline_contains, timeline_start, previous, current):
     }
 
     c_ids = set(e for e in current.values())
-    tc_ids = set(e for e in timeline_contains.values())
-    p_ids = set(e for e in previous.values())
     ts_ids = set(e for e in timeline_start.values())
+    tsm_ids = set(e for e in timeline_start_members.values())
+    p_ids = set(e for e in previous.values())
+    tc_ids = set(e for e in timeline_contains.values())
 
-    state_ids = ((c_ids | ts_ids) - p_ids) - tc_ids
+    state_ids = (((c_ids | ts_ids) - p_ids) - tc_ids) | tsm_ids
 
     return {
         event_id_to_key[e]: e for e in state_ids

From f0f9a0605b1bddc1b01d1bbb6af93f00763b8496 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Tue, 13 Mar 2018 22:12:15 +0000
Subject: [PATCH 011/830] remove comment now #2969 is fixed

---
 synapse/storage/state.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index 4ab16e18b267..4291cde7abe7 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -561,12 +561,6 @@ def _get_state_for_groups(self, groups, types=None):
         with matching types. `types` is a list of `(type, state_key)`, where
         a `state_key` of None matches all state_keys. If `types` is None then
         all events are returned.
-
-        XXX: is it really true that `state_key` of None in `types` matches all
-        state_keys? it looks like _get-some_state_from_cache does the right thing,
-        but _get_state_groups_from_groups_txn treats ths None is turned into
-        'AND state_key = NULL' or similar (at least until i just fixed it) --Matthew
-        I've filed this as https://github.com/matrix-org/synapse/issues/2969
         """
         if types:
             types = frozenset(types)

From ccca02846d07124f537b0c475308f9a26bfb3fb1 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Tue, 13 Mar 2018 22:31:41 +0000
Subject: [PATCH 012/830] make it work

---
 synapse/handlers/sync.py |  6 +++---
 synapse/storage/state.py | 10 ++++++----
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index c754cfdeeb70..c05e3d107fae 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -498,7 +498,7 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                     if filter_members:
                         member_state_ids = {
                             t: state_ids[t]
-                            for t in state_ids if t[0] == EventTypes.member
+                            for t in state_ids if t[0] == EventTypes.Member
                         }
 
                 else:
@@ -511,7 +511,7 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                     if filter_members:
                         member_state_ids = {
                             t: state_ids[t]
-                            for t in state_ids if t[0] == EventTypes.member
+                            for t in state_ids if t[0] == EventTypes.Member
                         }
 
                 timeline_state = {
@@ -542,7 +542,7 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                 if filter_members:
                     member_state_ids = {
                         t: state_at_timeline_start[t]
-                        for t in state_ids if t[0] == EventTypes.member
+                        for t in state_ids if t[0] == EventTypes.Member
                     }
 
                 timeline_state = {
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index 4291cde7abe7..9c9994c07362 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -257,10 +257,11 @@ def _get_state_groups_from_groups_txn(self, txn, groups, types=None):
                 if include_other_types:
                     # XXX: check whether this slows postgres down like a list of
                     # ORs does too?
+                    unique_types = set([ t for (t, _) in types ])
                     clause_to_args.append(
                         (
-                            "AND type <> ? " * len(types),
-                            [t for (t, _) in types]
+                            "AND type <> ? " * len(unique_types),
+                            list(unique_types)
                         )
                     )
             else:
@@ -293,10 +294,11 @@ def _get_state_groups_from_groups_txn(self, txn, groups, types=None):
                         where_args.extend([typ[0], typ[1]])
 
                 if include_other_types:
+                    unique_types = set([ t for (t, _) in types ])                    
                     where_clauses.append(
-                        "(" + " AND ".join(["type <> ?"] * len(types)) + ")"
+                        "(" + " AND ".join(["type <> ?"] * len(unique_types)) + ")"
                     )
-                    where_args.extend(t for (t, _) in types)
+                    where_args.extend(list(unique_types))
 
                 where_clause = "AND (%s)" % (" OR ".join(where_clauses))
             else:

From c9d72e4571752554dfe01d755ae23f55c5f84ade Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Tue, 13 Mar 2018 23:46:45 +0000
Subject: [PATCH 013/830] oops

---
 synapse/handlers/sync.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index c05e3d107fae..887624c431c2 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -542,7 +542,7 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                 if filter_members:
                     member_state_ids = {
                         t: state_at_timeline_start[t]
-                        for t in state_ids if t[0] == EventTypes.Member
+                        for t in state_at_timeline_start if t[0] == EventTypes.Member
                     }
 
                 timeline_state = {

From 4d0cfef6ee023bfe83113a0378321830ebde1619 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Wed, 14 Mar 2018 00:02:20 +0000
Subject: [PATCH 014/830] add copyright to nudge CI

---
 synapse/handlers/sync.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 887624c431c2..edbd2ae7710f 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
-# Copyright 2015 - 2016 OpenMarket Ltd
+# Copyright 2015, 2016 OpenMarket Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

From 9f77001e2747c36046e136c5d3c706c0aef54b15 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Wed, 14 Mar 2018 00:07:47 +0000
Subject: [PATCH 015/830] pep8

---
 synapse/storage/state.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index 9c9994c07362..55159e64d0b8 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -257,7 +257,7 @@ def _get_state_groups_from_groups_txn(self, txn, groups, types=None):
                 if include_other_types:
                     # XXX: check whether this slows postgres down like a list of
                     # ORs does too?
-                    unique_types = set([ t for (t, _) in types ])
+                    unique_types = set([t for (t, _) in types])
                     clause_to_args.append(
                         (
                             "AND type <> ? " * len(unique_types),
@@ -294,7 +294,7 @@ def _get_state_groups_from_groups_txn(self, txn, groups, types=None):
                         where_args.extend([typ[0], typ[1]])
 
                 if include_other_types:
-                    unique_types = set([ t for (t, _) in types ])                    
+                    unique_types = set([t for (t, _) in types])
                     where_clauses.append(
                         "(" + " AND ".join(["type <> ?"] * len(unique_types)) + ")"
                     )

From 3bc5bd2d22e6b53ec1f89760301df1517e71b53a Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Fri, 16 Mar 2018 00:52:04 +0000
Subject: [PATCH 016/830] make incr syncs work

---
 synapse/handlers/sync.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index edbd2ae7710f..84c894ca4a39 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -499,7 +499,7 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                     if filter_members:
                         member_state_ids = {
                             t: state_ids[t]
-                            for t in state_ids if t[0] == EventTypes.Member
+                            for t in state_ids if state_ids[t][0] == EventTypes.Member
                         }
 
                 else:
@@ -512,7 +512,7 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                     if filter_members:
                         member_state_ids = {
                             t: state_ids[t]
-                            for t in state_ids if t[0] == EventTypes.Member
+                            for t in state_ids if state_ids[t][0] == EventTypes.Member
                         }
 
                 timeline_state = {
@@ -543,7 +543,8 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                 if filter_members:
                     member_state_ids = {
                         t: state_at_timeline_start[t]
-                        for t in state_at_timeline_start if t[0] == EventTypes.Member
+                        for t in state_at_timeline_start
+                        if state_at_timeline_start[t][0] == EventTypes.Member
                     }
 
                 timeline_state = {

From bf49d2dca8db6d82f09441a35cd3655c746b6b4d Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 15 Mar 2018 23:38:43 +0000
Subject: [PATCH 017/830] Replace some ujson with simplejson to make it work

---
 synapse/http/server.py               | 3 ++-
 synapse/rest/client/v2_alpha/sync.py | 2 +-
 synapse/storage/events.py            | 2 +-
 synapse/storage/events_worker.py     | 2 +-
 4 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/synapse/http/server.py b/synapse/http/server.py
index 4b567215c812..3c7a0ef97aa3 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -38,6 +38,7 @@
 import logging
 import urllib
 import ujson
+import simplejson
 
 logger = logging.getLogger(__name__)
 
@@ -462,7 +463,7 @@ def respond_with_json(request, code, json_object, send_cors=False,
             json_bytes = encode_canonical_json(json_object)
         else:
             # ujson doesn't like frozen_dicts.
-            json_bytes = ujson.dumps(json_object, ensure_ascii=False)
+            json_bytes = simplejson.dumps(json_object)
 
     return respond_with_json_bytes(
         request, code, json_bytes,
diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py
index a0a8e4b8e46c..eb91c0b2934c 100644
--- a/synapse/rest/client/v2_alpha/sync.py
+++ b/synapse/rest/client/v2_alpha/sync.py
@@ -33,7 +33,7 @@
 import itertools
 import logging
 
-import ujson as json
+import simplejson as json
 
 logger = logging.getLogger(__name__)
 
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 38908781700b..9fc65229fd80 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -38,7 +38,7 @@
 import synapse.metrics
 
 import logging
-import ujson as json
+import simplejson as json
 
 # these are only included to make the type annotations work
 from synapse.events import EventBase    # noqa: F401
diff --git a/synapse/storage/events_worker.py b/synapse/storage/events_worker.py
index 86c3b48ad40a..2e23dd78ba98 100644
--- a/synapse/storage/events_worker.py
+++ b/synapse/storage/events_worker.py
@@ -28,7 +28,7 @@
 from collections import namedtuple
 
 import logging
-import ujson as json
+import simplejson as json
 
 # these are only included to make the type annotations work
 from synapse.events import EventBase    # noqa: F401

From 5b3b3aada8952b53f82723227c9758ed47450a2e Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Fri, 16 Mar 2018 01:17:34 +0000
Subject: [PATCH 018/830] simplify timeline_start_members

---
 synapse/handlers/sync.py | 27 +++++++++++++--------------
 1 file changed, 13 insertions(+), 14 deletions(-)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 84c894ca4a39..ffb4f7915e24 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -496,12 +496,6 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                         batch.events[0].event_id, types=types
                     )
 
-                    if filter_members:
-                        member_state_ids = {
-                            t: state_ids[t]
-                            for t in state_ids if state_ids[t][0] == EventTypes.Member
-                        }
-
                 else:
                     current_state_ids = yield self.get_state_at(
                         room_id, stream_position=now_token, types=types
@@ -509,11 +503,13 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
 
                     state_ids = current_state_ids
 
-                    if filter_members:
-                        member_state_ids = {
-                            t: state_ids[t]
-                            for t in state_ids if state_ids[t][0] == EventTypes.Member
-                        }
+                if filter_members:
+                    logger.info("Finding members from %r", state_ids)
+                    member_state_ids = {
+                        e: state_ids[e]
+                        for e in state_ids if state_ids[e][0] == EventTypes.Member
+                    }
+                    logger.info("Found members %r", member_state_ids)
 
                 timeline_state = {
                     (event.type, event.state_key): event.event_id
@@ -541,11 +537,14 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                 )
 
                 if filter_members:
+                    logger.info("Finding members from %r", state_at_timeline_start)
                     member_state_ids = {
-                        t: state_at_timeline_start[t]
-                        for t in state_at_timeline_start
-                        if state_at_timeline_start[t][0] == EventTypes.Member
+                        e: state_at_timeline_start[e]
+                        for e in state_at_timeline_start
+                        if state_at_timeline_start[e][0] == EventTypes.Member
                     }
+                    logger.info("Found members %r", member_state_ids)
+
 
                 timeline_state = {
                     (event.type, event.state_key): event.event_id

From f7dcc404f216383bfd62e4611c6a28c3f13576dc Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Fri, 16 Mar 2018 01:37:53 +0000
Subject: [PATCH 019/830] add state_ids for timeline entries

---
 synapse/handlers/sync.py | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index ffb4f7915e24..9b7e598e74bb 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -474,6 +474,7 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
             if filter_members:
                 # We only request state for the members needed to display the
                 # timeline:
+
                 types = [
                     (EventTypes.Member, state_key)
                     for state_key in set(
@@ -481,11 +482,14 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                         for event in batch.events
                     )
                 ]
-                types.append((None, None))  # don't just filter to room members
 
-                # TODO: we should opportunistically deduplicate these members too
+                # TODO: we should opportunistically deduplicate these members here
                 # within the same sync series (based on an in-memory cache)
 
+                if not types:
+                    filter_members = False
+                types.append((None, None))  # don't just filter to room members
+
             if full_state:
                 if batch:
                     current_state_ids = yield self.store.get_state_ids_for_event(
@@ -545,7 +549,6 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                     }
                     logger.info("Found members %r", member_state_ids)
 
-
                 timeline_state = {
                     (event.type, event.state_key): event.event_id
                     for event in batch.events if event.is_state()
@@ -559,7 +562,14 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                     current=current_state_ids,
                 )
             else:
-                state_ids = {}
+                if filter_members:
+                    # strip off the (None, None) and filter to just room members
+                    types = types[:-1]
+                    state_ids = yield self.store.get_state_ids_for_event(
+                        batch.events[0].event_id, types=types
+                    )
+                else:
+                    state_ids = {}
 
         state = {}
         if state_ids:

From 4f0493c850d4611e8ada42c1de54a18e8dc15a37 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Fri, 16 Mar 2018 01:43:37 +0000
Subject: [PATCH 020/830] fix tsm search again

---
 synapse/handlers/sync.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 9b7e598e74bb..4bf85a128fc7 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -510,8 +510,8 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                 if filter_members:
                     logger.info("Finding members from %r", state_ids)
                     member_state_ids = {
-                        e: state_ids[e]
-                        for e in state_ids if state_ids[e][0] == EventTypes.Member
+                        t: state_ids[t]
+                        for t in state_ids if t[0] == EventTypes.Member
                     }
                     logger.info("Found members %r", member_state_ids)
 
@@ -543,9 +543,8 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                 if filter_members:
                     logger.info("Finding members from %r", state_at_timeline_start)
                     member_state_ids = {
-                        e: state_at_timeline_start[e]
-                        for e in state_at_timeline_start
-                        if state_at_timeline_start[e][0] == EventTypes.Member
+                        t: state_at_timeline_start[t]
+                        for t in state_at_timeline_start if t[0] == EventTypes.Member
                     }
                     logger.info("Found members %r", member_state_ids)
 

From fc5397fdf5acefd33bd3b808b6d8cc7c31b69b55 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Fri, 16 Mar 2018 01:44:55 +0000
Subject: [PATCH 021/830] remove debug

---
 synapse/handlers/sync.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 4bf85a128fc7..b7f42bd59430 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -508,12 +508,10 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                     state_ids = current_state_ids
 
                 if filter_members:
-                    logger.info("Finding members from %r", state_ids)
                     member_state_ids = {
                         t: state_ids[t]
                         for t in state_ids if t[0] == EventTypes.Member
                     }
-                    logger.info("Found members %r", member_state_ids)
 
                 timeline_state = {
                     (event.type, event.state_key): event.event_id
@@ -541,12 +539,10 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                 )
 
                 if filter_members:
-                    logger.info("Finding members from %r", state_at_timeline_start)
                     member_state_ids = {
                         t: state_at_timeline_start[t]
                         for t in state_at_timeline_start if t[0] == EventTypes.Member
                     }
-                    logger.info("Found members %r", member_state_ids)
 
                 timeline_state = {
                     (event.type, event.state_key): event.event_id

From 0b56290f0b14025939515d01aa72f512f0c629dc Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Fri, 16 Mar 2018 01:45:49 +0000
Subject: [PATCH 022/830] remove stale import

---
 synapse/http/server.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/synapse/http/server.py b/synapse/http/server.py
index 3c7a0ef97aa3..e64aa927299c 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -37,7 +37,6 @@
 import collections
 import logging
 import urllib
-import ujson
 import simplejson
 
 logger = logging.getLogger(__name__)

From 366f730bf697fe8fbb18a509ec1852987bc80410 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Sun, 18 Mar 2018 21:40:35 +0000
Subject: [PATCH 023/830] only get member state IDs for incremental syncs if
 we're filtering

---
 synapse/handlers/sync.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index b7f42bd59430..6b57afd97b43 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -557,14 +557,14 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                     current=current_state_ids,
                 )
             else:
+                state_ids = {}
                 if filter_members:
                     # strip off the (None, None) and filter to just room members
                     types = types[:-1]
-                    state_ids = yield self.store.get_state_ids_for_event(
-                        batch.events[0].event_id, types=types
-                    )
-                else:
-                    state_ids = {}
+                    if types:
+                        state_ids = yield self.store.get_state_ids_for_event(
+                            batch.events[0].event_id, types=types
+                        )
 
         state = {}
         if state_ids:

From 478af0f72005708dbbed23e30c547c3d66c07c0e Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Mon, 19 Mar 2018 01:00:12 +0000
Subject: [PATCH 024/830] reshuffle todo & comments

---
 synapse/handlers/sync.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 6b57afd97b43..76f5057377e4 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -483,11 +483,15 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                     )
                 ]
 
-                # TODO: we should opportunistically deduplicate these members here
-                # within the same sync series (based on an in-memory cache)
+                # We can't remove redundant member types at this stage as it has
+                # to be done based on event_id, and we don't have the member
+                # event ids until we've pulled them out of the DB.
 
                 if not types:
+                    # an optimisation to stop needlessly trying to calculate
+                    # member_state_ids
                     filter_members = False
+
                 types.append((None, None))  # don't just filter to room members
 
             if full_state:
@@ -559,6 +563,10 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
             else:
                 state_ids = {}
                 if filter_members:
+                    # TODO: filter out redundant members based on their mxids (not their
+                    # event_ids) at this point. We know we can do it based on mxid as this
+                    # is an non-gappy incremental sync.
+
                     # strip off the (None, None) and filter to just room members
                     types = types[:-1]
                     if types:

From b2f22829475ccfe19e994aedddb8d04995018bf4 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Mon, 19 Mar 2018 01:15:13 +0000
Subject: [PATCH 025/830] make lazy_load_members configurable in filters

---
 synapse/api/filtering.py |  6 ++++++
 synapse/handlers/sync.py | 18 +++++++++---------
 2 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py
index 83206348e549..339e4a31d621 100644
--- a/synapse/api/filtering.py
+++ b/synapse/api/filtering.py
@@ -260,6 +260,9 @@ def presence_limit(self):
     def ephemeral_limit(self):
         return self._room_ephemeral_filter.limit()
 
+    def lazy_load_members(self):
+        return self._room_state_filter.lazy_load_members()
+
     def filter_presence(self, events):
         return self._presence_filter.filter(events)
 
@@ -416,6 +419,9 @@ def filter(self, events):
     def limit(self):
         return self.filter_json.get("limit", 10)
 
+    def lazy_load_members(self):
+        return self.filter_json.get("lazy_load_members", False)
+
 
 def _matches_wildcard(actual_value, filter_value):
     if filter_value.endswith("*"):
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 76f5057377e4..f521d22e91d4 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -442,7 +442,7 @@ def get_state_at(self, room_id, stream_position, types=None):
 
     @defer.inlineCallbacks
     def compute_state_delta(self, room_id, batch, sync_config, since_token, now_token,
-                            full_state, filter_members):
+                            full_state):
         """ Works out the differnce in state between the start of the timeline
         and the previous sync.
 
@@ -455,7 +455,7 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                 be None.
             now_token(str): Token of the end of the current batch.
             full_state(bool): Whether to force returning the full state.
-            filter_members(bool): Whether to only return state for members
+            lazy_load_members(bool): Whether to only return state for members
                 referenced in this timeline segment
 
         Returns:
@@ -470,8 +470,9 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
 
             types = None
             member_state_ids = {}
+            lazy_load_members = sync_config.filter_collection.lazy_load_members()
 
-            if filter_members:
+            if lazy_load_members:
                 # We only request state for the members needed to display the
                 # timeline:
 
@@ -490,7 +491,7 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                 if not types:
                     # an optimisation to stop needlessly trying to calculate
                     # member_state_ids
-                    filter_members = False
+                    lazy_load_members = False
 
                 types.append((None, None))  # don't just filter to room members
 
@@ -511,7 +512,7 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
 
                     state_ids = current_state_ids
 
-                if filter_members:
+                if lazy_load_members:
                     member_state_ids = {
                         t: state_ids[t]
                         for t in state_ids if t[0] == EventTypes.Member
@@ -542,7 +543,7 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                     batch.events[0].event_id, types=types
                 )
 
-                if filter_members:
+                if lazy_load_members:
                     member_state_ids = {
                         t: state_at_timeline_start[t]
                         for t in state_at_timeline_start if t[0] == EventTypes.Member
@@ -562,7 +563,7 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                 )
             else:
                 state_ids = {}
-                if filter_members:
+                if lazy_load_members:
                     # TODO: filter out redundant members based on their mxids (not their
                     # event_ids) at this point. We know we can do it based on mxid as this
                     # is an non-gappy incremental sync.
@@ -1380,8 +1381,7 @@ def _generate_room_entry(self, sync_result_builder, ignored_users,
             return
 
         state = yield self.compute_state_delta(
-            room_id, batch, sync_config, since_token, now_token,
-            full_state=full_state, filter_members=True
+            room_id, batch, sync_config, since_token, now_token, full_state=full_state
         )
 
         if room_builder.rtype == "joined":

From a6c8f7c875348ff8d63a7032c2f73a08551c516c Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Tue, 29 May 2018 01:09:55 +0100
Subject: [PATCH 026/830] add pydoc

---
 synapse/handlers/sync.py | 18 +++++++---
 synapse/storage/state.py | 76 ++++++++++++++++++++++++++++------------
 2 files changed, 67 insertions(+), 27 deletions(-)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 05bf6d46dd5f..8e380783322c 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -423,7 +423,11 @@ def get_state_after_event(self, event, types=None):
 
         Args:
             event(synapse.events.EventBase): event of interest
-
+            types(list[(str|None, str|None)]|None): List of (type, state_key) tuples
+                which are used to filter the state fetched. If `state_key` is None,
+                all events are returned of the given type.  Presence of type of `None`
+                indicates that types not in the list should not be filtered out.
+                May be None, which matches any key.
         Returns:
             A Deferred map from ((type, state_key)->Event)
         """
@@ -440,6 +444,11 @@ def get_state_at(self, room_id, stream_position, types=None):
         Args:
             room_id(str): room for which to get state
             stream_position(StreamToken): point at which to get state
+            types(list[(str|None, str|None)]|None): List of (type, state_key) tuples
+                which are used to filter the state fetched. If `state_key` is None,
+                all events are returned of the given type.  Presence of type of `None`
+                indicates that types not in the list should not be filtered out.
+                May be None, which matches any key.
 
         Returns:
             A Deferred map from ((type, state_key)->Event)
@@ -472,8 +481,6 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                 be None.
             now_token(str): Token of the end of the current batch.
             full_state(bool): Whether to force returning the full state.
-            lazy_load_members(bool): Whether to only return state for members
-                referenced in this timeline segment
 
         Returns:
              A deferred new event dictionary
@@ -496,7 +503,7 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                 types = [
                     (EventTypes.Member, state_key)
                     for state_key in set(
-                        event.sender  # FIXME: we also care about targets etc.
+                        event.sender  # FIXME: we also care about invite targets etc.
                         for event in batch.events
                     )
                 ]
@@ -1398,7 +1405,8 @@ def _generate_room_entry(self, sync_result_builder, ignored_users,
             return
 
         state = yield self.compute_state_delta(
-            room_id, batch, sync_config, since_token, now_token, full_state=full_state
+            room_id, batch, sync_config, since_token, now_token,
+            full_state=full_state
         )
 
         if room_builder.rtype == "joined":
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index 55159e64d0b8..63b683420215 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -182,7 +182,19 @@ def get_state_groups(self, room_id, event_ids):
 
     @defer.inlineCallbacks
     def _get_state_groups_from_groups(self, groups, types):
-        """Returns dictionary state_group -> (dict of (type, state_key) -> event id)
+        """Returns the state groups for a given set of groups, filtering on
+        types of state events.
+
+        Args:
+            groups(list[int]): list of state group IDs to query
+            types(list[str|None, str|None])|None: List of 2-tuples of the form
+                (`type`, `state_key`), where a `state_key` of `None` matches all
+                state_keys for the `type`. Presence of type of `None` indicates
+                that types not in the list should not be filtered out. If None,
+                all types are returned.
+
+        Returns:
+            dictionary state_group -> (dict of (type, state_key) -> event id)
         """
         results = {}
 
@@ -204,6 +216,9 @@ def _get_state_groups_from_groups_txn(self, txn, groups, types=None):
         if types is not None:
             type_set = set(types)
             if (None, None) in type_set:
+                # special case (None, None) to mean that other types should be
+                # returned - i.e. we were just filtering down the state keys
+                # for particular types.
                 include_other_types = True
                 type_set.remove((None, None))
             types = list(type_set)  # deduplicate types list
@@ -360,10 +375,12 @@ def get_state_for_events(self, event_ids, types):
         that are in the `types` list.
 
         Args:
-            event_ids (list)
-            types (list): List of (type, state_key) tuples which are used to
-                filter the state fetched. `state_key` may be None, which matches
-                any `state_key`
+            event_ids (list[string])
+            types (list[(str|None, str|None)]|None): List of (type, state_key) tuples
+                which are used to filter the state fetched. If `state_key` is None,
+                all events are returned of the given type.  Presence of type of `None`
+                indicates that types not in the list should not be filtered out.
+                May be None, which matches any key.
 
         Returns:
             deferred: A list of dicts corresponding to the event_ids given.
@@ -399,9 +416,11 @@ def get_state_ids_for_events(self, event_ids, types=None):
 
         Args:
             event_ids(list(str)): events whose state should be returned
-            types(list[(str, str)]|None): List of (type, state_key) tuples
-                which are used to filter the state fetched. May be None, which
-                matches any key
+            types(list[(str|None, str|None)]|None): List of (type, state_key) tuples
+                which are used to filter the state fetched. If `state_key` is None,
+                all events are returned of the given type.  Presence of type of `None`
+                indicates that types not in the list should not be filtered out.
+                May be None, which matches any key.
 
         Returns:
             A deferred dict from event_id -> (type, state_key) -> state_event
@@ -427,9 +446,11 @@ def get_state_for_event(self, event_id, types=None):
 
         Args:
             event_id(str): event whose state should be returned
-            types(list[(str, str)]|None): List of (type, state_key) tuples
-                which are used to filter the state fetched. May be None, which
-                matches any key
+            types(list[(str|None, str|None)]|None): List of (type, state_key) tuples
+                which are used to filter the state fetched. If `state_key` is None,
+                all events are returned of the given type.  Presence of type of `None`
+                indicates that types not in the list should not be filtered out.
+                May be None, which matches any key.
 
         Returns:
             A deferred dict from (type, state_key) -> state_event
@@ -444,9 +465,11 @@ def get_state_ids_for_event(self, event_id, types=None):
 
         Args:
             event_id(str): event whose state should be returned
-            types(list[(str, str)]|None): List of (type, state_key) tuples
-                which are used to filter the state fetched. May be None, which
-                matches any key
+            types(list[(str|None, str|None)]|None): List of (type, state_key) tuples
+                which are used to filter the state fetched. If `state_key` is None,
+                all events are returned of the given type.  Presence of type of `None`
+                indicates that types not in the list should not be filtered out.
+                May be None, which matches any key.
 
         Returns:
             A deferred dict from (type, state_key) -> state_event
@@ -492,11 +515,11 @@ def _get_some_state_from_cache(self, group, types):
         missing state.
 
         Args:
-            group: The state group to lookup
-            types (list): List of 2-tuples of the form (`type`, `state_key`),
-                where a `state_key` of `None` matches all state_keys for the
-                `type`. Presence of type of `None` indicates that types not
-                in the list should not be filtered out.
+            group(int): The state group to lookup
+            types(list[str|None, str|None]): List of 2-tuples of the form
+                (`type`, `state_key`), where a `state_key` of `None` matches all
+                state_keys for the `type`. Presence of type of `None` indicates
+                that types not in the list should not be filtered out.
         """
         is_all, known_absent, state_dict_ids = self._state_group_cache.get(group)
 
@@ -560,9 +583,18 @@ def _get_all_state_from_cache(self, group):
     @defer.inlineCallbacks
     def _get_state_for_groups(self, groups, types=None):
         """Given list of groups returns dict of group -> list of state events
-        with matching types. `types` is a list of `(type, state_key)`, where
-        a `state_key` of None matches all state_keys. If `types` is None then
-        all events are returned.
+        with matching types.
+
+        Args:
+            groups(list[int]): list of groups whose state to query
+            types(list[str|None, str|None]|None): List of 2-tuples of the form
+                (`type`, `state_key`), where a `state_key` of `None` matches all
+                state_keys for the `type`. Presence of type of `None` indicates
+                that types not in the list should not be filtered out. If None,
+                all events are returned.
+
+        Returns:
+            dict of group -> list of state events
         """
         if types:
             types = frozenset(types)

From 5e6b31f0da8ba0806de856e4a9823206ac4434f9 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Tue, 29 May 2018 02:21:52 +0100
Subject: [PATCH 027/830] fix dumb typo

---
 tests/test_dns.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/test_dns.py b/tests/test_dns.py
index af607d626f31..3b360a0fc70c 100644
--- a/tests/test_dns.py
+++ b/tests/test_dns.py
@@ -62,7 +62,7 @@ def test_from_cache_expired_and_dns_fail(self):
         dns_client_mock = Mock()
         dns_client_mock.lookupService.return_value = defer.fail(error.DNSServerError())
 
-        service_name = "test_service.examle.com"
+        service_name = "test_service.example.com"
 
         entry = Mock(spec_set=["expires"])
         entry.expires = 0
@@ -87,7 +87,7 @@ def test_from_cache(self):
         dns_client_mock = Mock(spec_set=['lookupService'])
         dns_client_mock.lookupService = Mock(spec_set=[])
 
-        service_name = "test_service.examle.com"
+        service_name = "test_service.example.com"
 
         entry = Mock(spec_set=["expires"])
         entry.expires = 999999999
@@ -111,7 +111,7 @@ def test_empty_cache(self):
 
         dns_client_mock.lookupService.return_value = defer.fail(error.DNSServerError())
 
-        service_name = "test_service.examle.com"
+        service_name = "test_service.example.com"
 
         cache = {}
 
@@ -126,7 +126,7 @@ def test_name_error(self):
 
         dns_client_mock.lookupService.return_value = defer.fail(error.DNSNameError())
 
-        service_name = "test_service.examle.com"
+        service_name = "test_service.example.com"
 
         cache = {}
 

From b69ff33d9e26e9efe89c745e0fda4801db8bede0 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Tue, 29 May 2018 02:22:27 +0100
Subject: [PATCH 028/830] disable CPUMetrics if no /proc/self/stat

fixes build on macOS again
---
 synapse/metrics/__init__.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index 371c300cac7d..6b5b0c94712e 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -34,6 +34,7 @@
 all_collectors = []
 all_gauges = {}
 
+HAVE_PROC_SELF_STAT = os.path.exists("/proc/self/stat")
 
 class RegistryProxy(object):
 
@@ -99,6 +100,8 @@ def __init__(self):
         self.ticks_per_sec = ticks_per_sec
 
     def collect(self):
+        if not HAVE_PROC_SELF_STAT:
+            return
 
         with open("/proc/self/stat") as s:
             line = s.read()

From 8df7bad839f04cbcabf066fe549df14879639ef5 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Tue, 29 May 2018 02:32:15 +0100
Subject: [PATCH 029/830] pep8

---
 synapse/metrics/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index 6b5b0c94712e..bfdbbc9a2389 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -36,6 +36,7 @@
 
 HAVE_PROC_SELF_STAT = os.path.exists("/proc/self/stat")
 
+
 class RegistryProxy(object):
 
     def collect(self):

From 9bbb9f5556496529478753d2123526ca6894535c Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Tue, 29 May 2018 04:26:10 +0100
Subject: [PATCH 030/830] add lazy_load_members to the filter json schema

---
 synapse/api/filtering.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py
index 7158dd75e9d7..fd5896186257 100644
--- a/synapse/api/filtering.py
+++ b/synapse/api/filtering.py
@@ -112,7 +112,10 @@
         },
         "contains_url": {
             "type": "boolean"
-        }
+        },
+        "lazy_load_members": {
+            "type": "boolean"
+        },
     }
 }
 

From 5f6122fe102f994e023d530cb6076730f31f619f Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Mon, 4 Jun 2018 00:08:52 +0300
Subject: [PATCH 031/830] more comments

---
 synapse/handlers/sync.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 8e380783322c..7ab97b24a658 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -515,6 +515,9 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                 if not types:
                     # an optimisation to stop needlessly trying to calculate
                     # member_state_ids
+                    #
+                    # XXX: i can't remember what this trying to do. why would
+                    # types ever be []? --matthew
                     lazy_load_members = False
 
                 types.append((None, None))  # don't just filter to room members
@@ -568,6 +571,10 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                 )
 
                 if lazy_load_members:
+                    # TODO: filter out redundant members based on their event_ids
+                    # (not mxids) at this point. In practice, limited syncs are
+                    # relatively rare so it's not a total disaster to send redundant
+                    # members down at this point.
                     member_state_ids = {
                         t: state_at_timeline_start[t]
                         for t in state_at_timeline_start if t[0] == EventTypes.Member

From 8503dd0047119caa5b98a3fd56ac2b14dd09af0b Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 7 Jun 2018 16:03:16 +0100
Subject: [PATCH 032/830] Remove event re-signing hacks

These "temporary fixes" have been here three and a half years, and I can't find
any events in the matrix.org database where the calculated signature differs
from what's in the db. It's time for them to go away.
---
 synapse/handlers/federation.py | 43 ----------------------------------
 1 file changed, 43 deletions(-)

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index fcf94befb766..60b97b140e33 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -938,16 +938,6 @@ def on_event_auth(self, event_id):
             [auth_id for auth_id, _ in event.auth_events],
             include_given=True
         )
-
-        for event in auth:
-            event.signatures.update(
-                compute_event_signature(
-                    event,
-                    self.hs.hostname,
-                    self.hs.config.signing_key[0]
-                )
-            )
-
         defer.returnValue([e for e in auth])
 
     @log_function
@@ -1405,18 +1395,6 @@ def get_state_for_pdu(self, room_id, event_id):
                     del results[(event.type, event.state_key)]
 
             res = list(results.values())
-            for event in res:
-                # We sign these again because there was a bug where we
-                # incorrectly signed things the first time round
-                if self.is_mine_id(event.event_id):
-                    event.signatures.update(
-                        compute_event_signature(
-                            event,
-                            self.hs.hostname,
-                            self.hs.config.signing_key[0]
-                        )
-                    )
-
             defer.returnValue(res)
         else:
             defer.returnValue([])
@@ -1481,18 +1459,6 @@ def get_persisted_pdu(self, origin, event_id, do_auth=True):
         )
 
         if event:
-            if self.is_mine_id(event.event_id):
-                # FIXME: This is a temporary work around where we occasionally
-                # return events slightly differently than when they were
-                # originally signed
-                event.signatures.update(
-                    compute_event_signature(
-                        event,
-                        self.hs.hostname,
-                        self.hs.config.signing_key[0]
-                    )
-                )
-
             if do_auth:
                 in_room = yield self.auth.check_host_in_room(
                     event.room_id,
@@ -1760,15 +1726,6 @@ def on_query_auth(self, origin, event_id, remote_auth_chain, rejects,
             local_auth_chain, remote_auth_chain
         )
 
-        for event in ret["auth_chain"]:
-            event.signatures.update(
-                compute_event_signature(
-                    event,
-                    self.hs.hostname,
-                    self.hs.config.signing_key[0]
-                )
-            )
-
         logger.debug("on_query_auth returning: %s", ret)
 
         defer.returnValue(ret)

From 76c80e3fdf96f93cface7d8251910bb085ce2a24 Mon Sep 17 00:00:00 2001
From: Nicholas Eckardt <neckardt@umich.edu>
Date: Sun, 10 Jun 2018 18:15:15 -0700
Subject: [PATCH 033/830] The project `matrix-synapse-auto-deploy` does not
 seem to be maintained anymore.

It has been over a year since any code has been commited. Some of the relevant links in
the documentation are broken, but since no pull requests are being accepted, they
won't get fixed. We should probably remove this from the README.
---
 README.rst | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/README.rst b/README.rst
index 4fe54b0c90a3..68e31ba8cf7f 100644
--- a/README.rst
+++ b/README.rst
@@ -161,10 +161,6 @@ There is an offical synapse image available at https://hub.docker.com/r/matrixdo
 
 Alternatively, Andreas Peters (previously Silvio Fricke) has contributed a Dockerfile to automate a synapse server in a single Docker image, at https://hub.docker.com/r/avhost/docker-matrix/tags/
 
-Also, Martin Giess has created an auto-deployment process with vagrant/ansible,
-tested with VirtualBox/AWS/DigitalOcean - see https://github.com/EMnify/matrix-synapse-auto-deploy
-for details.
-
 Configuring synapse
 -------------------
 

From 94700e55fa93653d678fb5e27322fde4c0a15f3d Mon Sep 17 00:00:00 2001
From: Michael Telatynski <7t3chguy@gmail.com>
Date: Wed, 13 Jun 2018 10:31:01 +0100
Subject: [PATCH 034/830] if inviter_display_name == ""||None then default to
 inviter MXID to prevent email invite from "None"

---
 synapse/handlers/room_member.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index b3f979b246bf..1b8dfa825479 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -565,6 +565,10 @@ def _make_and_store_3pid_invite(
             inviter_display_name = member_event.content.get("displayname", "")
             inviter_avatar_url = member_event.content.get("avatar_url", "")
 
+        # if user has no display name, default to their MXID
+        if not inviter_display_name:
+            inviter_display_name = user.to_string()
+
         canonical_room_alias = ""
         canonical_alias_event = room_state.get((EventTypes.CanonicalAlias, ""))
         if canonical_alias_event:

From 3d605853c8e649ab4b3f91fb0a32cc77ef05d71f Mon Sep 17 00:00:00 2001
From: Jeroen <vo.jeroen@gmail.com>
Date: Sun, 24 Jun 2018 22:38:43 +0200
Subject: [PATCH 035/830] send SNI for federation requests

---
 synapse/app/client_reader.py           |  2 ++
 synapse/app/event_creator.py           |  2 ++
 synapse/app/federation_reader.py       |  2 ++
 synapse/app/federation_sender.py       |  2 ++
 synapse/app/frontend_proxy.py          |  2 ++
 synapse/app/homeserver.py              |  2 ++
 synapse/app/media_repository.py        |  2 ++
 synapse/app/user_dir.py                |  2 ++
 synapse/config/tls.py                  |  9 +++++++
 synapse/crypto/context_factory.py      | 34 +++++++++++++++++++++++++-
 synapse/crypto/keyclient.py            |  4 +--
 synapse/crypto/keyring.py              |  4 +--
 synapse/http/endpoint.py               | 11 ++++-----
 synapse/http/matrixfederationclient.py |  4 +--
 tests/utils.py                         |  2 ++
 15 files changed, 71 insertions(+), 13 deletions(-)

diff --git a/synapse/app/client_reader.py b/synapse/app/client_reader.py
index 654ddb8414d5..1c885e321cdb 100644
--- a/synapse/app/client_reader.py
+++ b/synapse/app/client_reader.py
@@ -153,11 +153,13 @@ def start(config_options):
     database_engine = create_engine(config.database_config)
 
     tls_server_context_factory = context_factory.ServerContextFactory(config)
+    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
 
     ss = ClientReaderServer(
         config.server_name,
         db_config=config.database_config,
         tls_server_context_factory=tls_server_context_factory,
+        tls_client_options_factory=tls_client_options_factory,
         config=config,
         version_string="Synapse/" + get_version_string(synapse),
         database_engine=database_engine,
diff --git a/synapse/app/event_creator.py b/synapse/app/event_creator.py
index 441467093a4b..ccbdd8b2df75 100644
--- a/synapse/app/event_creator.py
+++ b/synapse/app/event_creator.py
@@ -171,11 +171,13 @@ def start(config_options):
     database_engine = create_engine(config.database_config)
 
     tls_server_context_factory = context_factory.ServerContextFactory(config)
+    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
 
     ss = EventCreatorServer(
         config.server_name,
         db_config=config.database_config,
         tls_server_context_factory=tls_server_context_factory,
+        tls_client_options_factory=tls_client_options_factory,
         config=config,
         version_string="Synapse/" + get_version_string(synapse),
         database_engine=database_engine,
diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py
index b2415cc6712f..7fd20322f4ec 100644
--- a/synapse/app/federation_reader.py
+++ b/synapse/app/federation_reader.py
@@ -142,11 +142,13 @@ def start(config_options):
     database_engine = create_engine(config.database_config)
 
     tls_server_context_factory = context_factory.ServerContextFactory(config)
+    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
 
     ss = FederationReaderServer(
         config.server_name,
         db_config=config.database_config,
         tls_server_context_factory=tls_server_context_factory,
+        tls_client_options_factory=tls_client_options_factory,
         config=config,
         version_string="Synapse/" + get_version_string(synapse),
         database_engine=database_engine,
diff --git a/synapse/app/federation_sender.py b/synapse/app/federation_sender.py
index 13d2b7005310..67dffaccbb03 100644
--- a/synapse/app/federation_sender.py
+++ b/synapse/app/federation_sender.py
@@ -185,11 +185,13 @@ def start(config_options):
     config.send_federation = True
 
     tls_server_context_factory = context_factory.ServerContextFactory(config)
+    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
 
     ps = FederationSenderServer(
         config.server_name,
         db_config=config.database_config,
         tls_server_context_factory=tls_server_context_factory,
+        tls_client_options_factory=tls_client_options_factory,
         config=config,
         version_string="Synapse/" + get_version_string(synapse),
         database_engine=database_engine,
diff --git a/synapse/app/frontend_proxy.py b/synapse/app/frontend_proxy.py
index d2bae4ad0316..d5638087a0e8 100644
--- a/synapse/app/frontend_proxy.py
+++ b/synapse/app/frontend_proxy.py
@@ -209,11 +209,13 @@ def start(config_options):
     database_engine = create_engine(config.database_config)
 
     tls_server_context_factory = context_factory.ServerContextFactory(config)
+    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
 
     ss = FrontendProxyServer(
         config.server_name,
         db_config=config.database_config,
         tls_server_context_factory=tls_server_context_factory,
+        tls_client_options_factory=tls_client_options_factory,
         config=config,
         version_string="Synapse/" + get_version_string(synapse),
         database_engine=database_engine,
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index ae5fc751d550..d48d68bc7eaa 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -321,6 +321,7 @@ def setup(config_options):
     events.USE_FROZEN_DICTS = config.use_frozen_dicts
 
     tls_server_context_factory = context_factory.ServerContextFactory(config)
+    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
 
     database_engine = create_engine(config.database_config)
     config.database_config["args"]["cp_openfun"] = database_engine.on_new_connection
@@ -329,6 +330,7 @@ def setup(config_options):
         config.server_name,
         db_config=config.database_config,
         tls_server_context_factory=tls_server_context_factory,
+        tls_client_options_factory=tls_client_options_factory,
         config=config,
         version_string="Synapse/" + get_version_string(synapse),
         database_engine=database_engine,
diff --git a/synapse/app/media_repository.py b/synapse/app/media_repository.py
index 19a682cce30f..cc3f31a4384e 100644
--- a/synapse/app/media_repository.py
+++ b/synapse/app/media_repository.py
@@ -156,11 +156,13 @@ def start(config_options):
     database_engine = create_engine(config.database_config)
 
     tls_server_context_factory = context_factory.ServerContextFactory(config)
+    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
 
     ss = MediaRepositoryServer(
         config.server_name,
         db_config=config.database_config,
         tls_server_context_factory=tls_server_context_factory,
+        tls_client_options_factory=tls_client_options_factory,
         config=config,
         version_string="Synapse/" + get_version_string(synapse),
         database_engine=database_engine,
diff --git a/synapse/app/user_dir.py b/synapse/app/user_dir.py
index f5726e3df6eb..51e7917ce248 100644
--- a/synapse/app/user_dir.py
+++ b/synapse/app/user_dir.py
@@ -213,11 +213,13 @@ def start(config_options):
     config.update_user_directory = True
 
     tls_server_context_factory = context_factory.ServerContextFactory(config)
+    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
 
     ps = UserDirectoryServer(
         config.server_name,
         db_config=config.database_config,
         tls_server_context_factory=tls_server_context_factory,
+        tls_client_options_factory=tls_client_options_factory,
         config=config,
         version_string="Synapse/" + get_version_string(synapse),
         database_engine=database_engine,
diff --git a/synapse/config/tls.py b/synapse/config/tls.py
index b66154bc7ced..4e7d1bd93ea9 100644
--- a/synapse/config/tls.py
+++ b/synapse/config/tls.py
@@ -47,6 +47,8 @@ def read_config(self, config):
 
         self.tls_fingerprints = config["tls_fingerprints"]
 
+        self.tls_ignore_certificate_validation = config.get("tls_ignore_certificate_validation", False)
+
         # Check that our own certificate is included in the list of fingerprints
         # and include it if it is not.
         x509_certificate_bytes = crypto.dump_certificate(
@@ -73,6 +75,8 @@ def default_config(self, config_dir_path, server_name, **kwargs):
         tls_private_key_path = base_key_name + ".tls.key"
         tls_dh_params_path = base_key_name + ".tls.dh"
 
+        tls_ignore_certificate_validation = False
+
         return """\
         # PEM encoded X509 certificate for TLS.
         # You can replace the self-signed certificate that synapse
@@ -117,6 +121,11 @@ def default_config(self, config_dir_path, server_name, **kwargs):
         #
         tls_fingerprints: []
         # tls_fingerprints: [{"sha256": "<base64_encoded_sha256_fingerprint>"}]
+
+        # Ignore certificate validation for TLS client connections to other
+        # homeservers using federation. Don't enable this in a production
+        # environment, unless you know what you are doing!
+        tls_ignore_certificate_validation: %(tls_ignore_certificate_validation)s
         """ % locals()
 
     def read_tls_certificate(self, cert_path):
diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py
index 0397f73ab4f0..297a5fb045df 100644
--- a/synapse/crypto/context_factory.py
+++ b/synapse/crypto/context_factory.py
@@ -14,7 +14,8 @@
 
 from twisted.internet import ssl
 from OpenSSL import SSL, crypto
-from twisted.internet._sslverify import _defaultCurveName
+from twisted.internet._sslverify import _defaultCurveName, ClientTLSOptions, OpenSSLCertificateOptions, \
+    optionsForClientTLS
 
 import logging
 
@@ -48,3 +49,34 @@ def configure_context(context, config):
 
     def getContext(self):
         return self._context
+
+
+class ClientTLSOptionsNoCertVerification(ClientTLSOptions):
+    """Redefinition of ClientTLSOptions to completely ignore certificate
+    validation. Should be kept in sync with the original class in Twisted.
+    This version of ClientTLSOptions is only intended for development use."""
+
+    def __init__(self, *args, **kwargs):
+        super(ClientTLSOptionsNoCertVerification, self).__init__(*args, **kwargs)
+
+        def do_nothing(*_args, **_kwargs):
+            pass
+
+        self._ctx.set_info_callback(do_nothing)
+
+
+class ClientTLSOptionsFactory(object):
+    """Factory for Twisted ClientTLSOptions that are used to make connections
+    to remote servers for federation."""
+
+    def __init__(self, config):
+        self._ignore_certificate_validation = config.tls_ignore_certificate_validation
+
+    def get_options(self, host):
+        if self._ignore_certificate_validation:
+            return ClientTLSOptionsNoCertVerification(
+                unicode(host),
+                OpenSSLCertificateOptions(verify=False).getContext()
+            )
+        else:
+            return optionsForClientTLS(unicode(host))
diff --git a/synapse/crypto/keyclient.py b/synapse/crypto/keyclient.py
index f1fd488b9057..8e48f8a9cff9 100644
--- a/synapse/crypto/keyclient.py
+++ b/synapse/crypto/keyclient.py
@@ -28,14 +28,14 @@
 
 
 @defer.inlineCallbacks
-def fetch_server_key(server_name, ssl_context_factory, path=KEY_API_V1):
+def fetch_server_key(server_name, tls_client_options_factory, path=KEY_API_V1):
     """Fetch the keys for a remote server."""
 
     factory = SynapseKeyClientFactory()
     factory.path = path
     factory.host = server_name
     endpoint = matrix_federation_endpoint(
-        reactor, server_name, ssl_context_factory, timeout=30
+        reactor, server_name, tls_client_options_factory, timeout=30
     )
 
     for i in range(5):
diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py
index 9b17ef0a08f0..9a1bb211fbc3 100644
--- a/synapse/crypto/keyring.py
+++ b/synapse/crypto/keyring.py
@@ -510,7 +510,7 @@ def get_server_verify_key_v2_direct(self, server_name, key_ids):
                 continue
 
             (response, tls_certificate) = yield fetch_server_key(
-                server_name, self.hs.tls_server_context_factory,
+                server_name, self.tls_client_options_factory,
                 path=(b"/_matrix/key/v2/server/%s" % (
                     urllib.quote(requested_key_id),
                 )).encode("ascii"),
@@ -653,7 +653,7 @@ def get_server_verify_key_v1_direct(self, server_name, key_ids):
         # Try to fetch the key from the remote server.
 
         (response, tls_certificate) = yield fetch_server_key(
-            server_name, self.hs.tls_server_context_factory
+            server_name, self.hs.tls_client_options_factory
         )
 
         # Check the response.
diff --git a/synapse/http/endpoint.py b/synapse/http/endpoint.py
index 87a482650df1..e783f95719fc 100644
--- a/synapse/http/endpoint.py
+++ b/synapse/http/endpoint.py
@@ -26,7 +26,6 @@
 
 logger = logging.getLogger(__name__)
 
-
 SERVER_CACHE = {}
 
 # our record of an individual server which can be tried to reach a destination.
@@ -38,15 +37,15 @@
 )
 
 
-def matrix_federation_endpoint(reactor, destination, ssl_context_factory=None,
+def matrix_federation_endpoint(reactor, destination, tls_client_options_factory=None,
                                timeout=None):
     """Construct an endpoint for the given matrix destination.
 
     Args:
         reactor: Twisted reactor.
         destination (bytes): The name of the server to connect to.
-        ssl_context_factory (twisted.internet.ssl.ContextFactory): Factory
-            which generates SSL contexts to use for TLS.
+        tls_client_options_factory (synapse.crypto.context_factory.ClientTLSOptionsFactory): 
+            Factory which generates TLS options for client connections.
         timeout (int): connection timeout in seconds
     """
 
@@ -59,13 +58,13 @@ def matrix_federation_endpoint(reactor, destination, ssl_context_factory=None,
     if timeout is not None:
         endpoint_kw_args.update(timeout=timeout)
 
-    if ssl_context_factory is None:
+    if tls_client_options_factory is None:
         transport_endpoint = HostnameEndpoint
         default_port = 8008
     else:
         def transport_endpoint(reactor, host, port, timeout):
             return wrapClientTLS(
-                ssl_context_factory,
+                tls_client_options_factory.get_options(unicode(host)),
                 HostnameEndpoint(reactor, host, port, timeout=timeout))
         default_port = 8448
 
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 4e0399e7629a..66796a202f4e 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -62,14 +62,14 @@
 
 class MatrixFederationEndpointFactory(object):
     def __init__(self, hs):
-        self.tls_server_context_factory = hs.tls_server_context_factory
+        self.tls_client_options_factory = hs.tls_client_options_factory
 
     def endpointForURI(self, uri):
         destination = uri.netloc
 
         return matrix_federation_endpoint(
             reactor, destination, timeout=10,
-            ssl_context_factory=self.tls_server_context_factory
+            tls_client_options_factory = self.tls_client_options_factory
         )
 
 
diff --git a/tests/utils.py b/tests/utils.py
index 189fd2711cc7..0b501e0786c3 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -114,6 +114,7 @@ def setup_test_homeserver(name="test", datastore=None, config=None, reactor=None
             database_engine=db_engine,
             room_list_handler=object(),
             tls_server_context_factory=Mock(),
+            tls_client_options_factory=Mock(),
             reactor=reactor,
             **kargs
         )
@@ -134,6 +135,7 @@ def setup_test_homeserver(name="test", datastore=None, config=None, reactor=None
             database_engine=db_engine,
             room_list_handler=object(),
             tls_server_context_factory=Mock(),
+            tls_client_options_factory=Mock(),
             **kargs
         )
 

From 07b4f88de9ee534629db29aa4fc398607e7d866e Mon Sep 17 00:00:00 2001
From: Jeroen <vo.jeroen@gmail.com>
Date: Mon, 25 Jun 2018 12:31:16 +0200
Subject: [PATCH 036/830] formatting changes for pep8

---
 synapse/config/tls.py                  | 4 +++-
 synapse/crypto/context_factory.py      | 4 ++--
 synapse/http/endpoint.py               | 3 ++-
 synapse/http/matrixfederationclient.py | 2 +-
 4 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/synapse/config/tls.py b/synapse/config/tls.py
index 4e7d1bd93ea9..b09dc986ab29 100644
--- a/synapse/config/tls.py
+++ b/synapse/config/tls.py
@@ -47,7 +47,9 @@ def read_config(self, config):
 
         self.tls_fingerprints = config["tls_fingerprints"]
 
-        self.tls_ignore_certificate_validation = config.get("tls_ignore_certificate_validation", False)
+        self.tls_ignore_certificate_validation = config.get(
+            "tls_ignore_certificate_validation", False
+        )
 
         # Check that our own certificate is included in the list of fingerprints
         # and include it if it is not.
diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py
index 297a5fb045df..415899e62bdd 100644
--- a/synapse/crypto/context_factory.py
+++ b/synapse/crypto/context_factory.py
@@ -14,8 +14,8 @@
 
 from twisted.internet import ssl
 from OpenSSL import SSL, crypto
-from twisted.internet._sslverify import _defaultCurveName, ClientTLSOptions, OpenSSLCertificateOptions, \
-    optionsForClientTLS
+from twisted.internet._sslverify import _defaultCurveName, ClientTLSOptions, \
+    OpenSSLCertificateOptions, optionsForClientTLS
 
 import logging
 
diff --git a/synapse/http/endpoint.py b/synapse/http/endpoint.py
index e783f95719fc..abf486208418 100644
--- a/synapse/http/endpoint.py
+++ b/synapse/http/endpoint.py
@@ -44,7 +44,8 @@ def matrix_federation_endpoint(reactor, destination, tls_client_options_factory=
     Args:
         reactor: Twisted reactor.
         destination (bytes): The name of the server to connect to.
-        tls_client_options_factory (synapse.crypto.context_factory.ClientTLSOptionsFactory): 
+        tls_client_options_factory
+            (synapse.crypto.context_factory.ClientTLSOptionsFactory):
             Factory which generates TLS options for client connections.
         timeout (int): connection timeout in seconds
     """
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 66796a202f4e..b48d05fcd2f0 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -69,7 +69,7 @@ def endpointForURI(self, uri):
 
         return matrix_federation_endpoint(
             reactor, destination, timeout=10,
-            tls_client_options_factory = self.tls_client_options_factory
+            tls_client_options_factory=self.tls_client_options_factory
         )
 
 

From b7f34ee348c9f064d98922dfdbe24a3cf18ca00a Mon Sep 17 00:00:00 2001
From: Jeroen <vo.jeroen@gmail.com>
Date: Tue, 26 Jun 2018 20:41:05 +0200
Subject: [PATCH 037/830] allow self-signed certificates

---
 synapse/config/tls.py             | 11 ------
 synapse/crypto/context_factory.py | 58 +++++++++++++++++++------------
 synapse/http/endpoint.py          |  2 +-
 3 files changed, 36 insertions(+), 35 deletions(-)

diff --git a/synapse/config/tls.py b/synapse/config/tls.py
index b09dc986ab29..b66154bc7ced 100644
--- a/synapse/config/tls.py
+++ b/synapse/config/tls.py
@@ -47,10 +47,6 @@ def read_config(self, config):
 
         self.tls_fingerprints = config["tls_fingerprints"]
 
-        self.tls_ignore_certificate_validation = config.get(
-            "tls_ignore_certificate_validation", False
-        )
-
         # Check that our own certificate is included in the list of fingerprints
         # and include it if it is not.
         x509_certificate_bytes = crypto.dump_certificate(
@@ -77,8 +73,6 @@ def default_config(self, config_dir_path, server_name, **kwargs):
         tls_private_key_path = base_key_name + ".tls.key"
         tls_dh_params_path = base_key_name + ".tls.dh"
 
-        tls_ignore_certificate_validation = False
-
         return """\
         # PEM encoded X509 certificate for TLS.
         # You can replace the self-signed certificate that synapse
@@ -123,11 +117,6 @@ def default_config(self, config_dir_path, server_name, **kwargs):
         #
         tls_fingerprints: []
         # tls_fingerprints: [{"sha256": "<base64_encoded_sha256_fingerprint>"}]
-
-        # Ignore certificate validation for TLS client connections to other
-        # homeservers using federation. Don't enable this in a production
-        # environment, unless you know what you are doing!
-        tls_ignore_certificate_validation: %(tls_ignore_certificate_validation)s
         """ % locals()
 
     def read_tls_certificate(self, cert_path):
diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py
index 415899e62bdd..e93afbf975c8 100644
--- a/synapse/crypto/context_factory.py
+++ b/synapse/crypto/context_factory.py
@@ -11,18 +11,19 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import logging
 
-from twisted.internet import ssl
+import idna
 from OpenSSL import SSL, crypto
-from twisted.internet._sslverify import _defaultCurveName, ClientTLSOptions, \
-    OpenSSLCertificateOptions, optionsForClientTLS
-
-import logging
+from twisted.internet.ssl import ContextFactory, CertificateOptions
+from twisted.internet._sslverify import _defaultCurveName, _tolerateErrors
+from twisted.internet.interfaces import IOpenSSLClientConnectionCreator
+from zope.interface import implementer
 
 logger = logging.getLogger(__name__)
 
 
-class ServerContextFactory(ssl.ContextFactory):
+class ServerContextFactory(ContextFactory):
     """Factory for PyOpenSSL SSL contexts that are used to handle incoming
     connections and to make connections to remote servers."""
 
@@ -51,18 +52,31 @@ def getContext(self):
         return self._context
 
 
-class ClientTLSOptionsNoCertVerification(ClientTLSOptions):
-    """Redefinition of ClientTLSOptions to completely ignore certificate
-    validation. Should be kept in sync with the original class in Twisted.
-    This version of ClientTLSOptions is only intended for development use."""
+@implementer(IOpenSSLClientConnectionCreator)
+class ClientTLSOptions(object):
+    """
+    Client creator for TLS without certificate identity verification. This is a
+    copy of twisted.internet._sslverify.ClientTLSOptions with the identity
+    verification left out. For documentation, see the twisted documentation.
+    """
 
-    def __init__(self, *args, **kwargs):
-        super(ClientTLSOptionsNoCertVerification, self).__init__(*args, **kwargs)
+    def __init__(self, hostname, ctx):
+        self._ctx = ctx
+        self._hostname = hostname
+        self._hostnameBytes = idna.encode(hostname)
+        ctx.set_info_callback(
+            _tolerateErrors(self._identityVerifyingInfoCallback)
+        )
 
-        def do_nothing(*_args, **_kwargs):
-            pass
+    def clientConnectionForTLS(self, tlsProtocol):
+        context = self._ctx
+        connection = SSL.Connection(context, None)
+        connection.set_app_data(tlsProtocol)
+        return connection
 
-        self._ctx.set_info_callback(do_nothing)
+    def _identityVerifyingInfoCallback(self, connection, where, ret):
+        if where & SSL.SSL_CB_HANDSHAKE_START:
+            connection.set_tlsext_host_name(self._hostnameBytes)
 
 
 class ClientTLSOptionsFactory(object):
@@ -70,13 +84,11 @@ class ClientTLSOptionsFactory(object):
     to remote servers for federation."""
 
     def __init__(self, config):
-        self._ignore_certificate_validation = config.tls_ignore_certificate_validation
+        # We don't use config options yet
+        pass
 
     def get_options(self, host):
-        if self._ignore_certificate_validation:
-            return ClientTLSOptionsNoCertVerification(
-                unicode(host),
-                OpenSSLCertificateOptions(verify=False).getContext()
-            )
-        else:
-            return optionsForClientTLS(unicode(host))
+        return ClientTLSOptions(
+            unicode(host),
+            CertificateOptions(verify=False).getContext()
+        )
diff --git a/synapse/http/endpoint.py b/synapse/http/endpoint.py
index abf486208418..39432da4527a 100644
--- a/synapse/http/endpoint.py
+++ b/synapse/http/endpoint.py
@@ -65,7 +65,7 @@ def matrix_federation_endpoint(reactor, destination, tls_client_options_factory=
     else:
         def transport_endpoint(reactor, host, port, timeout):
             return wrapClientTLS(
-                tls_client_options_factory.get_options(unicode(host)),
+                tls_client_options_factory.get_options(host),
                 HostnameEndpoint(reactor, host, port, timeout=timeout))
         default_port = 8448
 

From 26651b0d6a55788ff7ee62eda2051f7a85aa45e7 Mon Sep 17 00:00:00 2001
From: Jeroen <vo.jeroen@gmail.com>
Date: Tue, 26 Jun 2018 21:10:45 +0200
Subject: [PATCH 038/830] towncrier changelog

---
 changelog.d/1491.feature | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/1491.feature

diff --git a/changelog.d/1491.feature b/changelog.d/1491.feature
new file mode 100644
index 000000000000..77b6d6ca09ea
--- /dev/null
+++ b/changelog.d/1491.feature
@@ -0,0 +1 @@
+Add support for the SNI extension to federation TLS connections
\ No newline at end of file

From 95341a8f6f9c3b7ebcb3a428de65b18740234f3e Mon Sep 17 00:00:00 2001
From: Jeroen <vo.jeroen@gmail.com>
Date: Tue, 26 Jun 2018 21:15:14 +0200
Subject: [PATCH 039/830] take idna implementation from twisted

---
 synapse/crypto/context_factory.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py
index e93afbf975c8..57694e18b0a5 100644
--- a/synapse/crypto/context_factory.py
+++ b/synapse/crypto/context_factory.py
@@ -13,8 +13,8 @@
 # limitations under the License.
 import logging
 
-import idna
 from OpenSSL import SSL, crypto
+from twisted.internet._idna import _idnaBytes
 from twisted.internet.ssl import ContextFactory, CertificateOptions
 from twisted.internet._sslverify import _defaultCurveName, _tolerateErrors
 from twisted.internet.interfaces import IOpenSSLClientConnectionCreator
@@ -63,7 +63,7 @@ class ClientTLSOptions(object):
     def __init__(self, hostname, ctx):
         self._ctx = ctx
         self._hostname = hostname
-        self._hostnameBytes = idna.encode(hostname)
+        self._hostnameBytes = _idnaBytes(hostname)
         ctx.set_info_callback(
             _tolerateErrors(self._identityVerifyingInfoCallback)
         )

From ff659161085d082e3071be6e9896bf63156e59d7 Mon Sep 17 00:00:00 2001
From: Sverre Moe <sverre@spacetec.no>
Date: Sun, 1 Jul 2018 00:06:51 +0200
Subject: [PATCH 040/830] Add instructions for install on OpenSUSE and SLES

---
 README.rst | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/README.rst b/README.rst
index 4fe54b0c90a3..b372842b5bb8 100644
--- a/README.rst
+++ b/README.rst
@@ -362,6 +362,19 @@ Synapse is in the Fedora repositories as ``matrix-synapse``::
 Oleg Girko provides Fedora RPMs at
 https://obs.infoserver.lv/project/monitor/matrix-synapse
 
+OpenSUSE
+--------
+
+Synapse is in the OpenSUSE repositories as ``matrix-synapse``::
+
+    sudo zypper install matrix-synapse
+
+SUSE Linux Enterprise Server
+----------------------------
+
+Unofficial package are built for SLES 15 in the openSUSE:Backports:SLE-15 repository at
+https://download.opensuse.org/repositories/openSUSE:/Backports:/SLE-15/standard/
+
 ArchLinux
 ---------
 

From c4e7ad0e0fbc27401d021a396579cfdefee5dbf9 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 4 Jul 2018 09:15:45 +0100
Subject: [PATCH 041/830] Add changelog

---
 changelog.d/3367.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3367.misc

diff --git a/changelog.d/3367.misc b/changelog.d/3367.misc
new file mode 100644
index 000000000000..1f21ddea488a
--- /dev/null
+++ b/changelog.d/3367.misc
@@ -0,0 +1 @@
+Remove unnecessary event re-signing hacks
\ No newline at end of file

From 2ef3f84945c3a6ef653d6f589c06e2d8882cca05 Mon Sep 17 00:00:00 2001
From: Peter <github@haas-en-berg.nl>
Date: Tue, 10 Jul 2018 21:14:22 +0200
Subject: [PATCH 042/830] change http links to https

Especially useful for the debian repo, as this makes it easier to get the key in a secure way
---
 README.rst | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/README.rst b/README.rst
index 4fe54b0c90a3..84b8cf572eb8 100644
--- a/README.rst
+++ b/README.rst
@@ -71,7 +71,7 @@ We'd like to invite you to join #matrix:matrix.org (via
 https://matrix.org/docs/projects/try-matrix-now.html), run a homeserver, take a look
 at the `Matrix spec <https://matrix.org/docs/spec>`_, and experiment with the
 `APIs <https://matrix.org/docs/api>`_ and `Client SDKs
-<http://matrix.org/docs/projects/try-matrix-now.html#client-sdks>`_.
+<https://matrix.org/docs/projects/try-matrix-now.html#client-sdks>`_.
 
 Thanks for using Matrix!
 
@@ -283,7 +283,7 @@ Connecting to Synapse from a client
 
 The easiest way to try out your new Synapse installation is by connecting to it
 from a web client. The easiest option is probably the one at
-http://riot.im/app. You will need to specify a "Custom server" when you log on
+https://riot.im/app. You will need to specify a "Custom server" when you log on
 or register: set this to ``https://domain.tld`` if you setup a reverse proxy
 following the recommended setup, or ``https://localhost:8448`` - remember to specify the
 port (``:8448``) if not ``:443`` unless you changed the configuration. (Leave the identity
@@ -329,7 +329,7 @@ Security Note
 =============
 
 Matrix serves raw user generated data in some APIs - specifically the `content
-repository endpoints <http://matrix.org/docs/spec/client_server/latest.html#get-matrix-media-r0-download-servername-mediaid>`_.
+repository endpoints <https://matrix.org/docs/spec/client_server/latest.html#get-matrix-media-r0-download-servername-mediaid>`_.
 
 Whilst we have tried to mitigate against possible XSS attacks (e.g.
 https://github.com/matrix-org/synapse/pull/1021) we recommend running
@@ -348,7 +348,7 @@ Platform-Specific Instructions
 Debian
 ------
 
-Matrix provides official Debian packages via apt from http://matrix.org/packages/debian/.
+Matrix provides official Debian packages via apt from https://matrix.org/packages/debian/.
 Note that these packages do not include a client - choose one from
 https://matrix.org/docs/projects/try-matrix-now.html (or build your own with one of our SDKs :)
 
@@ -524,7 +524,7 @@ Troubleshooting Running
 -----------------------
 
 If synapse fails with ``missing "sodium.h"`` crypto errors, you may need
-to manually upgrade PyNaCL, as synapse uses NaCl (http://nacl.cr.yp.to/) for
+to manually upgrade PyNaCL, as synapse uses NaCl (https://nacl.cr.yp.to/) for
 encryption and digital signatures.
 Unfortunately PyNACL currently has a few issues
 (https://github.com/pyca/pynacl/issues/53) and
@@ -672,8 +672,8 @@ useful just for development purposes. See `<demo/README>`_.
 Using PostgreSQL
 ================
 
-As of Synapse 0.9, `PostgreSQL <http://www.postgresql.org>`_ is supported as an
-alternative to the `SQLite <http://sqlite.org/>`_ database that Synapse has
+As of Synapse 0.9, `PostgreSQL <https://www.postgresql.org>`_ is supported as an
+alternative to the `SQLite <https://sqlite.org/>`_ database that Synapse has
 traditionally used for convenience and simplicity.
 
 The advantages of Postgres include:
@@ -697,7 +697,7 @@ Using a reverse proxy with Synapse
 It is recommended to put a reverse proxy such as
 `nginx <https://nginx.org/en/docs/http/ngx_http_proxy_module.html>`_,
 `Apache <https://httpd.apache.org/docs/current/mod/mod_proxy_http.html>`_ or
-`HAProxy <http://www.haproxy.org/>`_ in front of Synapse. One advantage of
+`HAProxy <https://www.haproxy.org/>`_ in front of Synapse. One advantage of
 doing so is that it means that you can expose the default https port (443) to
 Matrix clients without needing to run Synapse with root privileges.
 

From 36f4fd3e1edfa6032a5ace89e38ec7268e06dee6 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Wed, 11 Jul 2018 15:45:44 +0100
Subject: [PATCH 043/830] Comment dummy TURN parameters in default config

This default config is parsed and used a base before the actual
config is overlaid, so with these values not commented out, the
code to detect when no turn params were set and refuse to generate
credentials was never firing because the dummy default was always set.
---
 changelog.d/3511.bugfix | 1 +
 synapse/config/voip.py  | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)
 create mode 100644 changelog.d/3511.bugfix

diff --git a/changelog.d/3511.bugfix b/changelog.d/3511.bugfix
new file mode 100644
index 000000000000..460fe24ac9e2
--- /dev/null
+++ b/changelog.d/3511.bugfix
@@ -0,0 +1 @@
+Don't generate TURN credentials if no TURN config options are set
diff --git a/synapse/config/voip.py b/synapse/config/voip.py
index 3a4e16fa9648..d07bd24ffd7b 100644
--- a/synapse/config/voip.py
+++ b/synapse/config/voip.py
@@ -30,10 +30,10 @@ def default_config(self, **kwargs):
         ## Turn ##
 
         # The public URIs of the TURN server to give to clients
-        turn_uris: []
+        #turn_uris: []
 
         # The shared secret used to compute passwords for the TURN server
-        turn_shared_secret: "YOUR_SHARED_SECRET"
+        #turn_shared_secret: "YOUR_SHARED_SECRET"
 
         # The Username and password if the TURN server needs them and
         # does not use a token

From 1b5425527cb514499cbba7d6fe06bde7d07d3ec2 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Wed, 11 Jul 2018 16:02:29 +0100
Subject: [PATCH 044/830] I failed to correctly guess the PR number

---
 changelog.d/{3511.bugfix => 3514.bugfix} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename changelog.d/{3511.bugfix => 3514.bugfix} (100%)

diff --git a/changelog.d/3511.bugfix b/changelog.d/3514.bugfix
similarity index 100%
rename from changelog.d/3511.bugfix
rename to changelog.d/3514.bugfix

From 5797f5542b687b73ebdd8613f64ce0f38e637b55 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Thu, 12 Jul 2018 01:32:39 +0100
Subject: [PATCH 045/830] WIP to announce deleted devices over federation

Previously we queued up the poke correctly when the device was deleted,
but then the actual EDU wouldn't get sent, as the device was no longer known.
Instead, we now send EDUs for deleted devices too if there's a poke for them.
---
 synapse/notifier.py                |  2 +-
 synapse/storage/devices.py         | 40 +++++++++++++++++++++---------
 synapse/storage/end_to_end_keys.py | 16 +++++++++++-
 3 files changed, 44 insertions(+), 14 deletions(-)

diff --git a/synapse/notifier.py b/synapse/notifier.py
index 51cbd66f0628..e650c3e4941a 100644
--- a/synapse/notifier.py
+++ b/synapse/notifier.py
@@ -274,7 +274,7 @@ def _notify_app_services(self, room_stream_id):
             logger.exception("Error notifying application services of event")
 
     def on_new_event(self, stream_key, new_token, users=[], rooms=[]):
-        """ Used to inform listeners that something has happend event wise.
+        """ Used to inform listeners that something has happened event wise.
 
         Will wake up all listeners for the given users and rooms.
         """
diff --git a/synapse/storage/devices.py b/synapse/storage/devices.py
index ec68e39f1e87..0c797f9f3e22 100644
--- a/synapse/storage/devices.py
+++ b/synapse/storage/devices.py
@@ -239,6 +239,7 @@ def mark_remote_user_device_list_as_unsubscribed(self, user_id):
     def update_remote_device_list_cache_entry(self, user_id, device_id, content,
                                               stream_id):
         """Updates a single user's device in the cache.
+           If the content is null, delete the device from the cache.
         """
         return self.runInteraction(
             "update_remote_device_list_cache_entry",
@@ -248,17 +249,32 @@ def update_remote_device_list_cache_entry(self, user_id, device_id, content,
 
     def _update_remote_device_list_cache_entry_txn(self, txn, user_id, device_id,
                                                    content, stream_id):
-        self._simple_upsert_txn(
-            txn,
-            table="device_lists_remote_cache",
-            keyvalues={
-                "user_id": user_id,
-                "device_id": device_id,
-            },
-            values={
-                "content": json.dumps(content),
-            }
-        )
+        if content is None:
+            self._simple_delete_txn(
+                txn,
+                table="device_lists_remote_cache",
+                keyvalues={
+                    "user_id": user_id,
+                    "device_id": device_id,
+                },
+            )
+
+            # Do we need this?
+            txn.call_after(
+                self.device_id_exists_cache.invalidate, (user_id, device_id,)
+            )
+        else:
+            self._simple_upsert_txn(
+                txn,
+                table="device_lists_remote_cache",
+                keyvalues={
+                    "user_id": user_id,
+                    "device_id": device_id,
+                },
+                values={
+                    "content": json.dumps(content),
+                }
+            )
 
         txn.call_after(self._get_cached_user_device.invalidate, (user_id, device_id,))
         txn.call_after(self._get_cached_devices_for_user.invalidate, (user_id,))
@@ -366,7 +382,7 @@ def _get_devices_by_remote_txn(self, txn, destination, from_stream_id,
             now_stream_id = max(stream_id for stream_id in itervalues(query_map))
 
         devices = self._get_e2e_device_keys_txn(
-            txn, query_map.keys(), include_all_devices=True
+            txn, query_map.keys(), include_all_devices=True, include_deleted_devices=True
         )
 
         prev_sent_id_sql = """
diff --git a/synapse/storage/end_to_end_keys.py b/synapse/storage/end_to_end_keys.py
index 7ae5c6548275..f61553cec8db 100644
--- a/synapse/storage/end_to_end_keys.py
+++ b/synapse/storage/end_to_end_keys.py
@@ -64,12 +64,17 @@ def _set_e2e_device_keys_txn(txn):
         )
 
     @defer.inlineCallbacks
-    def get_e2e_device_keys(self, query_list, include_all_devices=False):
+    def get_e2e_device_keys(
+        self, query_list, include_all_devices=False,
+        include_deleted_devices=False
+    ):
         """Fetch a list of device keys.
         Args:
             query_list(list): List of pairs of user_ids and device_ids.
             include_all_devices (bool): whether to include entries for devices
                 that don't have device keys
+            include_deleted_devices (bool): whether to include null entries for
+                devices which no longer exist (but where in the query_list)
         Returns:
             Dict mapping from user-id to dict mapping from device_id to
             dict containing "key_json", "device_display_name".
@@ -82,10 +87,19 @@ def get_e2e_device_keys(self, query_list, include_all_devices=False):
             query_list, include_all_devices,
         )
 
+        if include_deleted_devices:
+            deleted_devices = set(query_list)
+
         for user_id, device_keys in iteritems(results):
             for device_id, device_info in iteritems(device_keys):
+                if include_deleted_devices:
+                    deleted_devices -= (user_id, device_id)
                 device_info["keys"] = json.loads(device_info.pop("key_json"))
 
+        if include_deleted_devices:
+            for user_id, device_id in deleted_devices:
+                results.setdefault(user_id, {})[device_id] = None
+
         defer.returnValue(results)
 
     def _get_e2e_device_keys_txn(self, txn, query_list, include_all_devices):

From 12ec58301f946ced9702afbf6dfbfbc8c3dfd3dd Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Thu, 12 Jul 2018 11:39:43 +0100
Subject: [PATCH 046/830] shift to using an explicit deleted flag on
 m.device_list_update EDUs

and generally make it work.
---
 synapse/storage/devices.py         | 18 ++++++++++--------
 synapse/storage/end_to_end_keys.py | 27 +++++++++++++++------------
 2 files changed, 25 insertions(+), 20 deletions(-)

diff --git a/synapse/storage/devices.py b/synapse/storage/devices.py
index 0c797f9f3e22..203f50f07d21 100644
--- a/synapse/storage/devices.py
+++ b/synapse/storage/devices.py
@@ -239,7 +239,6 @@ def mark_remote_user_device_list_as_unsubscribed(self, user_id):
     def update_remote_device_list_cache_entry(self, user_id, device_id, content,
                                               stream_id):
         """Updates a single user's device in the cache.
-           If the content is null, delete the device from the cache.
         """
         return self.runInteraction(
             "update_remote_device_list_cache_entry",
@@ -249,7 +248,7 @@ def update_remote_device_list_cache_entry(self, user_id, device_id, content,
 
     def _update_remote_device_list_cache_entry_txn(self, txn, user_id, device_id,
                                                    content, stream_id):
-        if content is None:
+        if content.get("deleted"):
             self._simple_delete_txn(
                 txn,
                 table="device_lists_remote_cache",
@@ -409,12 +408,15 @@ def _get_devices_by_remote_txn(self, txn, destination, from_stream_id,
 
                 prev_id = stream_id
 
-                key_json = device.get("key_json", None)
-                if key_json:
-                    result["keys"] = json.loads(key_json)
-                device_display_name = device.get("device_display_name", None)
-                if device_display_name:
-                    result["device_display_name"] = device_display_name
+                if device is not None:
+                    key_json = device.get("key_json", None)
+                    if key_json:
+                        result["keys"] = json.loads(key_json)
+                    device_display_name = device.get("device_display_name", None)
+                    if device_display_name:
+                        result["device_display_name"] = device_display_name
+                else:
+                    result["deleted"] = True
 
                 results.append(result)
 
diff --git a/synapse/storage/end_to_end_keys.py b/synapse/storage/end_to_end_keys.py
index f61553cec8db..6c287194206f 100644
--- a/synapse/storage/end_to_end_keys.py
+++ b/synapse/storage/end_to_end_keys.py
@@ -74,7 +74,7 @@ def get_e2e_device_keys(
             include_all_devices (bool): whether to include entries for devices
                 that don't have device keys
             include_deleted_devices (bool): whether to include null entries for
-                devices which no longer exist (but where in the query_list)
+                devices which no longer exist (but were in the query_list)
         Returns:
             Dict mapping from user-id to dict mapping from device_id to
             dict containing "key_json", "device_display_name".
@@ -84,28 +84,25 @@ def get_e2e_device_keys(
 
         results = yield self.runInteraction(
             "get_e2e_device_keys", self._get_e2e_device_keys_txn,
-            query_list, include_all_devices,
+            query_list, include_all_devices, include_deleted_devices,
         )
 
-        if include_deleted_devices:
-            deleted_devices = set(query_list)
-
         for user_id, device_keys in iteritems(results):
             for device_id, device_info in iteritems(device_keys):
-                if include_deleted_devices:
-                    deleted_devices -= (user_id, device_id)
                 device_info["keys"] = json.loads(device_info.pop("key_json"))
 
-        if include_deleted_devices:
-            for user_id, device_id in deleted_devices:
-                results.setdefault(user_id, {})[device_id] = None
-
         defer.returnValue(results)
 
-    def _get_e2e_device_keys_txn(self, txn, query_list, include_all_devices):
+    def _get_e2e_device_keys_txn(
+        self, txn, query_list, include_all_devices=False,
+        include_deleted_devices=False,
+    ):
         query_clauses = []
         query_params = []
 
+        if include_deleted_devices:
+            deleted_devices = set(query_list)
+
         for (user_id, device_id) in query_list:
             query_clause = "user_id = ?"
             query_params.append(user_id)
@@ -133,8 +130,14 @@ def _get_e2e_device_keys_txn(self, txn, query_list, include_all_devices):
 
         result = {}
         for row in rows:
+            if include_deleted_devices:
+                deleted_devices.remove((row["user_id"], row["device_id"]))
             result.setdefault(row["user_id"], {})[row["device_id"]] = row
 
+        if include_deleted_devices:
+            for user_id, device_id in deleted_devices:
+                result.setdefault(user_id, {})[device_id] = None
+
         return result
 
     @defer.inlineCallbacks

From 37c4fba0ac95e0425193b1eaeaf96bc348e094d9 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Thu, 12 Jul 2018 11:45:33 +0100
Subject: [PATCH 047/830] changelog

---
 changelog.d/3520.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3520.bugfix

diff --git a/changelog.d/3520.bugfix b/changelog.d/3520.bugfix
new file mode 100644
index 000000000000..9278cb3708db
--- /dev/null
+++ b/changelog.d/3520.bugfix
@@ -0,0 +1 @@
+Correctly announce deleted devices over federation

From fe089b13cbfb1eeb99de30378f01fdc71ca097c1 Mon Sep 17 00:00:00 2001
From: Benedikt Heine <bebe@bebehei.de>
Date: Tue, 17 Jul 2018 09:02:45 +0200
Subject: [PATCH 048/830] [Docker] Build docker image via compose

It's much easier to build the image via docker-compose instead of an
error-prone low-level docker call.

Signed-off-by: Benedikt Heine <bebe@bebehei.de>
---
 contrib/docker/README.md          | 8 +-------
 contrib/docker/docker-compose.yml | 1 +
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/contrib/docker/README.md b/contrib/docker/README.md
index 61592109cbe0..562cdaac2b8c 100644
--- a/contrib/docker/README.md
+++ b/contrib/docker/README.md
@@ -9,13 +9,7 @@ use that server.
 
 ## Build
 
-Build the docker image with the `docker build` command from the root of the synapse repository.
-
-```
-docker build -t docker.io/matrixdotorg/synapse .
-```
-
-The `-t` option sets the image tag. Official images are tagged `matrixdotorg/synapse:<version>` where `<version>` is the same as the release tag in the synapse git repository.
+Build the docker image with the `docker-compose build` command.
 
 You may have a local Python wheel cache available, in which case copy the relevant packages in the ``cache/`` directory at the root of the project.
 
diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml
index 0b531949e0a6..3a8dfbae34ef 100644
--- a/contrib/docker/docker-compose.yml
+++ b/contrib/docker/docker-compose.yml
@@ -6,6 +6,7 @@ version: '3'
 services:
 
   synapse:
+    build: ../..
     image: docker.io/matrixdotorg/synapse:latest
     # Since snyapse does not retry to connect to the database, restart upon
     # failure

From 6e3fc657b4c8db10e872ba1c7264e82b7fed5ed0 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 18 Jul 2018 10:50:33 +0100
Subject: [PATCH 049/830] Resource tracking for background processes

This introduces a mechanism for tracking resource usage by background
processes, along with an example of how it will be used.

This will help address #3518, but more importantly will give us better insights
into things which are happening but not being shown up by the request metrics.

We *could* do this with Measure blocks, but:
 - I think having them pulled out as a completely separate metric class will
   make it easier to distinguish top-level processes from those which are
   nested.

 - I want to be able to report on in-flight background processes, and I don't
   think we want to do this for *all* Measure blocks.
---
 synapse/federation/transaction_queue.py       |  12 +-
 synapse/metrics/background_process_metrics.py | 179 ++++++++++++++++++
 2 files changed, 186 insertions(+), 5 deletions(-)
 create mode 100644 synapse/metrics/background_process_metrics.py

diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py
index 5a956ecfb3b0..5c5a73b73c5f 100644
--- a/synapse/federation/transaction_queue.py
+++ b/synapse/federation/transaction_queue.py
@@ -30,7 +30,8 @@
     sent_edus_counter,
     sent_transactions_counter,
 )
-from synapse.util import PreserveLoggingContext, logcontext
+from synapse.metrics.background_process_metrics import run_as_background_process
+from synapse.util import logcontext
 from synapse.util.metrics import measure_func
 from synapse.util.retryutils import NotRetryingDestination, get_retry_limiter
 
@@ -165,10 +166,11 @@ def notify_new_events(self, current_id):
         if self._is_processing:
             return
 
-        # fire off a processing loop in the background. It's likely it will
-        # outlast the current request, so run it in the sentinel logcontext.
-        with PreserveLoggingContext():
-            self._process_event_queue_loop()
+        # fire off a processing loop in the background
+        run_as_background_process(
+            "process_transaction_queue",
+            self._process_event_queue_loop,
+        )
 
     @defer.inlineCallbacks
     def _process_event_queue_loop(self):
diff --git a/synapse/metrics/background_process_metrics.py b/synapse/metrics/background_process_metrics.py
new file mode 100644
index 000000000000..9d820e44a6bd
--- /dev/null
+++ b/synapse/metrics/background_process_metrics.py
@@ -0,0 +1,179 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import six
+
+from prometheus_client.core import REGISTRY, Counter, GaugeMetricFamily
+
+from twisted.internet import defer
+
+from synapse.util.logcontext import LoggingContext, PreserveLoggingContext
+
+_background_process_start_count = Counter(
+    "synapse_background_process_start_count",
+    "Number of background processes started",
+    ["name"],
+)
+
+# we set registry=None in all of these to stop them getting registered with
+# the default registry. Instead we collect them all via the CustomCollector,
+# which ensures that we can update them before they are collected.
+#
+_background_process_ru_utime = Counter(
+    "synapse_background_process_ru_utime_seconds",
+    "User CPU time used by background processes, in seconds",
+    ["name"],
+    registry=None,
+)
+
+_background_process_ru_stime = Counter(
+    "synapse_background_process_ru_stime_seconds",
+    "System CPU time used by background processes, in seconds",
+    ["name"],
+    registry=None,
+)
+
+_background_process_db_txn_count = Counter(
+    "synapse_background_process_db_txn_count",
+    "Number of database transactions done by background processes",
+    ["name"],
+    registry=None,
+)
+
+_background_process_db_txn_duration = Counter(
+    "synapse_background_process_db_txn_duration_seconds",
+    ("Seconds spent by background processes waiting for database "
+     "transactions, excluding scheduling time"),
+    ["name"],
+    registry=None,
+)
+
+_background_process_db_sched_duration = Counter(
+    "synapse_background_process_db_sched_duration_seconds",
+    "Seconds spent by background processes waiting for database connections",
+    ["name"],
+    registry=None,
+)
+
+# map from description to a counter, so that we can name our logcontexts
+# incrementally. (It actually duplicates _background_process_start_count, but
+# it's much simpler to do so than to try to combine them.)
+_background_process_counts = dict()  # type: dict[str, int]
+
+# map from description to the currently running background processes.
+#
+# it's kept as a dict of sets rather than a big set so that we can keep track
+# of process descriptions that no longer have any active processes.
+_background_processes = dict()  # type: dict[str, set[_BackgroundProcess]]
+
+
+class _Collector(object):
+    """A custom metrics collector for the background process metrics.
+
+    Ensures that all of the metrics are up-to-date with any in-flight processes
+    before they are returned.
+    """
+    def collect(self):
+        background_process_in_flight_count = GaugeMetricFamily(
+            "synapse_background_process_in_flight_count",
+            "Number of background processes in flight",
+            labels=["name"],
+        )
+
+        for desc, processes in six.iteritems(_background_processes):
+            background_process_in_flight_count.add_metric(
+                (desc,), len(processes),
+            )
+            for process in processes:
+                process.update_metrics()
+
+        yield background_process_in_flight_count
+
+        # now we need to run collect() over each of the static Counters, and
+        # yield each metric they return.
+        for m in (
+                _background_process_ru_utime,
+                _background_process_ru_stime,
+                _background_process_db_txn_count,
+                _background_process_db_txn_duration,
+                _background_process_db_sched_duration,
+        ):
+            for r in m.collect():
+                yield r
+
+
+REGISTRY.register(_Collector())
+
+
+class _BackgroundProcess(object):
+    def __init__(self, desc, ctx):
+        self.desc = desc
+        self._context = ctx
+        self._reported_stats = None
+
+    def update_metrics(self):
+        """Updates the metrics with values from this process."""
+        new_stats = self._context.get_resource_usage()
+        if self._reported_stats is None:
+            diff = new_stats
+        else:
+            diff = new_stats - self._reported_stats
+        self._reported_stats = new_stats
+
+        _background_process_ru_utime.labels(self.desc).inc(diff.ru_utime)
+        _background_process_ru_stime.labels(self.desc).inc(diff.ru_stime)
+        _background_process_db_txn_count.labels(self.desc).inc(
+            diff.db_txn_count,
+        )
+        _background_process_db_txn_duration.labels(self.desc).inc(
+            diff.db_txn_duration_sec,
+        )
+        _background_process_db_sched_duration.labels(self.desc).inc(
+            diff.db_sched_duration_sec,
+        )
+
+
+def run_as_background_process(desc, func, *args, **kwargs):
+    """Run the given function in its own logcontext, with resource metrics
+
+    This should be used to wrap processes which are fired off to run in the
+    background, instead of being associated with a particular request.
+
+    Args:
+        desc (str): a description for this background process type
+        func: a function, which may return a Deferred
+        args: positional args for func
+        kwargs: keyword args for func
+
+    Returns: None
+    """
+    @defer.inlineCallbacks
+    def run():
+        count = _background_process_counts.get(desc, 0)
+        _background_process_counts[desc] = count + 1
+        _background_process_start_count.labels(desc).inc()
+
+        with LoggingContext(desc) as context:
+            context.request = "%s-%i" % (desc, count)
+            proc = _BackgroundProcess(desc, context)
+            _background_processes.setdefault(desc, set()).add(proc)
+            try:
+                yield func(*args, **kwargs)
+            finally:
+                proc.update_metrics()
+                _background_processes[desc].remove(proc)
+
+    with PreserveLoggingContext():
+        run()

From 92aecd557bdd4c6eb60bdb7afa449ac6ba0c4a0f Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 18 Jul 2018 11:29:48 +0100
Subject: [PATCH 050/830] changelog

---
 changelog.d/3553.feature | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3553.feature

diff --git a/changelog.d/3553.feature b/changelog.d/3553.feature
new file mode 100644
index 000000000000..77a294cb9f42
--- /dev/null
+++ b/changelog.d/3553.feature
@@ -0,0 +1 @@
+Add metrics to track resource usage by background processes

From e45a46b6e4990befe6567aaf26354320dce4337a Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 18 Jul 2018 13:59:36 +0100
Subject: [PATCH 051/830] Add response code to response timer metrics

---
 synapse/http/request_metrics.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py
index f24b4b949c88..c029a69ee54e 100644
--- a/synapse/http/request_metrics.py
+++ b/synapse/http/request_metrics.py
@@ -38,7 +38,7 @@
 )
 
 response_timer = Histogram(
-    "synapse_http_server_response_time_seconds", "sec", ["method", "servlet", "tag"]
+    "synapse_http_server_response_time_seconds", "sec", ["method", "servlet", "tag", "code"]
 )
 
 response_ru_utime = Counter(
@@ -171,11 +171,13 @@ def stop(self, time_sec, request):
                 )
                 return
 
-        outgoing_responses_counter.labels(request.method, str(request.code)).inc()
+        response_code = str(request.code)
+
+        outgoing_responses_counter.labels(request.method, response_code).inc()
 
         response_count.labels(request.method, self.name, tag).inc()
 
-        response_timer.labels(request.method, self.name, tag).observe(
+        response_timer.labels(request.method, self.name, tag, response_code).observe(
             time_sec - self.start
         )
 

From 00845c49d287a60100c73790225cc168801eb52f Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 18 Jul 2018 14:03:16 +0100
Subject: [PATCH 052/830] Newsfile

---
 changelog.d/3554.feature | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3554.feature

diff --git a/changelog.d/3554.feature b/changelog.d/3554.feature
new file mode 100644
index 000000000000..b00397872c4c
--- /dev/null
+++ b/changelog.d/3554.feature
@@ -0,0 +1 @@
+Add `code` label to `synapse_http_server_response_time_seconds` prometheus metric

From 5bd0a47fcdea7dada06dc5d42e4340c8748386e8 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 18 Jul 2018 14:19:00 +0100
Subject: [PATCH 053/830] pep8

---
 synapse/http/request_metrics.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py
index c029a69ee54e..588e280571e0 100644
--- a/synapse/http/request_metrics.py
+++ b/synapse/http/request_metrics.py
@@ -38,7 +38,8 @@
 )
 
 response_timer = Histogram(
-    "synapse_http_server_response_time_seconds", "sec", ["method", "servlet", "tag", "code"]
+    "synapse_http_server_response_time_seconds", "sec",
+    ["method", "servlet", "tag", "code"],
 )
 
 response_ru_utime = Counter(

From 8cb8df55e9bb5de27d9e6570441560eb81db36df Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 18 Jul 2018 15:22:02 +0100
Subject: [PATCH 054/830] Split MessageHandler into read only and writers

This will let us call the read only parts from workers, and so be able
to move some APIs off of master, e.g. the `/state` API.
---
 synapse/handlers/__init__.py    |   2 -
 synapse/handlers/message.py     | 281 +++++++++++++++++---------------
 synapse/rest/client/v1/admin.py |   8 +-
 synapse/rest/client/v1/room.py  |  20 +--
 synapse/server.py               |  14 +-
 5 files changed, 176 insertions(+), 149 deletions(-)

diff --git a/synapse/handlers/__init__.py b/synapse/handlers/__init__.py
index 4b9923d8c0ba..0bad3e0a2e2c 100644
--- a/synapse/handlers/__init__.py
+++ b/synapse/handlers/__init__.py
@@ -17,7 +17,6 @@
 from .directory import DirectoryHandler
 from .federation import FederationHandler
 from .identity import IdentityHandler
-from .message import MessageHandler
 from .register import RegistrationHandler
 from .room import RoomContextHandler
 from .search import SearchHandler
@@ -44,7 +43,6 @@ class Handlers(object):
 
     def __init__(self, hs):
         self.registration_handler = RegistrationHandler(hs)
-        self.message_handler = MessageHandler(hs)
         self.federation_handler = FederationHandler(hs)
         self.directory_handler = DirectoryHandler(hs)
         self.admin_handler = AdminHandler(hs)
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index a39b852cebfc..3c6f9860d53e 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -75,12 +75,159 @@ def asdict(self):
         }
 
 
-class MessageHandler(BaseHandler):
+class MessageHandler(object):
+    """Contains some read only APIs to get state about a room
+    """
 
     def __init__(self, hs):
-        super(MessageHandler, self).__init__(hs)
-        self.hs = hs
+        self.auth = hs.get_auth()
+        self.clock = hs.get_clock()
         self.state = hs.get_state_handler()
+        self.store = hs.get_datastore()
+
+    @defer.inlineCallbacks
+    def get_room_data(self, user_id=None, room_id=None,
+                      event_type=None, state_key="", is_guest=False):
+        """ Get data from a room.
+
+        Args:
+            event : The room path event
+        Returns:
+            The path data content.
+        Raises:
+            SynapseError if something went wrong.
+        """
+        membership, membership_event_id = yield self._check_in_room_or_world_readable(
+            room_id, user_id
+        )
+
+        if membership == Membership.JOIN:
+            data = yield self.state.get_current_state(
+                room_id, event_type, state_key
+            )
+        elif membership == Membership.LEAVE:
+            key = (event_type, state_key)
+            room_state = yield self.store.get_state_for_events(
+                [membership_event_id], [key]
+            )
+            data = room_state[membership_event_id].get(key)
+
+        defer.returnValue(data)
+
+    @defer.inlineCallbacks
+    def _check_in_room_or_world_readable(self, room_id, user_id):
+        try:
+            # check_user_was_in_room will return the most recent membership
+            # event for the user if:
+            #  * The user is a non-guest user, and was ever in the room
+            #  * The user is a guest user, and has joined the room
+            # else it will throw.
+            member_event = yield self.auth.check_user_was_in_room(room_id, user_id)
+            defer.returnValue((member_event.membership, member_event.event_id))
+            return
+        except AuthError:
+            visibility = yield self.state.get_current_state(
+                room_id, EventTypes.RoomHistoryVisibility, ""
+            )
+            if (
+                visibility and
+                visibility.content["history_visibility"] == "world_readable"
+            ):
+                defer.returnValue((Membership.JOIN, None))
+                return
+            raise AuthError(
+                403, "Guest access not allowed", errcode=Codes.GUEST_ACCESS_FORBIDDEN
+            )
+
+    @defer.inlineCallbacks
+    def get_state_events(self, user_id, room_id, is_guest=False):
+        """Retrieve all state events for a given room. If the user is
+        joined to the room then return the current state. If the user has
+        left the room return the state events from when they left.
+
+        Args:
+            user_id(str): The user requesting state events.
+            room_id(str): The room ID to get all state events from.
+        Returns:
+            A list of dicts representing state events. [{}, {}, {}]
+        """
+        membership, membership_event_id = yield self._check_in_room_or_world_readable(
+            room_id, user_id
+        )
+
+        if membership == Membership.JOIN:
+            room_state = yield self.state.get_current_state(room_id)
+        elif membership == Membership.LEAVE:
+            room_state = yield self.store.get_state_for_events(
+                [membership_event_id], None
+            )
+            room_state = room_state[membership_event_id]
+
+        now = self.clock.time_msec()
+        defer.returnValue(
+            [serialize_event(c, now) for c in room_state.values()]
+        )
+
+    @defer.inlineCallbacks
+    def get_joined_members(self, requester, room_id):
+        """Get all the joined members in the room and their profile information.
+
+        If the user has left the room return the state events from when they left.
+
+        Args:
+            requester(Requester): The user requesting state events.
+            room_id(str): The room ID to get all state events from.
+        Returns:
+            A dict of user_id to profile info
+        """
+        user_id = requester.user.to_string()
+        if not requester.app_service:
+            # We check AS auth after fetching the room membership, as it
+            # requires us to pull out all joined members anyway.
+            membership, _ = yield self._check_in_room_or_world_readable(
+                room_id, user_id
+            )
+            if membership != Membership.JOIN:
+                raise NotImplementedError(
+                    "Getting joined members after leaving is not implemented"
+                )
+
+        users_with_profile = yield self.state.get_current_user_in_room(room_id)
+
+        # If this is an AS, double check that they are allowed to see the members.
+        # This can either be because the AS user is in the room or because there
+        # is a user in the room that the AS is "interested in"
+        if requester.app_service and user_id not in users_with_profile:
+            for uid in users_with_profile:
+                if requester.app_service.is_interested_in_user(uid):
+                    break
+            else:
+                # Loop fell through, AS has no interested users in room
+                raise AuthError(403, "Appservice not in room")
+
+        defer.returnValue({
+            user_id: {
+                "avatar_url": profile.avatar_url,
+                "display_name": profile.display_name,
+            }
+            for user_id, profile in iteritems(users_with_profile)
+        })
+
+
+class PaginationHandler(MessageHandler):
+    """Handles pagination and purge history requests.
+
+    These are in the same handler due to the fact we need to block clients
+    paginating during a purge.
+
+    This subclasses MessageHandler to get at _check_in_room_or_world_readable
+    """
+
+    def __init__(self, hs):
+        super(PaginationHandler, self).__init__(hs)
+
+        self.hs = hs
+        self.store = hs.get_datastore()
         self.clock = hs.get_clock()
 
         self.pagination_lock = ReadWriteLock()
@@ -274,134 +421,6 @@ def get_messages(self, requester, room_id=None, pagin_config=None,
 
         defer.returnValue(chunk)
 
-    @defer.inlineCallbacks
-    def get_room_data(self, user_id=None, room_id=None,
-                      event_type=None, state_key="", is_guest=False):
-        """ Get data from a room.
-
-        Args:
-            event : The room path event
-        Returns:
-            The path data content.
-        Raises:
-            SynapseError if something went wrong.
-        """
-        membership, membership_event_id = yield self._check_in_room_or_world_readable(
-            room_id, user_id
-        )
-
-        if membership == Membership.JOIN:
-            data = yield self.state_handler.get_current_state(
-                room_id, event_type, state_key
-            )
-        elif membership == Membership.LEAVE:
-            key = (event_type, state_key)
-            room_state = yield self.store.get_state_for_events(
-                [membership_event_id], [key]
-            )
-            data = room_state[membership_event_id].get(key)
-
-        defer.returnValue(data)
-
-    @defer.inlineCallbacks
-    def _check_in_room_or_world_readable(self, room_id, user_id):
-        try:
-            # check_user_was_in_room will return the most recent membership
-            # event for the user if:
-            #  * The user is a non-guest user, and was ever in the room
-            #  * The user is a guest user, and has joined the room
-            # else it will throw.
-            member_event = yield self.auth.check_user_was_in_room(room_id, user_id)
-            defer.returnValue((member_event.membership, member_event.event_id))
-            return
-        except AuthError:
-            visibility = yield self.state_handler.get_current_state(
-                room_id, EventTypes.RoomHistoryVisibility, ""
-            )
-            if (
-                visibility and
-                visibility.content["history_visibility"] == "world_readable"
-            ):
-                defer.returnValue((Membership.JOIN, None))
-                return
-            raise AuthError(
-                403, "Guest access not allowed", errcode=Codes.GUEST_ACCESS_FORBIDDEN
-            )
-
-    @defer.inlineCallbacks
-    def get_state_events(self, user_id, room_id, is_guest=False):
-        """Retrieve all state events for a given room. If the user is
-        joined to the room then return the current state. If the user has
-        left the room return the state events from when they left.
-
-        Args:
-            user_id(str): The user requesting state events.
-            room_id(str): The room ID to get all state events from.
-        Returns:
-            A list of dicts representing state events. [{}, {}, {}]
-        """
-        membership, membership_event_id = yield self._check_in_room_or_world_readable(
-            room_id, user_id
-        )
-
-        if membership == Membership.JOIN:
-            room_state = yield self.state_handler.get_current_state(room_id)
-        elif membership == Membership.LEAVE:
-            room_state = yield self.store.get_state_for_events(
-                [membership_event_id], None
-            )
-            room_state = room_state[membership_event_id]
-
-        now = self.clock.time_msec()
-        defer.returnValue(
-            [serialize_event(c, now) for c in room_state.values()]
-        )
-
-    @defer.inlineCallbacks
-    def get_joined_members(self, requester, room_id):
-        """Get all the joined members in the room and their profile information.
-
-        If the user has left the room return the state events from when they left.
-
-        Args:
-            requester(Requester): The user requesting state events.
-            room_id(str): The room ID to get all state events from.
-        Returns:
-            A dict of user_id to profile info
-        """
-        user_id = requester.user.to_string()
-        if not requester.app_service:
-            # We check AS auth after fetching the room membership, as it
-            # requires us to pull out all joined members anyway.
-            membership, _ = yield self._check_in_room_or_world_readable(
-                room_id, user_id
-            )
-            if membership != Membership.JOIN:
-                raise NotImplementedError(
-                    "Getting joined members after leaving is not implemented"
-                )
-
-        users_with_profile = yield self.state.get_current_user_in_room(room_id)
-
-        # If this is an AS, double check that they are allowed to see the members.
-        # This can either be because the AS user is in the room or because there
-        # is a user in the room that the AS is "interested in"
-        if requester.app_service and user_id not in users_with_profile:
-            for uid in users_with_profile:
-                if requester.app_service.is_interested_in_user(uid):
-                    break
-            else:
-                # Loop fell through, AS has no interested users in room
-                raise AuthError(403, "Appservice not in room")
-
-        defer.returnValue({
-            user_id: {
-                "avatar_url": profile.avatar_url,
-                "display_name": profile.display_name,
-            }
-            for user_id, profile in iteritems(users_with_profile)
-        })
-
 
 class EventCreationHandler(object):
     def __init__(self, hs):
diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py
index 2dc50e582b14..13fd63a5b24a 100644
--- a/synapse/rest/client/v1/admin.py
+++ b/synapse/rest/client/v1/admin.py
@@ -123,7 +123,7 @@ def __init__(self, hs):
             hs (synapse.server.HomeServer)
         """
         super(PurgeHistoryRestServlet, self).__init__(hs)
-        self.handlers = hs.get_handlers()
+        self.pagination_handler = hs.get_pagination_handler()
         self.store = hs.get_datastore()
 
     @defer.inlineCallbacks
@@ -198,7 +198,7 @@ def on_POST(self, request, room_id, event_id):
                 errcode=Codes.BAD_JSON,
             )
 
-        purge_id = yield self.handlers.message_handler.start_purge_history(
+        purge_id = yield self.pagination_handler.start_purge_history(
             room_id, token,
             delete_local_events=delete_local_events,
         )
@@ -220,7 +220,7 @@ def __init__(self, hs):
             hs (synapse.server.HomeServer)
         """
         super(PurgeHistoryStatusRestServlet, self).__init__(hs)
-        self.handlers = hs.get_handlers()
+        self.pagination_handler = hs.get_pagination_handler()
 
     @defer.inlineCallbacks
     def on_GET(self, request, purge_id):
@@ -230,7 +230,7 @@ def on_GET(self, request, purge_id):
         if not is_admin:
             raise AuthError(403, "You are not a server admin")
 
-        purge_status = self.handlers.message_handler.get_purge_status(purge_id)
+        purge_status = self.pagination_handler.get_purge_status(purge_id)
         if purge_status is None:
             raise NotFoundError("purge id '%s' not found" % purge_id)
 
diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py
index 3d6244785427..8b6be9da9672 100644
--- a/synapse/rest/client/v1/room.py
+++ b/synapse/rest/client/v1/room.py
@@ -90,6 +90,7 @@ def __init__(self, hs):
         self.handlers = hs.get_handlers()
         self.event_creation_hander = hs.get_event_creation_handler()
         self.room_member_handler = hs.get_room_member_handler()
+        self.message_handler = hs.get_message_handler()
 
     def register(self, http_server):
         # /room/$roomid/state/$eventtype
@@ -124,7 +125,7 @@ def on_GET(self, request, room_id, event_type, state_key):
         format = parse_string(request, "format", default="content",
                               allowed_values=["content", "event"])
 
-        msg_handler = self.handlers.message_handler
+        msg_handler = self.message_handler
         data = yield msg_handler.get_room_data(
             user_id=requester.user.to_string(),
             room_id=room_id,
@@ -377,14 +378,13 @@ class RoomMemberListRestServlet(ClientV1RestServlet):
 
     def __init__(self, hs):
         super(RoomMemberListRestServlet, self).__init__(hs)
-        self.handlers = hs.get_handlers()
+        self.message_handler = hs.get_message_handler()
 
     @defer.inlineCallbacks
     def on_GET(self, request, room_id):
         # TODO support Pagination stream API (limit/tokens)
         requester = yield self.auth.get_user_by_req(request)
-        handler = self.handlers.message_handler
-        events = yield handler.get_state_events(
+        events = yield self.message_handler.get_state_events(
             room_id=room_id,
             user_id=requester.user.to_string(),
         )
@@ -406,7 +406,7 @@ class JoinedRoomMemberListRestServlet(ClientV1RestServlet):
 
     def __init__(self, hs):
         super(JoinedRoomMemberListRestServlet, self).__init__(hs)
-        self.message_handler = hs.get_handlers().message_handler
+        self.message_handler = hs.get_message_handler()
 
     @defer.inlineCallbacks
     def on_GET(self, request, room_id):
@@ -427,7 +427,7 @@ class RoomMessageListRestServlet(ClientV1RestServlet):
 
     def __init__(self, hs):
         super(RoomMessageListRestServlet, self).__init__(hs)
-        self.handlers = hs.get_handlers()
+        self.pagination_handler = hs.get_pagination_handler()
 
     @defer.inlineCallbacks
     def on_GET(self, request, room_id):
@@ -442,8 +442,7 @@ def on_GET(self, request, room_id):
             event_filter = Filter(json.loads(filter_json))
         else:
             event_filter = None
-        handler = self.handlers.message_handler
-        msgs = yield handler.get_messages(
+        msgs = yield self.pagination_handler.get_messages(
             room_id=room_id,
             requester=requester,
             pagin_config=pagination_config,
@@ -460,14 +459,13 @@ class RoomStateRestServlet(ClientV1RestServlet):
 
     def __init__(self, hs):
         super(RoomStateRestServlet, self).__init__(hs)
-        self.handlers = hs.get_handlers()
+        self.message_handler = hs.get_message_handler()
 
     @defer.inlineCallbacks
     def on_GET(self, request, room_id):
         requester = yield self.auth.get_user_by_req(request, allow_guest=True)
-        handler = self.handlers.message_handler
         # Get all the current state for this room
-        events = yield handler.get_state_events(
+        events = yield self.message_handler.get_state_events(
             room_id=room_id,
             user_id=requester.user.to_string(),
             is_guest=requester.is_guest,
diff --git a/synapse/server.py b/synapse/server.py
index 92bea96c5c01..b93bd6d7d952 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -52,7 +52,11 @@
 from synapse.handlers.events import EventHandler, EventStreamHandler
 from synapse.handlers.groups_local import GroupsLocalHandler
 from synapse.handlers.initial_sync import InitialSyncHandler
-from synapse.handlers.message import EventCreationHandler
+from synapse.handlers.message import (
+    EventCreationHandler,
+    MessageHandler,
+    PaginationHandler,
+)
 from synapse.handlers.presence import PresenceHandler
 from synapse.handlers.profile import ProfileHandler
 from synapse.handlers.read_marker import ReadMarkerHandler
@@ -163,6 +167,8 @@ def build_DEPENDENCY(self)
         'federation_registry',
         'server_notices_manager',
         'server_notices_sender',
+        'message_handler',
+        'pagination_handler',
     ]
 
     def __init__(self, hostname, reactor=None, **kwargs):
@@ -426,6 +432,12 @@ def build_server_notices_sender(self):
             return WorkerServerNoticesSender(self)
         return ServerNoticesSender(self)
 
+    def build_message_handler(self):
+        return MessageHandler(self)
+
+    def build_pagination_handler(self):
+        return PaginationHandler(self)
+
     def remove_pusher(self, app_id, push_key, user_id):
         return self.get_pusherpool().remove_pusher(app_id, push_key, user_id)
 

From bacdf0cbf9fdbf9bbab2420b86308830ac4e4592 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 18 Jul 2018 15:29:45 +0100
Subject: [PATCH 055/830] Move RoomContextHandler out of Handlers

This is in preparation for moving GET /context/ to a worker
---
 synapse/handlers/__init__.py   | 2 --
 synapse/handlers/room.py       | 6 +++++-
 synapse/rest/client/v1/room.py | 4 ++--
 synapse/server.py              | 6 +++++-
 4 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/synapse/handlers/__init__.py b/synapse/handlers/__init__.py
index 0bad3e0a2e2c..413425fed1ea 100644
--- a/synapse/handlers/__init__.py
+++ b/synapse/handlers/__init__.py
@@ -18,7 +18,6 @@
 from .federation import FederationHandler
 from .identity import IdentityHandler
 from .register import RegistrationHandler
-from .room import RoomContextHandler
 from .search import SearchHandler
 
 
@@ -48,4 +47,3 @@ def __init__(self, hs):
         self.admin_handler = AdminHandler(hs)
         self.identity_handler = IdentityHandler(hs)
         self.search_handler = SearchHandler(hs)
-        self.room_context_handler = RoomContextHandler(hs)
diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index f67512078ba9..6150b7e22650 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -395,7 +395,11 @@ def send(etype, content, **kwargs):
             )
 
 
-class RoomContextHandler(BaseHandler):
+class RoomContextHandler(object):
+    def __init__(self, hs):
+        self.hs = hs
+        self.store = hs.get_datastore()
+
     @defer.inlineCallbacks
     def get_event_context(self, user, room_id, event_id, limit):
         """Retrieves events, pagination tokens and state around a given event
diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py
index 8b6be9da9672..2c9459534e46 100644
--- a/synapse/rest/client/v1/room.py
+++ b/synapse/rest/client/v1/room.py
@@ -523,7 +523,7 @@ class RoomEventContextServlet(ClientV1RestServlet):
     def __init__(self, hs):
         super(RoomEventContextServlet, self).__init__(hs)
         self.clock = hs.get_clock()
-        self.handlers = hs.get_handlers()
+        self.room_context_handler = hs.get_room_context_handler()
 
     @defer.inlineCallbacks
     def on_GET(self, request, room_id, event_id):
@@ -531,7 +531,7 @@ def on_GET(self, request, room_id, event_id):
 
         limit = parse_integer(request, "limit", default=10)
 
-        results = yield self.handlers.room_context_handler.get_event_context(
+        results = yield self.room_context_handler.get_event_context(
             requester.user,
             room_id,
             event_id,
diff --git a/synapse/server.py b/synapse/server.py
index b93bd6d7d952..a24ea158df4b 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -61,7 +61,7 @@
 from synapse.handlers.profile import ProfileHandler
 from synapse.handlers.read_marker import ReadMarkerHandler
 from synapse.handlers.receipts import ReceiptsHandler
-from synapse.handlers.room import RoomCreationHandler
+from synapse.handlers.room import RoomContextHandler, RoomCreationHandler
 from synapse.handlers.room_list import RoomListHandler
 from synapse.handlers.room_member import RoomMemberMasterHandler
 from synapse.handlers.room_member_worker import RoomMemberWorkerHandler
@@ -169,6 +169,7 @@ def build_DEPENDENCY(self)
         'server_notices_sender',
         'message_handler',
         'pagination_handler',
+        'room_context_handler',
     ]
 
     def __init__(self, hostname, reactor=None, **kwargs):
@@ -438,6 +439,9 @@ def build_message_handler(self):
     def build_pagination_handler(self):
         return PaginationHandler(self)
 
+    def build_room_context_handler(self):
+        return RoomContextHandler(self)
+
     def remove_pusher(self, app_id, push_key, user_id):
         return self.get_pusherpool().remove_pusher(app_id, push_key, user_id)
 

From 9b596177aeb12211e86cce27aa409bc596909ed0 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 18 Jul 2018 15:30:43 +0100
Subject: [PATCH 056/830] Add some room read only APIs to client_reader

---
 synapse/app/client_reader.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/synapse/app/client_reader.py b/synapse/app/client_reader.py
index b0ea26dcb41a..398bb36602dc 100644
--- a/synapse/app/client_reader.py
+++ b/synapse/app/client_reader.py
@@ -40,7 +40,13 @@
 from synapse.replication.slave.storage.room import RoomStore
 from synapse.replication.slave.storage.transactions import TransactionStore
 from synapse.replication.tcp.client import ReplicationClientHandler
-from synapse.rest.client.v1.room import PublicRoomListRestServlet
+from synapse.rest.client.v1.room import (
+    JoinedRoomMemberListRestServlet,
+    PublicRoomListRestServlet,
+    RoomEventContextServlet,
+    RoomMemberListRestServlet,
+    RoomStateRestServlet,
+)
 from synapse.server import HomeServer
 from synapse.storage.engines import create_engine
 from synapse.util.httpresourcetree import create_resource_tree
@@ -82,7 +88,13 @@ def _listen_http(self, listener_config):
                     resources[METRICS_PREFIX] = MetricsResource(RegistryProxy)
                 elif name == "client":
                     resource = JsonResource(self, canonical_json=False)
+
                     PublicRoomListRestServlet(self).register(resource)
+                    RoomMemberListRestServlet(self).register(resource)
+                    JoinedRoomMemberListRestServlet(self).register(resource)
+                    RoomStateRestServlet(self).register(resource)
+                    RoomEventContextServlet(self).register(resource)
+
                     resources.update({
                         "/_matrix/client/r0": resource,
                         "/_matrix/client/unstable": resource,

From 3a993a660d0d05578c189591980a9383812d7627 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 18 Jul 2018 15:39:49 +0100
Subject: [PATCH 057/830] Newsfile

---
 changelog.d/3555.feature | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3555.feature

diff --git a/changelog.d/3555.feature b/changelog.d/3555.feature
new file mode 100644
index 000000000000..ea4a85e0aee7
--- /dev/null
+++ b/changelog.d/3555.feature
@@ -0,0 +1 @@
+Add support for client_reader to handle more APIs

From 667fba68f3ca808f48143a2a739a54665b0162c6 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 18 Jul 2018 14:35:24 +0100
Subject: [PATCH 058/830] Run things as background processes

This fixes #3518, and ensures that we get useful logs and metrics for lots of
things that happen in the background.

(There are certainly more things that happen in the background; these are just
the common ones I've found running a single-process synapse locally).
---
 synapse/federation/transaction_queue.py | 15 ++++++---------
 synapse/storage/background_updates.py   | 10 ++++++++--
 synapse/storage/client_ips.py           | 15 +++++++++++----
 synapse/storage/events.py               | 10 ++++------
 synapse/storage/events_worker.py        | 10 ++++++----
 synapse/util/caches/expiringcache.py    |  6 +++++-
 synapse/util/distributor.py             |  4 ++++
 7 files changed, 44 insertions(+), 26 deletions(-)

diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py
index 5c5a73b73c5f..6996d6b695b9 100644
--- a/synapse/federation/transaction_queue.py
+++ b/synapse/federation/transaction_queue.py
@@ -168,7 +168,7 @@ def notify_new_events(self, current_id):
 
         # fire off a processing loop in the background
         run_as_background_process(
-            "process_transaction_queue",
+            "process_event_queue_for_federation",
             self._process_event_queue_loop,
         )
 
@@ -434,14 +434,11 @@ def _attempt_new_transaction(self, destination):
 
         logger.debug("TX [%s] Starting transaction loop", destination)
 
-        # Drop the logcontext before starting the transaction. It doesn't
-        # really make sense to log all the outbound transactions against
-        # whatever path led us to this point: that's pretty arbitrary really.
-        #
-        # (this also means we can fire off _perform_transaction without
-        # yielding)
-        with logcontext.PreserveLoggingContext():
-            self._transaction_transmission_loop(destination)
+        run_as_background_process(
+            "federation_transaction_transmission_loop",
+            self._transaction_transmission_loop,
+            destination,
+        )
 
     @defer.inlineCallbacks
     def _transaction_transmission_loop(self, destination):
diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py
index dc9eca7d153d..5fe1ca2de7f6 100644
--- a/synapse/storage/background_updates.py
+++ b/synapse/storage/background_updates.py
@@ -19,6 +19,8 @@
 
 from twisted.internet import defer
 
+from synapse.metrics.background_process_metrics import run_as_background_process
+
 from . import engines
 from ._base import SQLBaseStore
 
@@ -87,10 +89,14 @@ def __init__(self, db_conn, hs):
         self._background_update_handlers = {}
         self._all_done = False
 
-    @defer.inlineCallbacks
     def start_doing_background_updates(self):
-        logger.info("Starting background schema updates")
+        run_as_background_process(
+            "background_updates", self._run_background_updates,
+        )
 
+    @defer.inlineCallbacks
+    def _run_background_updates(self):
+        logger.info("Starting background schema updates")
         while True:
             yield self.hs.get_clock().sleep(
                 self.BACKGROUND_UPDATE_INTERVAL_MS / 1000.)
diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py
index b78eda3413f5..77ae10da3d7f 100644
--- a/synapse/storage/client_ips.py
+++ b/synapse/storage/client_ips.py
@@ -19,6 +19,7 @@
 
 from twisted.internet import defer
 
+from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.util.caches import CACHE_SIZE_FACTOR
 
 from . import background_updates
@@ -93,10 +94,16 @@ def insert_client_ip(self, user_id, access_token, ip, user_agent, device_id,
         self._batch_row_update[key] = (user_agent, device_id, now)
 
     def _update_client_ips_batch(self):
-        to_update = self._batch_row_update
-        self._batch_row_update = {}
-        return self.runInteraction(
-            "_update_client_ips_batch", self._update_client_ips_batch_txn, to_update
+        def update():
+            to_update = self._batch_row_update
+            self._batch_row_update = {}
+            return self.runInteraction(
+                "_update_client_ips_batch", self._update_client_ips_batch_txn,
+                to_update,
+            )
+
+        run_as_background_process(
+            "update_client_ips", update,
         )
 
     def _update_client_ips_batch_txn(self, txn, to_update):
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 2aaab0d02c7f..4ff0fdc4abab 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -33,12 +33,13 @@
 # these are only included to make the type annotations work
 from synapse.events import EventBase  # noqa: F401
 from synapse.events.snapshot import EventContext  # noqa: F401
+from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.storage.events_worker import EventsWorkerStore
 from synapse.types import RoomStreamToken, get_domain_from_id
 from synapse.util.async import ObservableDeferred
 from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
 from synapse.util.frozenutils import frozendict_json_encoder
-from synapse.util.logcontext import PreserveLoggingContext, make_deferred_yieldable
+from synapse.util.logcontext import make_deferred_yieldable
 from synapse.util.logutils import log_function
 from synapse.util.metrics import Measure
 
@@ -155,11 +156,8 @@ def handle_queue_loop():
                     self._event_persist_queues[room_id] = queue
                 self._currently_persisting_rooms.discard(room_id)
 
-        # set handle_queue_loop off on the background. We don't want to
-        # attribute work done in it to the current request, so we drop the
-        # logcontext altogether.
-        with PreserveLoggingContext():
-            handle_queue_loop()
+        # set handle_queue_loop off in the background
+        run_as_background_process("persist_events", handle_queue_loop)
 
     def _get_drainining_queue(self, room_id):
         queue = self._event_persist_queues.setdefault(room_id, deque())
diff --git a/synapse/storage/events_worker.py b/synapse/storage/events_worker.py
index 67433606c647..f28239a808ee 100644
--- a/synapse/storage/events_worker.py
+++ b/synapse/storage/events_worker.py
@@ -25,6 +25,7 @@
 from synapse.events import FrozenEvent
 from synapse.events.snapshot import EventContext  # noqa: F401
 from synapse.events.utils import prune_event
+from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.util.logcontext import (
     LoggingContext,
     PreserveLoggingContext,
@@ -322,10 +323,11 @@ def _enqueue_events(self, events, check_redacted=True, allow_rejected=False):
                 should_start = False
 
         if should_start:
-            with PreserveLoggingContext():
-                self.runWithConnection(
-                    self._do_fetch
-                )
+            run_as_background_process(
+                "fetch_events",
+                self.runWithConnection,
+                self._do_fetch,
+            )
 
         logger.debug("Loading %d events", len(events))
         with PreserveLoggingContext():
diff --git a/synapse/util/caches/expiringcache.py b/synapse/util/caches/expiringcache.py
index 4abca91f6d52..465adc54a8e6 100644
--- a/synapse/util/caches/expiringcache.py
+++ b/synapse/util/caches/expiringcache.py
@@ -16,6 +16,7 @@
 import logging
 from collections import OrderedDict
 
+from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.util.caches import register_cache
 
 logger = logging.getLogger(__name__)
@@ -63,7 +64,10 @@ def start(self):
             return
 
         def f():
-            self._prune_cache()
+            run_as_background_process(
+                "prune_cache_%s" % self._cache_name,
+                self._prune_cache,
+            )
 
         self._clock.looping_call(f, self._expiry_ms / 2)
 
diff --git a/synapse/util/distributor.py b/synapse/util/distributor.py
index 734331caaae6..d91ae400eb9b 100644
--- a/synapse/util/distributor.py
+++ b/synapse/util/distributor.py
@@ -75,6 +75,10 @@ def observe(self, name, observer):
             self.pre_registration[name].append(observer)
 
     def fire(self, name, *args, **kwargs):
+        """Dispatches the given signal to the registered observers.
+
+        Runs the observers as a background process. Does not return a deferred.
+        """
         if name not in self.signals:
             raise KeyError("%r does not have a signal named %s" % (self, name))
 

From 08436c556a6dd6053ce778d581159f82f8668a63 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 18 Jul 2018 16:14:01 +0100
Subject: [PATCH 059/830] changelog

---
 changelog.d/3556.feature | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3556.feature

diff --git a/changelog.d/3556.feature b/changelog.d/3556.feature
new file mode 100644
index 000000000000..77a294cb9f42
--- /dev/null
+++ b/changelog.d/3556.feature
@@ -0,0 +1 @@
+Add metrics to track resource usage by background processes

From 8c69b735e3186875a8bf676241f990470fb7c592 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 18 Jul 2018 15:33:13 +0100
Subject: [PATCH 060/830] Make Distributor run its processes as a background
 process

This is more involved than it might otherwise be, because the current
implementation just drops its logcontexts and runs everything in the sentinel
context.

It turns out that we aren't actually using a bunch of the functionality here
(notably suppress_failures and the fact that Distributor.fire returns a
deferred), so the easiest way to fix this is actually by simplifying a bunch of
code.
---
 synapse/util/distributor.py | 44 ++++++++++++-----------------
 tests/test_distributor.py   | 56 +++----------------------------------
 2 files changed, 22 insertions(+), 78 deletions(-)

diff --git a/synapse/util/distributor.py b/synapse/util/distributor.py
index d91ae400eb9b..194da87639f3 100644
--- a/synapse/util/distributor.py
+++ b/synapse/util/distributor.py
@@ -17,20 +17,18 @@
 
 from twisted.internet import defer
 
-from synapse.util import unwrapFirstError
-from synapse.util.logcontext import PreserveLoggingContext
+from synapse.metrics.background_process_metrics import run_as_background_process
+from synapse.util.logcontext import make_deferred_yieldable, run_in_background
 
 logger = logging.getLogger(__name__)
 
 
 def user_left_room(distributor, user, room_id):
-    with PreserveLoggingContext():
-        distributor.fire("user_left_room", user=user, room_id=room_id)
+    distributor.fire("user_left_room", user=user, room_id=room_id)
 
 
 def user_joined_room(distributor, user, room_id):
-    with PreserveLoggingContext():
-        distributor.fire("user_joined_room", user=user, room_id=room_id)
+    distributor.fire("user_joined_room", user=user, room_id=room_id)
 
 
 class Distributor(object):
@@ -44,9 +42,7 @@ class Distributor(object):
       model will do for today.
     """
 
-    def __init__(self, suppress_failures=True):
-        self.suppress_failures = suppress_failures
-
+    def __init__(self):
         self.signals = {}
         self.pre_registration = {}
 
@@ -56,7 +52,6 @@ def declare(self, name):
 
         self.signals[name] = Signal(
             name,
-            suppress_failures=self.suppress_failures,
         )
 
         if name in self.pre_registration:
@@ -82,7 +77,11 @@ def fire(self, name, *args, **kwargs):
         if name not in self.signals:
             raise KeyError("%r does not have a signal named %s" % (self, name))
 
-        return self.signals[name].fire(*args, **kwargs)
+        run_as_background_process(
+            name,
+            self.signals[name].fire,
+            *args, **kwargs
+        )
 
 
 class Signal(object):
@@ -95,9 +94,8 @@ class Signal(object):
     method into all of the observers.
     """
 
-    def __init__(self, name, suppress_failures):
+    def __init__(self, name):
         self.name = name
-        self.suppress_failures = suppress_failures
         self.observers = []
 
     def observe(self, observer):
@@ -107,7 +105,6 @@ def observe(self, observer):
         Each observer callable may return a Deferred."""
         self.observers.append(observer)
 
-    @defer.inlineCallbacks
     def fire(self, *args, **kwargs):
         """Invokes every callable in the observer list, passing in the args and
         kwargs. Exceptions thrown by observers are logged but ignored. It is
@@ -125,22 +122,17 @@ def eb(failure):
                         failure.type,
                         failure.value,
                         failure.getTracebackObject()))
-                if not self.suppress_failures:
-                    return failure
 
             return defer.maybeDeferred(observer, *args, **kwargs).addErrback(eb)
 
-        with PreserveLoggingContext():
-            deferreds = [
-                do(observer)
-                for observer in self.observers
-            ]
-
-            res = yield defer.gatherResults(
-                deferreds, consumeErrors=True
-            ).addErrback(unwrapFirstError)
+        deferreds = [
+            run_in_background(do, o)
+            for o in self.observers
+        ]
 
-        defer.returnValue(res)
+        return make_deferred_yieldable(defer.gatherResults(
+            deferreds, consumeErrors=True,
+        ))
 
     def __repr__(self):
         return "<Signal name=%r>" % (self.name,)
diff --git a/tests/test_distributor.py b/tests/test_distributor.py
index 04a88056f190..71d11cda77c9 100644
--- a/tests/test_distributor.py
+++ b/tests/test_distributor.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,8 +16,6 @@
 
 from mock import Mock, patch
 
-from twisted.internet import defer
-
 from synapse.util.distributor import Distributor
 
 from . import unittest
@@ -27,38 +26,15 @@ class DistributorTestCase(unittest.TestCase):
     def setUp(self):
         self.dist = Distributor()
 
-    @defer.inlineCallbacks
     def test_signal_dispatch(self):
         self.dist.declare("alert")
 
         observer = Mock()
         self.dist.observe("alert", observer)
 
-        d = self.dist.fire("alert", 1, 2, 3)
-        yield d
-        self.assertTrue(d.called)
+        self.dist.fire("alert", 1, 2, 3)
         observer.assert_called_with(1, 2, 3)
 
-    @defer.inlineCallbacks
-    def test_signal_dispatch_deferred(self):
-        self.dist.declare("whine")
-
-        d_inner = defer.Deferred()
-
-        def observer():
-            return d_inner
-
-        self.dist.observe("whine", observer)
-
-        d_outer = self.dist.fire("whine")
-
-        self.assertFalse(d_outer.called)
-
-        d_inner.callback(None)
-        yield d_outer
-        self.assertTrue(d_outer.called)
-
-    @defer.inlineCallbacks
     def test_signal_catch(self):
         self.dist.declare("alarm")
 
@@ -71,9 +47,7 @@ def test_signal_catch(self):
         with patch(
             "synapse.util.distributor.logger", spec=["warning"]
         ) as mock_logger:
-            d = self.dist.fire("alarm", "Go")
-            yield d
-            self.assertTrue(d.called)
+            self.dist.fire("alarm", "Go")
 
             observers[0].assert_called_once_with("Go")
             observers[1].assert_called_once_with("Go")
@@ -83,34 +57,12 @@ def test_signal_catch(self):
                 mock_logger.warning.call_args[0][0], str
             )
 
-    @defer.inlineCallbacks
-    def test_signal_catch_no_suppress(self):
-        # Gut-wrenching
-        self.dist.suppress_failures = False
-
-        self.dist.declare("whail")
-
-        class MyException(Exception):
-            pass
-
-        @defer.inlineCallbacks
-        def observer():
-            raise MyException("Oopsie")
-
-        self.dist.observe("whail", observer)
-
-        d = self.dist.fire("whail")
-
-        yield self.assertFailure(d, MyException)
-        self.dist.suppress_failures = True
-
-    @defer.inlineCallbacks
     def test_signal_prereg(self):
         observer = Mock()
         self.dist.observe("flare", observer)
 
         self.dist.declare("flare")
-        yield self.dist.fire("flare", 4, 5)
+        self.dist.fire("flare", 4, 5)
 
         observer.assert_called_with(4, 5)
 

From 3f9e649f1720b3419fd8008771955714baa9600b Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 18 Jul 2018 20:55:37 +0100
Subject: [PATCH 061/830] add config for pep8

Since, for better or worse, we seem to have configured isort to generate
89-character lines, pycharm is now complaining at me that our lines are too
long.

So, let's configure pep8 to behave consistently with flake8.
---
 setup.cfg | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/setup.cfg b/setup.cfg
index 9b5b75cd6076..3f84283a3810 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -14,12 +14,17 @@ ignore =
     pylint.cfg
     tox.ini
 
-[flake8]
+[pep8]
 max-line-length = 90
-#  W503 requires that binary operators be at the end, not start, of lines. Erik doesn't like it.
-#  E203 is contrary to PEP8.
+#  W503 requires that binary operators be at the end, not start, of lines. Erik
+#  doesn't like it.  E203 is contrary to PEP8.
 ignore = W503,E203
 
+[flake8]
+# note that flake8 inherits the "ignore" settings from "pep8" (because it uses
+# pep8 to do those checks), but not the "max-line-length" setting
+max-line-length = 90
+
 [isort]
 line_length = 89
 not_skip = __init__.py

From eed24893fabb95b9ebbda58db54831f17dbdf2e6 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 18 Jul 2018 22:12:19 +0100
Subject: [PATCH 062/830] changelog

---
 changelog.d/3559.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3559.misc

diff --git a/changelog.d/3559.misc b/changelog.d/3559.misc
new file mode 100644
index 000000000000..26df859e4563
--- /dev/null
+++ b/changelog.d/3559.misc
@@ -0,0 +1 @@
+add config for pep8

From f1dd89fe8662be52bd4012b36b402f9380dac4e5 Mon Sep 17 00:00:00 2001
From: Benedikt Heine <bebe@bebehei.de>
Date: Tue, 17 Jul 2018 09:07:03 +0200
Subject: [PATCH 063/830] [Docker] Use sorted multiline package lists

This matches docker best practices.

Signed-off-by: Benedikt Heine <bebe@bebehei.de>
---
 Dockerfile            | 22 +++++++++++++++++++---
 changelog.d/3543.misc |  1 +
 2 files changed, 20 insertions(+), 3 deletions(-)
 create mode 100644 changelog.d/3543.misc

diff --git a/Dockerfile b/Dockerfile
index 565341fee36f..0242be5f6868 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,16 +1,32 @@
 FROM docker.io/python:2-alpine3.7
 
-RUN apk add --no-cache --virtual .nacl_deps su-exec build-base libffi-dev zlib-dev libressl-dev libjpeg-turbo-dev linux-headers postgresql-dev libxslt-dev
+RUN apk add --no-cache --virtual .nacl_deps \
+        build-base \
+        libffi-dev \
+        libjpeg-turbo-dev \
+        libressl-dev \
+        libxslt-dev \
+        linux-headers \
+        postgresql-dev \
+        su-exec \
+        zlib-dev
 
 COPY . /synapse
 
 # A wheel cache may be provided in ./cache for faster build
 RUN cd /synapse \
- && pip install --upgrade pip setuptools psycopg2 lxml \
+ && pip install --upgrade \
+        lxml \
+        pip \
+        psycopg2 \
+        setuptools \
  && mkdir -p /synapse/cache \
  && pip install -f /synapse/cache --upgrade --process-dependency-links . \
  && mv /synapse/contrib/docker/start.py /synapse/contrib/docker/conf / \
- && rm -rf setup.py setup.cfg synapse
+ && rm -rf \
+        setup.cfg \
+        setup.py \
+        synapse
 
 VOLUME ["/data"]
 
diff --git a/changelog.d/3543.misc b/changelog.d/3543.misc
new file mode 100644
index 000000000000..d231d17749b5
--- /dev/null
+++ b/changelog.d/3543.misc
@@ -0,0 +1 @@
+Improve Dockerfile and docker-compose instructions

From c0685f67c001ab156fb6922877c35f70536100dc Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Thu, 19 Jul 2018 10:59:02 +0100
Subject: [PATCH 064/830] spell out that include_deleted_devices requires
 include_all_devices

---
 synapse/storage/end_to_end_keys.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/synapse/storage/end_to_end_keys.py b/synapse/storage/end_to_end_keys.py
index 6c287194206f..ffe4d7235ab8 100644
--- a/synapse/storage/end_to_end_keys.py
+++ b/synapse/storage/end_to_end_keys.py
@@ -74,7 +74,8 @@ def get_e2e_device_keys(
             include_all_devices (bool): whether to include entries for devices
                 that don't have device keys
             include_deleted_devices (bool): whether to include null entries for
-                devices which no longer exist (but were in the query_list)
+                devices which no longer exist (but were in the query_list).
+                This option only takes effect if include_all_devices is true.
         Returns:
             Dict mapping from user-id to dict mapping from device_id to
             dict containing "key_json", "device_display_name".
@@ -100,6 +101,9 @@ def _get_e2e_device_keys_txn(
         query_clauses = []
         query_params = []
 
+        if include_all_devices is False:
+            include_deleted_devices = False
+
         if include_deleted_devices:
             deleted_devices = set(query_list)
 

From a97c845271f9a89ebdb7186d4c9d04c099bd1beb Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Thu, 19 Jul 2018 20:03:33 +1000
Subject: [PATCH 065/830] Move v1-only APIs into their own module & isolate
 deprecated ones (#3460)

---
 changelog.d/3460.misc                         |  0
 setup.cfg                                     |  1 +
 synapse/http/client.py                        |  6 +-
 synapse/rest/__init__.py                      | 43 +++++++--
 synapse/rest/client/v1/room.py                |  5 +-
 synapse/rest/client/v1_only/__init__.py       |  3 +
 synapse/rest/client/v1_only/base.py           | 39 ++++++++
 .../rest/client/{v1 => v1_only}/register.py   |  7 +-
 tests/rest/client/v1/test_events.py           | 90 +++----------------
 tests/rest/client/v1/test_register.py         |  5 +-
 tests/rest/client/v1/test_rooms.py            |  2 +-
 tests/rest/client/v2_alpha/test_filter.py     |  8 +-
 tests/rest/client/v2_alpha/test_sync.py       |  8 +-
 13 files changed, 116 insertions(+), 101 deletions(-)
 create mode 100644 changelog.d/3460.misc
 create mode 100644 synapse/rest/client/v1_only/__init__.py
 create mode 100644 synapse/rest/client/v1_only/base.py
 rename synapse/rest/client/{v1 => v1_only}/register.py (98%)

diff --git a/changelog.d/3460.misc b/changelog.d/3460.misc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/setup.cfg b/setup.cfg
index 3f84283a3810..c2620be6c52c 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -36,3 +36,4 @@ known_compat = mock,six
 known_twisted=twisted,OpenSSL
 multi_line_output=3
 include_trailing_comma=true
+combine_as_imports=true
diff --git a/synapse/http/client.py b/synapse/http/client.py
index d6a0d75b2b35..25b63078848f 100644
--- a/synapse/http/client.py
+++ b/synapse/http/client.py
@@ -26,9 +26,11 @@
 from twisted.internet import defer, protocol, reactor, ssl, task
 from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS
 from twisted.web._newclient import ResponseDone
-from twisted.web.client import Agent, BrowserLikeRedirectAgent, ContentDecoderAgent
-from twisted.web.client import FileBodyProducer as TwistedFileBodyProducer
 from twisted.web.client import (
+    Agent,
+    BrowserLikeRedirectAgent,
+    ContentDecoderAgent,
+    FileBodyProducer as TwistedFileBodyProducer,
     GzipDecoder,
     HTTPConnectionPool,
     PartialDownloadError,
diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py
index 75c2a4ec8e55..3418f06fd6e5 100644
--- a/synapse/rest/__init__.py
+++ b/synapse/rest/__init__.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,13 +14,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from six import PY3
+
 from synapse.http.server import JsonResource
 from synapse.rest.client import versions
-from synapse.rest.client.v1 import admin, directory, events, initial_sync
-from synapse.rest.client.v1 import login as v1_login
-from synapse.rest.client.v1 import logout, presence, profile, push_rule, pusher
-from synapse.rest.client.v1 import register as v1_register
-from synapse.rest.client.v1 import room, voip
+from synapse.rest.client.v1 import (
+    admin,
+    directory,
+    events,
+    initial_sync,
+    login as v1_login,
+    logout,
+    presence,
+    profile,
+    push_rule,
+    pusher,
+    room,
+    voip,
+)
 from synapse.rest.client.v2_alpha import (
     account,
     account_data,
@@ -42,6 +54,11 @@
     user_directory,
 )
 
+if not PY3:
+    from synapse.rest.client.v1_only import (
+        register as v1_register,
+    )
+
 
 class ClientRestResource(JsonResource):
     """A resource for version 1 of the matrix client API."""
@@ -54,14 +71,22 @@ def __init__(self, hs):
     def register_servlets(client_resource, hs):
         versions.register_servlets(client_resource)
 
-        # "v1"
-        room.register_servlets(hs, client_resource)
+        if not PY3:
+            # "v1" (Python 2 only)
+            v1_register.register_servlets(hs, client_resource)
+
+        # Deprecated in r0
+        initial_sync.register_servlets(hs, client_resource)
+        room.register_deprecated_servlets(hs, client_resource)
+
+        # Partially deprecated in r0
         events.register_servlets(hs, client_resource)
-        v1_register.register_servlets(hs, client_resource)
+
+        # "v1" + "r0"
+        room.register_servlets(hs, client_resource)
         v1_login.register_servlets(hs, client_resource)
         profile.register_servlets(hs, client_resource)
         presence.register_servlets(hs, client_resource)
-        initial_sync.register_servlets(hs, client_resource)
         directory.register_servlets(hs, client_resource)
         voip.register_servlets(hs, client_resource)
         admin.register_servlets(hs, client_resource)
diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py
index 3d6244785427..b9512a2b61bc 100644
--- a/synapse/rest/client/v1/room.py
+++ b/synapse/rest/client/v1/room.py
@@ -832,10 +832,13 @@ def register_servlets(hs, http_server):
     RoomSendEventRestServlet(hs).register(http_server)
     PublicRoomListRestServlet(hs).register(http_server)
     RoomStateRestServlet(hs).register(http_server)
-    RoomInitialSyncRestServlet(hs).register(http_server)
     RoomRedactEventRestServlet(hs).register(http_server)
     RoomTypingRestServlet(hs).register(http_server)
     SearchRestServlet(hs).register(http_server)
     JoinedRoomsRestServlet(hs).register(http_server)
     RoomEventServlet(hs).register(http_server)
     RoomEventContextServlet(hs).register(http_server)
+
+
+def register_deprecated_servlets(hs, http_server):
+    RoomInitialSyncRestServlet(hs).register(http_server)
diff --git a/synapse/rest/client/v1_only/__init__.py b/synapse/rest/client/v1_only/__init__.py
new file mode 100644
index 000000000000..936f902ace4a
--- /dev/null
+++ b/synapse/rest/client/v1_only/__init__.py
@@ -0,0 +1,3 @@
+"""
+REST APIs that are only used in v1 (the legacy API).
+"""
diff --git a/synapse/rest/client/v1_only/base.py b/synapse/rest/client/v1_only/base.py
new file mode 100644
index 000000000000..9d4db7437c6d
--- /dev/null
+++ b/synapse/rest/client/v1_only/base.py
@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""This module contains base REST classes for constructing client v1 servlets.
+"""
+
+import re
+
+from synapse.api.urls import CLIENT_PREFIX
+
+
+def v1_only_client_path_patterns(path_regex, include_in_unstable=True):
+    """Creates a regex compiled client path with the correct client path
+    prefix.
+
+    Args:
+        path_regex (str): The regex string to match. This should NOT have a ^
+        as this will be prefixed.
+    Returns:
+        list of SRE_Pattern
+    """
+    patterns = [re.compile("^" + CLIENT_PREFIX + path_regex)]
+    if include_in_unstable:
+        unstable_prefix = CLIENT_PREFIX.replace("/api/v1", "/unstable")
+        patterns.append(re.compile("^" + unstable_prefix + path_regex))
+    return patterns
diff --git a/synapse/rest/client/v1/register.py b/synapse/rest/client/v1_only/register.py
similarity index 98%
rename from synapse/rest/client/v1/register.py
rename to synapse/rest/client/v1_only/register.py
index 25a143af8dbe..3439c3c6d454 100644
--- a/synapse/rest/client/v1/register.py
+++ b/synapse/rest/client/v1_only/register.py
@@ -24,9 +24,10 @@
 from synapse.api.constants import LoginType
 from synapse.api.errors import Codes, SynapseError
 from synapse.http.servlet import assert_params_in_dict, parse_json_object_from_request
+from synapse.rest.client.v1.base import ClientV1RestServlet
 from synapse.types import create_requester
 
-from .base import ClientV1RestServlet, client_path_patterns
+from .base import v1_only_client_path_patterns
 
 logger = logging.getLogger(__name__)
 
@@ -49,7 +50,7 @@ class RegisterRestServlet(ClientV1RestServlet):
     handler doesn't have a concept of multi-stages or sessions.
     """
 
-    PATTERNS = client_path_patterns("/register$", releases=(), include_in_unstable=False)
+    PATTERNS = v1_only_client_path_patterns("/register$", include_in_unstable=False)
 
     def __init__(self, hs):
         """
@@ -379,7 +380,7 @@ class CreateUserRestServlet(ClientV1RestServlet):
     """Handles user creation via a server-to-server interface
     """
 
-    PATTERNS = client_path_patterns("/createUser$", releases=())
+    PATTERNS = v1_only_client_path_patterns("/createUser$")
 
     def __init__(self, hs):
         super(CreateUserRestServlet, self).__init__(hs)
diff --git a/tests/rest/client/v1/test_events.py b/tests/rest/client/v1/test_events.py
index a5af36a99c64..50418153fa91 100644
--- a/tests/rest/client/v1/test_events.py
+++ b/tests/rest/client/v1/test_events.py
@@ -14,100 +14,30 @@
 # limitations under the License.
 
 """ Tests REST events for /events paths."""
+
 from mock import Mock, NonCallableMock
+from six import PY3
 
-# twisted imports
 from twisted.internet import defer
 
-import synapse.rest.client.v1.events
-import synapse.rest.client.v1.register
-import synapse.rest.client.v1.room
-
-from tests import unittest
-
 from ....utils import MockHttpResource, setup_test_homeserver
 from .utils import RestTestCase
 
 PATH_PREFIX = "/_matrix/client/api/v1"
 
 
-class EventStreamPaginationApiTestCase(unittest.TestCase):
-    """ Tests event streaming query parameters and start/end keys used in the
-    Pagination stream API. """
-    user_id = "sid1"
-
-    def setUp(self):
-        # configure stream and inject items
-        pass
-
-    def tearDown(self):
-        pass
-
-    def TODO_test_long_poll(self):
-        # stream from 'end' key, send (self+other) message, expect message.
-
-        # stream from 'END', send (self+other) message, expect message.
-
-        # stream from 'end' key, send (self+other) topic, expect topic.
-
-        # stream from 'END', send (self+other) topic, expect topic.
-
-        # stream from 'end' key, send (self+other) invite, expect invite.
-
-        # stream from 'END', send (self+other) invite, expect invite.
-
-        pass
-
-    def TODO_test_stream_forward(self):
-        # stream from START, expect injected items
-
-        # stream from 'start' key, expect same content
-
-        # stream from 'end' key, expect nothing
-
-        # stream from 'END', expect nothing
-
-        # The following is needed for cases where content is removed e.g. you
-        # left a room, so the token you're streaming from is > the one that
-        # would be returned naturally from START>END.
-        # stream from very new token (higher than end key), expect same token
-        # returned as end key
-        pass
-
-    def TODO_test_limits(self):
-        # stream from a key, expect limit_num items
-
-        # stream from START, expect limit_num items
-
-        pass
-
-    def TODO_test_range(self):
-        # stream from key to key, expect X items
-
-        # stream from key to END, expect X items
-
-        # stream from START to key, expect X items
-
-        # stream from START to END, expect all items
-        pass
-
-    def TODO_test_direction(self):
-        # stream from END to START and fwds, expect newest first
-
-        # stream from END to START and bwds, expect oldest first
-
-        # stream from START to END and fwds, expect oldest first
-
-        # stream from START to END and bwds, expect newest first
-
-        pass
-
-
 class EventStreamPermissionsTestCase(RestTestCase):
     """ Tests event streaming (GET /events). """
 
+    if PY3:
+        skip = "Skip on Py3 until ported to use not V1 only register."
+
     @defer.inlineCallbacks
     def setUp(self):
+        import synapse.rest.client.v1.events
+        import synapse.rest.client.v1_only.register
+        import synapse.rest.client.v1.room
+
         self.mock_resource = MockHttpResource(prefix=PATH_PREFIX)
 
         hs = yield setup_test_homeserver(
@@ -125,7 +55,7 @@ def setUp(self):
 
         hs.get_handlers().federation_handler = Mock()
 
-        synapse.rest.client.v1.register.register_servlets(hs, self.mock_resource)
+        synapse.rest.client.v1_only.register.register_servlets(hs, self.mock_resource)
         synapse.rest.client.v1.events.register_servlets(hs, self.mock_resource)
         synapse.rest.client.v1.room.register_servlets(hs, self.mock_resource)
 
diff --git a/tests/rest/client/v1/test_register.py b/tests/rest/client/v1/test_register.py
index f15fb3621333..83a23cd8fe3c 100644
--- a/tests/rest/client/v1/test_register.py
+++ b/tests/rest/client/v1/test_register.py
@@ -16,11 +16,12 @@
 import json
 
 from mock import Mock
+from six import PY3
 
 from twisted.test.proto_helpers import MemoryReactorClock
 
 from synapse.http.server import JsonResource
-from synapse.rest.client.v1.register import register_servlets
+from synapse.rest.client.v1_only.register import register_servlets
 from synapse.util import Clock
 
 from tests import unittest
@@ -31,6 +32,8 @@ class CreateUserServletTestCase(unittest.TestCase):
     """
     Tests for CreateUserRestServlet.
     """
+    if PY3:
+        skip = "Not ported to Python 3."
 
     def setUp(self):
         self.registration_handler = Mock()
diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py
index 6b5764095efd..00fc796787b0 100644
--- a/tests/rest/client/v1/test_rooms.py
+++ b/tests/rest/client/v1/test_rooms.py
@@ -20,7 +20,6 @@
 from mock import Mock, NonCallableMock
 from six.moves.urllib import parse as urlparse
 
-# twisted imports
 from twisted.internet import defer
 
 import synapse.rest.client.v1.room
@@ -86,6 +85,7 @@ def _insert_client_ip(*args, **kwargs):
 
         self.resource = JsonResource(self.hs)
         synapse.rest.client.v1.room.register_servlets(self.hs, self.resource)
+        synapse.rest.client.v1.room.register_deprecated_servlets(self.hs, self.resource)
         self.helper = RestHelper(self.hs, self.resource, self.user_id)
 
 
diff --git a/tests/rest/client/v2_alpha/test_filter.py b/tests/rest/client/v2_alpha/test_filter.py
index 5ea9cc825fa2..e890f0feac46 100644
--- a/tests/rest/client/v2_alpha/test_filter.py
+++ b/tests/rest/client/v2_alpha/test_filter.py
@@ -21,8 +21,12 @@
 from synapse.util import Clock
 
 from tests import unittest
-from tests.server import ThreadedMemoryReactorClock as MemoryReactorClock
-from tests.server import make_request, setup_test_homeserver, wait_until_result
+from tests.server import (
+    ThreadedMemoryReactorClock as MemoryReactorClock,
+    make_request,
+    setup_test_homeserver,
+    wait_until_result,
+)
 
 PATH_PREFIX = "/_matrix/client/v2_alpha"
 
diff --git a/tests/rest/client/v2_alpha/test_sync.py b/tests/rest/client/v2_alpha/test_sync.py
index 704cf97a4093..03ec3993b2df 100644
--- a/tests/rest/client/v2_alpha/test_sync.py
+++ b/tests/rest/client/v2_alpha/test_sync.py
@@ -20,8 +20,12 @@
 from synapse.util import Clock
 
 from tests import unittest
-from tests.server import ThreadedMemoryReactorClock as MemoryReactorClock
-from tests.server import make_request, setup_test_homeserver, wait_until_result
+from tests.server import (
+    ThreadedMemoryReactorClock as MemoryReactorClock,
+    make_request,
+    setup_test_homeserver,
+    wait_until_result,
+)
 
 PATH_PREFIX = "/_matrix/client/v2_alpha"
 

From f1a15ea20692e9f4b347cdddf6d714912356ba64 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 19 Jul 2018 11:14:20 +0100
Subject: [PATCH 066/830] revert 00bc979

... we've fixed the things that caused the warnings, so we should reinstate the
warning.
---
 synapse/storage/_base.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index a6a0e6ec9fb1..1d41d8d44535 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -343,10 +343,9 @@ def runWithConnection(self, func, *args, **kwargs):
         """
         parent_context = LoggingContext.current_context()
         if parent_context == LoggingContext.sentinel:
-            # warning disabled for 0.33.0 release; proper fixes will land imminently.
-            # logger.warn(
-            #    "Running db txn from sentinel context: metrics will be lost",
-            # )
+            logger.warn(
+                "Starting db connection from sentinel context: metrics will be lost",
+            )
             parent_context = None
 
         start_time = time.time()

From 9e40834f742d95c324183f3e71ae73aafe3c6a99 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Thu, 19 Jul 2018 11:15:10 +0100
Subject: [PATCH 067/830] yes, we do need to invalidate the
 device_id_exists_cache when deleting a remote device

---
 synapse/storage/devices.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/synapse/storage/devices.py b/synapse/storage/devices.py
index 203f50f07d21..cc3cdf2ebccc 100644
--- a/synapse/storage/devices.py
+++ b/synapse/storage/devices.py
@@ -258,7 +258,6 @@ def _update_remote_device_list_cache_entry_txn(self, txn, user_id, device_id,
                 },
             )
 
-            # Do we need this?
             txn.call_after(
                 self.device_id_exists_cache.invalidate, (user_id, device_id,)
             )

From 95ccb6e2ec57f2150a697ea9cde030e8f78d6db9 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Thu, 19 Jul 2018 20:58:18 +1000
Subject: [PATCH 068/830] Don't spew errors because we can't save metrics
 (#3563)

---
 changelog.d/3548.bugfix    |  1 +
 synapse/util/logcontext.py | 11 +++++++++++
 synapse/util/metrics.py    | 19 +++++++++++++------
 3 files changed, 25 insertions(+), 6 deletions(-)
 create mode 100644 changelog.d/3548.bugfix

diff --git a/changelog.d/3548.bugfix b/changelog.d/3548.bugfix
new file mode 100644
index 000000000000..38dc3b1232a4
--- /dev/null
+++ b/changelog.d/3548.bugfix
@@ -0,0 +1 @@
+Catch failures saving metrics captured by Measure, and instead log the faulty metrics information for further analysis.
diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py
index f6c7175f74bb..8dcae50b3936 100644
--- a/synapse/util/logcontext.py
+++ b/synapse/util/logcontext.py
@@ -99,6 +99,17 @@ def reset(self):
         self.db_sched_duration_sec = 0
         self.evt_db_fetch_count = 0
 
+    def __repr__(self):
+        return ("<ContextResourceUsage ru_stime='%r', ru_utime='%r', "
+                "db_txn_count='%r', db_txn_duration_sec='%r', "
+                "db_sched_duration_sec='%r', evt_db_fetch_count='%r'>") % (
+                    self.ru_stime,
+                    self.ru_utime,
+                    self.db_txn_count,
+                    self.db_txn_duration_sec,
+                    self.db_sched_duration_sec,
+                    self.evt_db_fetch_count,)
+
     def __iadd__(self, other):
         """Add another ContextResourceUsage's stats to this one's.
 
diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py
index 6ba71078963d..97f1267380bf 100644
--- a/synapse/util/metrics.py
+++ b/synapse/util/metrics.py
@@ -104,12 +104,19 @@ def __exit__(self, exc_type, exc_val, exc_tb):
             logger.warn("Expected context. (%r)", self.name)
             return
 
-        usage = context.get_resource_usage() - self.start_usage
-        block_ru_utime.labels(self.name).inc(usage.ru_utime)
-        block_ru_stime.labels(self.name).inc(usage.ru_stime)
-        block_db_txn_count.labels(self.name).inc(usage.db_txn_count)
-        block_db_txn_duration.labels(self.name).inc(usage.db_txn_duration_sec)
-        block_db_sched_duration.labels(self.name).inc(usage.db_sched_duration_sec)
+        current = context.get_resource_usage()
+        usage = current - self.start_usage
+        try:
+            block_ru_utime.labels(self.name).inc(usage.ru_utime)
+            block_ru_stime.labels(self.name).inc(usage.ru_stime)
+            block_db_txn_count.labels(self.name).inc(usage.db_txn_count)
+            block_db_txn_duration.labels(self.name).inc(usage.db_txn_duration_sec)
+            block_db_sched_duration.labels(self.name).inc(usage.db_sched_duration_sec)
+        except ValueError:
+            logger.warn(
+                "Failed to save metrics! OLD: %r, NEW: %r",
+                self.start_usage, current
+            )
 
         if self.created_context:
             self.start_context.__exit__(exc_type, exc_val, exc_tb)

From 252f80094c5c18c7a822fda1fba36d01e1e174a3 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Thu, 19 Jul 2018 21:25:33 +1000
Subject: [PATCH 069/830] rst -> md

---
 CHANGES.md  | 2454 +++++++++++++++++++++++++++++++++++++++++++
 CHANGES.rst | 2867 ---------------------------------------------------
 2 files changed, 2454 insertions(+), 2867 deletions(-)
 create mode 100644 CHANGES.md
 delete mode 100644 CHANGES.rst

diff --git a/CHANGES.md b/CHANGES.md
new file mode 100644
index 000000000000..d7e9e08904f6
--- /dev/null
+++ b/CHANGES.md
@@ -0,0 +1,2454 @@
+Synapse 0.33.0 (2018-07-19)
+===========================
+
+Features
+--------
+
+-   Enforce the specified API for report\_event ([\#3316](https://github.com/matrix-org/synapse/issues/3316))
+-   Include CPU time from database threads in request/block metrics. ([\#3496](https://github.com/matrix-org/synapse/issues/3496), [\#3501](https://github.com/matrix-org/synapse/issues/3501))
+-   Add CPU metrics for \_fetch\_event\_list ([\#3497](https://github.com/matrix-org/synapse/issues/3497))
+-   optimisation for /sync ([\#3505](https://github.com/matrix-org/synapse/issues/3505), [\#3521](https://github.com/matrix-org/synapse/issues/3521))
+-   Optimisation to make handling incoming federation requests more efficient. ([\#3541](https://github.com/matrix-org/synapse/issues/3541))
+
+Bugfixes
+--------
+
+-   Use more portable syntax in our use of the attrs package, widening the supported versions ([\#3498](https://github.com/matrix-org/synapse/issues/3498))
+-   Fix queued federation requests being processed in the wrong order ([\#3533](https://github.com/matrix-org/synapse/issues/3533))
+-   Ensure that erasure requests are correctly honoured for publicly accessible rooms when accessed over federation. ([\#3546](https://github.com/matrix-org/synapse/issues/3546))
+-   Disable a noisy warning about logcontexts ([\#3561](https://github.com/matrix-org/synapse/issues/3561))
+
+Misc
+----
+
+-   [\#3351](https://github.com/matrix-org/synapse/issues/3351), [\#3463](https://github.com/matrix-org/synapse/issues/3463), [\#3464](https://github.com/matrix-org/synapse/issues/3464), [\#3499](https://github.com/matrix-org/synapse/issues/3499), [\#3530](https://github.com/matrix-org/synapse/issues/3530), [\#3534](https://github.com/matrix-org/synapse/issues/3534), [\#3535](https://github.com/matrix-org/synapse/issues/3535), [\#3540](https://github.com/matrix-org/synapse/issues/3540), [\#3544](https://github.com/matrix-org/synapse/issues/3544)
+
+Synapse 0.32.2 (2018-07-07)
+===========================
+
+Bugfixes
+--------
+
+-   Amend the Python dependencies to depend on attrs from PyPI, not attr ([\#3492](https://github.com/matrix-org/synapse/issues/3492))
+
+Synapse 0.32.1 (2018-07-06)
+===========================
+
+Bugfixes
+--------
+
+-   Add explicit dependency on netaddr ([\#3488](https://github.com/matrix-org/synapse/issues/3488))
+
+Changes in synapse v0.32.0 (2018-07-06)
+=======================================
+
+No changes since 0.32.0rc1
+
+Synapse 0.32.0rc1 (2018-07-05)
+==============================
+
+Features
+--------
+
+-   Add blacklist & whitelist of servers allowed to send events to a room via `m.room.server_acl` event.
+-   Cache factor override system for specific caches ([\#3334](https://github.com/matrix-org/synapse/issues/3334))
+-   Add metrics to track appservice transactions ([\#3344](https://github.com/matrix-org/synapse/issues/3344))
+-   Try to log more helpful info when a sig verification fails ([\#3372](https://github.com/matrix-org/synapse/issues/3372))
+-   Synapse now uses the best performing JSON encoder/decoder according to your runtime (simplejson on CPython, stdlib json on PyPy). ([\#3462](https://github.com/matrix-org/synapse/issues/3462))
+-   Add optional ip\_range\_whitelist param to AS registration files to lock AS IP access ([\#3465](https://github.com/matrix-org/synapse/issues/3465))
+-   Reject invalid server names in federation requests ([\#3480](https://github.com/matrix-org/synapse/issues/3480))
+-   Reject invalid server names in homeserver.yaml ([\#3483](https://github.com/matrix-org/synapse/issues/3483))
+
+Bugfixes
+--------
+
+-   Strip access\_token from outgoing requests ([\#3327](https://github.com/matrix-org/synapse/issues/3327))
+-   Redact AS tokens in logs ([\#3349](https://github.com/matrix-org/synapse/issues/3349))
+-   Fix federation backfill from SQLite servers ([\#3355](https://github.com/matrix-org/synapse/issues/3355))
+-   Fix event-purge-by-ts admin API ([\#3363](https://github.com/matrix-org/synapse/issues/3363))
+-   Fix event filtering in get\_missing\_events handler ([\#3371](https://github.com/matrix-org/synapse/issues/3371))
+-   Synapse is now stricter regarding accepting events which it cannot retrieve the prev\_events for. ([\#3456](https://github.com/matrix-org/synapse/issues/3456))
+-   Fix bug where synapse would explode when receiving unicode in HTTP User-Agent header ([\#3470](https://github.com/matrix-org/synapse/issues/3470))
+-   Invalidate cache on correct thread to avoid race ([\#3473](https://github.com/matrix-org/synapse/issues/3473))
+
+Improved Documentation
+----------------------
+
+-   `doc/postgres.rst`: fix display of the last command block. Thanks to \@ArchangeGabriel! ([\#3340](https://github.com/matrix-org/synapse/issues/3340))
+
+Deprecations and Removals
+-------------------------
+
+-   Remove was\_forgotten\_at ([\#3324](https://github.com/matrix-org/synapse/issues/3324))
+
+Misc
+----
+
+-   [\#3332](https://github.com/matrix-org/synapse/issues/3332), [\#3341](https://github.com/matrix-org/synapse/issues/3341), [\#3347](https://github.com/matrix-org/synapse/issues/3347), [\#3348](https://github.com/matrix-org/synapse/issues/3348), [\#3356](https://github.com/matrix-org/synapse/issues/3356), [\#3385](https://github.com/matrix-org/synapse/issues/3385), [\#3446](https://github.com/matrix-org/synapse/issues/3446), [\#3447](https://github.com/matrix-org/synapse/issues/3447), [\#3467](https://github.com/matrix-org/synapse/issues/3467), [\#3474](https://github.com/matrix-org/synapse/issues/3474)
+
+Changes in synapse v0.31.2 (2018-06-14)
+=======================================
+
+SECURITY UPDATE: Prevent unauthorised users from setting state events in a room when there is no `m.room.power_levels` event in force in the room. (PR \#3397)
+
+Discussion around the Matrix Spec change proposal for this change can be followed at <https://github.com/matrix-org/matrix-doc/issues/1304>.
+
+Changes in synapse v0.31.1 (2018-06-08)
+=======================================
+
+v0.31.1 fixes a security bug in the `get_missing_events` federation API where event visibility rules were not applied correctly.
+
+We are not aware of it being actively exploited but please upgrade asap.
+
+Bug Fixes:
+
+-   Fix event filtering in get\_missing\_events handler (PR \#3371)
+
+Changes in synapse v0.31.0 (2018-06-06)
+=======================================
+
+Most notable change from v0.30.0 is to switch to the python prometheus library to improve system stats reporting. WARNING: this changes a number of prometheus metrics in a backwards-incompatible manner. For more details, see [docs/metrics-howto.rst](docs/metrics-howto.rst#removal-of-deprecated-metrics--time-based-counters-becoming-histograms-in-0310).
+
+Bug Fixes:
+
+-   Fix metric documentation tables (PR \#3341)
+-   Fix LaterGauge error handling (694968f)
+-   Fix replication metrics (b7e7fd2)
+
+Changes in synapse v0.31.0-rc1 (2018-06-04) ==========================================
+
+Features:
+
+-   Switch to the Python Prometheus library (PR \#3256, \#3274)
+-   Let users leave the server notice room after joining (PR \#3287)
+
+Changes:
+
+-   daily user type phone home stats (PR \#3264)
+-   Use iter\* methods for \_filter\_events\_for\_server (PR \#3267)
+-   Docs on consent bits (PR \#3268)
+-   Remove users from user directory on deactivate (PR \#3277)
+-   Avoid sending consent notice to guest users (PR \#3288)
+-   disable CPUMetrics if no /proc/self/stat (PR \#3299)
+-   Consistently use six\'s iteritems and wrap lazy keys/values in list() if they\'re not meant to be lazy (PR \#3307)
+-   Add private IPv6 addresses to example config for url preview blacklist (PR \#3317) Thanks to \@thegcat!
+-   Reduce stuck read-receipts: ignore depth when updating (PR \#3318)
+-   Put python\'s logs into Trial when running unit tests (PR \#3319)
+
+Changes, python 3 migration:
+
+-   Replace some more comparisons with six (PR \#3243) Thanks to \@NotAFile!
+-   replace some iteritems with six (PR \#3244) Thanks to \@NotAFile!
+-   Add batch\_iter to utils (PR \#3245) Thanks to \@NotAFile!
+-   use repr, not str (PR \#3246) Thanks to \@NotAFile!
+-   Misc Python3 fixes (PR \#3247) Thanks to \@NotAFile!
+-   Py3 storage/\_base.py (PR \#3278) Thanks to \@NotAFile!
+-   more six iteritems (PR \#3279) Thanks to \@NotAFile!
+-   More Misc. py3 fixes (PR \#3280) Thanks to \@NotAFile!
+-   remaining isintance fixes (PR \#3281) Thanks to \@NotAFile!
+-   py3-ize state.py (PR \#3283) Thanks to \@NotAFile!
+-   extend tox testing for py3 to avoid regressions (PR \#3302) Thanks to \@krombel!
+-   use memoryview in py3 (PR \#3303) Thanks to \@NotAFile!
+
+Bugs:
+
+-   Fix federation backfill bugs (PR \#3261)
+-   federation: fix LaterGauge usage (PR \#3328) Thanks to \@intelfx!
+
+Changes in synapse v0.30.0 (2018-05-24)
+=======================================
+
+\'Server Notices\' are a new feature introduced in Synapse 0.30. They provide a channel whereby server administrators can send messages to users on the server.
+
+They are used as part of communication of the server policies (see `docs/consent_tracking.md`), however the intention is that they may also find a use for features such as \"Message of the day\".
+
+This feature is specific to Synapse, but uses standard Matrix communication mechanisms, so should work with any Matrix client. For more details see `docs/server_notices.md`
+
+Further Server Notices/Consent Tracking Support:
+
+-   Allow overriding the server\_notices user\'s avatar (PR \#3273)
+-   Use the localpart in the consent uri (PR \#3272)
+-   Support for putting %(consent\_uri)s in messages (PR \#3271)
+-   Block attempts to send server notices to remote users (PR \#3270)
+-   Docs on consent bits (PR \#3268)
+
+Changes in synapse v0.30.0-rc1 (2018-05-23) ==========================================
+
+Server Notices/Consent Tracking Support:
+
+-   ConsentResource to gather policy consent from users (PR \#3213)
+-   Move RoomCreationHandler out of synapse.handlers.Handlers (PR \#3225)
+-   Infrastructure for a server notices room (PR \#3232)
+-   Send users a server notice about consent (PR \#3236)
+-   Reject attempts to send event before privacy consent is given (PR \#3257)
+-   Add a \'has\_consented\' template var to consent forms (PR \#3262)
+-   Fix dependency on jinja2 (PR \#3263)
+
+Features:
+
+-   Cohort analytics (PR \#3163, \#3241, \#3251)
+-   Add lxml to docker image for web previews (PR \#3239) Thanks to \@ptman!
+-   Add in flight request metrics (PR \#3252)
+
+Changes:
+
+-   Remove unused update\_external\_syncs (PR \#3233)
+-   Use stream rather depth ordering for push actions (PR \#3212)
+-   Make purge\_history operate on tokens (PR \#3221)
+-   Don\'t support limitless pagination (PR \#3265)
+
+Bug Fixes:
+
+-   Fix logcontext resource usage tracking (PR \#3258)
+-   Fix error in handling receipts (PR \#3235)
+-   Stop the transaction cache caching failures (PR \#3255)
+
+Changes in synapse v0.29.1 (2018-05-17)
+=======================================
+
+Changes:
+
+-   Update docker documentation (PR \#3222)
+
+Changes in synapse v0.29.0 (2018-05-16)
+=======================================
+
+Not changes since v0.29.0-rc1
+
+Changes in synapse v0.29.0-rc1 (2018-05-14)
+===========================================
+
+Notable changes, a docker file for running Synapse (Thanks to \@kaiyou!) and a closed spec bug in the Client Server API. Additionally further prep for Python 3 migration.
+
+Potentially breaking change:
+
+-   Make Client-Server API return 401 for invalid token (PR \#3161).
+
+    This changes the Client-server spec to return a 401 error code instead of 403 when the access token is unrecognised. This is the behaviour required by the specification, but some clients may be relying on the old, incorrect behaviour.
+
+    Thanks to \@NotAFile for fixing this.
+
+Features:
+
+-   Add a Dockerfile for synapse (PR \#2846) Thanks to \@kaiyou!
+
+Changes - General:
+
+-   nuke-room-from-db.sh: added postgresql option and help (PR \#2337) Thanks to \@rubo77!
+-   Part user from rooms on account deactivate (PR \#3201)
+-   Make \'unexpected logging context\' into warnings (PR \#3007)
+-   Set Server header in SynapseRequest (PR \#3208)
+-   remove duplicates from groups tables (PR \#3129)
+-   Improve exception handling for background processes (PR \#3138)
+-   Add missing consumeErrors to improve exception handling (PR \#3139)
+-   reraise exceptions more carefully (PR \#3142)
+-   Remove redundant call to preserve\_fn (PR \#3143)
+-   Trap exceptions thrown within run\_in\_background (PR \#3144)
+
+Changes - Refactors:
+
+-   Refactor /context to reuse pagination storage functions (PR \#3193)
+-   Refactor recent events func to use pagination func (PR \#3195)
+-   Refactor pagination DB API to return concrete type (PR \#3196)
+-   Refactor get\_recent\_events\_for\_room return type (PR \#3198)
+-   Refactor sync APIs to reuse pagination API (PR \#3199)
+-   Remove unused code path from member change DB func (PR \#3200)
+-   Refactor request handling wrappers (PR \#3203)
+-   transaction\_id, destination defined twice (PR \#3209) Thanks to \@damir-manapov!
+-   Refactor event storage to prepare for changes in state calculations (PR \#3141)
+-   Set Server header in SynapseRequest (PR \#3208)
+-   Use deferred.addTimeout instead of time\_bound\_deferred (PR \#3127, \#3178)
+-   Use run\_in\_background in preference to preserve\_fn (PR \#3140)
+
+Changes - Python 3 migration:
+
+-   Construct HMAC as bytes on py3 (PR \#3156) Thanks to \@NotAFile!
+-   run config tests on py3 (PR \#3159) Thanks to \@NotAFile!
+-   Open certificate files as bytes (PR \#3084) Thanks to \@NotAFile!
+-   Open config file in non-bytes mode (PR \#3085) Thanks to \@NotAFile!
+-   Make event properties raise AttributeError instead (PR \#3102) Thanks to \@NotAFile!
+-   Use six.moves.urlparse (PR \#3108) Thanks to \@NotAFile!
+-   Add py3 tests to tox with folders that work (PR \#3145) Thanks to \@NotAFile!
+-   Don\'t yield in list comprehensions (PR \#3150) Thanks to \@NotAFile!
+-   Move more xrange to six (PR \#3151) Thanks to \@NotAFile!
+-   make imports local (PR \#3152) Thanks to \@NotAFile!
+-   move httplib import to six (PR \#3153) Thanks to \@NotAFile!
+-   Replace stringIO imports with six (PR \#3154, \#3168) Thanks to \@NotAFile!
+-   more bytes strings (PR \#3155) Thanks to \@NotAFile!
+
+Bug Fixes:
+
+-   synapse fails to start under Twisted \>= 18.4 (PR \#3157)
+-   Fix a class of logcontext leaks (PR \#3170)
+-   Fix a couple of logcontext leaks in unit tests (PR \#3172)
+-   Fix logcontext leak in media repo (PR \#3174)
+-   Escape label values in prometheus metrics (PR \#3175, \#3186)
+-   Fix \'Unhandled Error\' logs with Twisted 18.4 (PR \#3182) Thanks to \@Half-Shot!
+-   Fix logcontext leaks in rate limiter (PR \#3183)
+-   notifications: Convert next\_token to string according to the spec (PR \#3190) Thanks to \@mujx!
+-   nuke-room-from-db.sh: fix deletion from search table (PR \#3194) Thanks to \@rubo77!
+-   add guard for None on purge\_history api (PR \#3160) Thanks to \@krombel!
+
+Changes in synapse v0.28.1 (2018-05-01)
+=======================================
+
+SECURITY UPDATE
+
+-   Clamp the allowed values of event depth received over federation to be \[0, 2\^63 - 1\]. This mitigates an attack where malicious events injected with depth = 2\^63 - 1 render rooms unusable. Depth is used to determine the cosmetic ordering of events within a room, and so the ordering of events in such a room will default to using stream\_ordering rather than depth (topological\_ordering).
+
+    This is a temporary solution to mitigate abuse in the wild, whilst a long term solution is being implemented to improve how the depth parameter is used.
+
+    Full details at <https://docs.google.com/document/d/1I3fi2S-XnpO45qrpCsowZv8P8dHcNZ4fsBsbOW7KABI>
+
+-   Pin Twisted to \<18.4 until we stop using the private \_OpenSSLECCurve API.
+
+Changes in synapse v0.28.0 (2018-04-26)
+=======================================
+
+Bug Fixes:
+
+-   Fix quarantine media admin API and search reindex (PR \#3130)
+-   Fix media admin APIs (PR \#3134)
+
+Changes in synapse v0.28.0-rc1 (2018-04-24)
+===========================================
+
+Minor performance improvement to federation sending and bug fixes.
+
+(Note: This release does not include the delta state resolution implementation discussed in matrix live)
+
+Features:
+
+-   Add metrics for event processing lag (PR \#3090)
+-   Add metrics for ResponseCache (PR \#3092)
+
+Changes:
+
+-   Synapse on PyPy (PR \#2760) Thanks to \@Valodim!
+-   move handling of auto\_join\_rooms to RegisterHandler (PR \#2996) Thanks to \@krombel!
+-   Improve handling of SRV records for federation connections (PR \#3016) Thanks to \@silkeh!
+-   Document the behaviour of ResponseCache (PR \#3059)
+-   Preparation for py3 (PR \#3061, \#3073, \#3074, \#3075, \#3103, \#3104, \#3106, \#3107, \#3109, \#3110) Thanks to \@NotAFile!
+-   update prometheus dashboard to use new metric names (PR \#3069) Thanks to \@krombel!
+-   use python3-compatible prints (PR \#3074) Thanks to \@NotAFile!
+-   Send federation events concurrently (PR \#3078)
+-   Limit concurrent event sends for a room (PR \#3079)
+-   Improve R30 stat definition (PR \#3086)
+-   Send events to ASes concurrently (PR \#3088)
+-   Refactor ResponseCache usage (PR \#3093)
+-   Clarify that SRV may not point to a CNAME (PR \#3100) Thanks to \@silkeh!
+-   Use str(e) instead of e.message (PR \#3103) Thanks to \@NotAFile!
+-   Use six.itervalues in some places (PR \#3106) Thanks to \@NotAFile!
+-   Refactor store.have\_events (PR \#3117)
+
+Bug Fixes:
+
+-   Return 401 for invalid access\_token on logout (PR \#2938) Thanks to \@dklug!
+-   Return a 404 rather than a 500 on rejoining empty rooms (PR \#3080)
+-   fix federation\_domain\_whitelist (PR \#3099)
+-   Avoid creating events with huge numbers of prev\_events (PR \#3113)
+-   Reject events which have lots of prev\_events (PR \#3118)
+
+Changes in synapse v0.27.4 (2018-04-13) ======================================
+
+Changes:
+
+-   Update canonicaljson dependency (\#3095)
+
+Changes in synapse v0.27.3 (2018-04-11) ======================================
+
+Bug fixes:
+
+-   URL quote path segments over federation (\#3082)
+
+Changes in synapse v0.27.3-rc2 (2018-04-09) ==========================================
+
+v0.27.3-rc1 used a stale version of the develop branch so the changelog overstates the functionality. v0.27.3-rc2 is up to date, rc1 should be ignored.
+
+Changes in synapse v0.27.3-rc1 (2018-04-09) =======================================
+
+Notable changes include API support for joinability of groups. Also new metrics and phone home stats. Phone home stats include better visibility of system usage so we can tweak synpase to work better for all users rather than our own experience with matrix.org. Also, recording \'r30\' stat which is the measure we use to track overal growth of the Matrix ecosystem. It is defined as:-
+
+Counts the number of native 30 day retained users, defined as:- \* Users who have created their accounts more than 30 days
+
+:   -   Where last seen at most 30 days ago
+    -   Where account creation and last\_seen are \> 30 days\"
+
+Features:
+
+-   Add joinability for groups (PR \#3045)
+-   Implement group join API (PR \#3046)
+-   Add counter metrics for calculating state delta (PR \#3033)
+-   R30 stats (PR \#3041)
+-   Measure time it takes to calculate state group ID (PR \#3043)
+-   Add basic performance statistics to phone home (PR \#3044)
+-   Add response size metrics (PR \#3071)
+-   phone home cache size configurations (PR \#3063)
+
+Changes:
+
+-   Add a blurb explaining the main synapse worker (PR \#2886) Thanks to \@turt2live!
+-   Replace old style error catching with \'as\' keyword (PR \#3000) Thanks to \@NotAFile!
+-   Use .iter\* to avoid copies in StateHandler (PR \#3006)
+-   Linearize calls to \_generate\_user\_id (PR \#3029)
+-   Remove last usage of ujson (PR \#3030)
+-   Use simplejson throughout (PR \#3048)
+-   Use static JSONEncoders (PR \#3049)
+-   Remove uses of events.content (PR \#3060)
+-   Improve database cache performance (PR \#3068)
+
+Bug fixes:
+
+-   Add room\_id to the response of rooms/{roomId}/join (PR \#2986) Thanks to \@jplatte!
+-   Fix replication after switch to simplejson (PR \#3015)
+-   404 correctly on missing paths via NoResource (PR \#3022)
+-   Fix error when claiming e2e keys from offline servers (PR \#3034)
+-   fix tests/storage/test\_user\_directory.py (PR \#3042)
+-   use PUT instead of POST for federating groups/m.join\_policy (PR \#3070) Thanks to \@krombel!
+-   postgres port script: fix state\_groups\_pkey error (PR \#3072)
+
+Changes in synapse v0.27.2 (2018-03-26)
+=======================================
+
+Bug fixes:
+
+-   Fix bug which broke TCP replication between workers (PR \#3015)
+
+Changes in synapse v0.27.1 (2018-03-26)
+=======================================
+
+Meta release as v0.27.0 temporarily pointed to the wrong commit
+
+Changes in synapse v0.27.0 (2018-03-26)
+=======================================
+
+No changes since v0.27.0-rc2
+
+Changes in synapse v0.27.0-rc2 (2018-03-19)
+===========================================
+
+Pulls in v0.26.1
+
+Bug fixes:
+
+-   Fix bug introduced in v0.27.0-rc1 that causes much increased memory usage in state cache (PR \#3005)
+
+Changes in synapse v0.26.1 (2018-03-15)
+=======================================
+
+Bug fixes:
+
+-   Fix bug where an invalid event caused server to stop functioning correctly, due to parsing and serializing bugs in ujson library (PR \#3008)
+
+Changes in synapse v0.27.0-rc1 (2018-03-14)
+===========================================
+
+The common case for running Synapse is not to run separate workers, but for those that do, be aware that synctl no longer starts the main synapse when using `-a` option with workers. A new worker file should be added with `worker_app: synapse.app.homeserver`.
+
+This release also begins the process of renaming a number of the metrics reported to prometheus. See [docs/metrics-howto.rst](docs/metrics-howto.rst#block-and-response-metrics-renamed-for-0-27-0). Note that the v0.28.0 release will remove the deprecated metric names.
+
+Features:
+
+-   Add ability for ASes to override message send time (PR \#2754)
+-   Add support for custom storage providers for media repository (PR \#2867, \#2777, \#2783, \#2789, \#2791, \#2804, \#2812, \#2814, \#2857, \#2868, \#2767)
+-   Add purge API features, see [docs/admin\_api/purge\_history\_api.rst](docs/admin_api/purge_history_api.rst) for full details (PR \#2858, \#2867, \#2882, \#2946, \#2962, \#2943)
+-   Add support for whitelisting 3PIDs that users can register. (PR \#2813)
+-   Add `/room/{id}/event/{id}` API (PR \#2766)
+-   Add an admin API to get all the media in a room (PR \#2818) Thanks to \@turt2live!
+-   Add `federation_domain_whitelist` option (PR \#2820, \#2821)
+
+Changes:
+
+-   Continue to factor out processing from main process and into worker processes. See updated [docs/workers.rst](docs/workers.rst) (PR \#2892 - \#2904, \#2913, \#2920 - \#2926, \#2947, \#2847, \#2854, \#2872, \#2873, \#2874, \#2928, \#2929, \#2934, \#2856, \#2976 - \#2984, \#2987 - \#2989, \#2991 - \#2993, \#2995, \#2784)
+-   Ensure state cache is used when persisting events (PR \#2864, \#2871, \#2802, \#2835, \#2836, \#2841, \#2842, \#2849)
+-   Change the default config to bind on both IPv4 and IPv6 on all platforms (PR \#2435) Thanks to \@silkeh!
+-   No longer require a specific version of saml2 (PR \#2695) Thanks to \@okurz!
+-   Remove `verbosity`/`log_file` from generated config (PR \#2755)
+-   Add and improve metrics and logging (PR \#2770, \#2778, \#2785, \#2786, \#2787, \#2793, \#2794, \#2795, \#2809, \#2810, \#2833, \#2834, \#2844, \#2965, \#2927, \#2975, \#2790, \#2796, \#2838)
+-   When using synctl with workers, don\'t start the main synapse automatically (PR \#2774)
+-   Minor performance improvements (PR \#2773, \#2792)
+-   Use a connection pool for non-federation outbound connections (PR \#2817)
+-   Make it possible to run unit tests against postgres (PR \#2829)
+-   Update pynacl dependency to 1.2.1 or higher (PR \#2888) Thanks to \@bachp!
+-   Remove ability for AS users to call /events and /sync (PR \#2948)
+-   Use bcrypt.checkpw (PR \#2949) Thanks to \@krombel!
+
+Bug fixes:
+
+-   Fix broken `ldap_config` config option (PR \#2683) Thanks to \@seckrv!
+-   Fix error message when user is not allowed to unban (PR \#2761) Thanks to \@turt2live!
+-   Fix publicised groups GET API (singular) over federation (PR \#2772)
+-   Fix user directory when using `user_directory_search_all_users` config option (PR \#2803, \#2831)
+-   Fix error on `/publicRooms` when no rooms exist (PR \#2827)
+-   Fix bug in quarantine\_media (PR \#2837)
+-   Fix url\_previews when no Content-Type is returned from URL (PR \#2845)
+-   Fix rare race in sync API when joining room (PR \#2944)
+-   Fix slow event search, switch back from GIST to GIN indexes (PR \#2769, \#2848)
+
+Changes in synapse v0.26.0 (2018-01-05)
+=======================================
+
+No changes since v0.26.0-rc1
+
+Changes in synapse v0.26.0-rc1 (2017-12-13)
+===========================================
+
+Features:
+
+-   Add ability for ASes to publicise groups for their users (PR \#2686)
+-   Add all local users to the user\_directory and optionally search them (PR \#2723)
+-   Add support for custom login types for validating users (PR \#2729)
+
+Changes:
+
+-   Update example Prometheus config to new format (PR \#2648) Thanks to \@krombel!
+-   Rename redact\_content option to include\_content in Push API (PR \#2650)
+-   Declare support for r0.3.0 (PR \#2677)
+-   Improve upserts (PR \#2684, \#2688, \#2689, \#2713)
+-   Improve documentation of workers (PR \#2700)
+-   Improve tracebacks on exceptions (PR \#2705)
+-   Allow guest access to group APIs for reading (PR \#2715)
+-   Support for posting content in federation\_client script (PR \#2716)
+-   Delete devices and pushers on logouts etc (PR \#2722)
+
+Bug fixes:
+
+-   Fix database port script (PR \#2673)
+-   Fix internal server error on login with ldap\_auth\_provider (PR \#2678) Thanks to \@jkolo!
+-   Fix error on sqlite 3.7 (PR \#2697)
+-   Fix OPTIONS on preview\_url (PR \#2707)
+-   Fix error handling on dns lookup (PR \#2711)
+-   Fix wrong avatars when inviting multiple users when creating room (PR \#2717)
+-   Fix 500 when joining matrix-dev (PR \#2719)
+
+Changes in synapse v0.25.1 (2017-11-17)
+=======================================
+
+Bug fixes:
+
+-   Fix login with LDAP and other password provider modules (PR \#2678). Thanks to \@jkolo!
+
+Changes in synapse v0.25.0 (2017-11-15)
+=======================================
+
+Bug fixes:
+
+-   Fix port script (PR \#2673)
+
+Changes in synapse v0.25.0-rc1 (2017-11-14)
+===========================================
+
+Features:
+
+-   Add is\_public to groups table to allow for private groups (PR \#2582)
+-   Add a route for determining who you are (PR \#2668) Thanks to \@turt2live!
+-   Add more features to the password providers (PR \#2608, \#2610, \#2620, \#2622, \#2623, \#2624, \#2626, \#2628, \#2629)
+-   Add a hook for custom rest endpoints (PR \#2627)
+-   Add API to update group room visibility (PR \#2651)
+
+Changes:
+
+-   Ignore \<noscript\> tags when generating URL preview descriptions (PR \#2576) Thanks to \@maximevaillancourt!
+-   Register some /unstable endpoints in /r0 as well (PR \#2579) Thanks to \@krombel!
+-   Support /keys/upload on /r0 as well as /unstable (PR \#2585)
+-   Front-end proxy: pass through auth header (PR \#2586)
+-   Allow ASes to deactivate their own users (PR \#2589)
+-   Remove refresh tokens (PR \#2613)
+-   Automatically set default displayname on register (PR \#2617)
+-   Log login requests (PR \#2618)
+-   Always return is\_public in the /groups/:group\_id/rooms API (PR \#2630)
+-   Avoid no-op media deletes (PR \#2637) Thanks to \@spantaleev!
+-   Fix various embarrassing typos around user\_directory and add some doc. (PR \#2643)
+-   Return whether a user is an admin within a group (PR \#2647)
+-   Namespace visibility options for groups (PR \#2657)
+-   Downcase UserIDs on registration (PR \#2662)
+-   Cache failures when fetching URL previews (PR \#2669)
+
+Bug fixes:
+
+-   Fix port script (PR \#2577)
+-   Fix error when running synapse with no logfile (PR \#2581)
+-   Fix UI auth when deleting devices (PR \#2591)
+-   Fix typo when checking if user is invited to group (PR \#2599)
+-   Fix the port script to drop NUL values in all tables (PR \#2611)
+-   Fix appservices being backlogged and not receiving new events due to a bug in notify\_interested\_services (PR \#2631) Thanks to \@xyzz!
+-   Fix updating rooms avatar/display name when modified by admin (PR \#2636) Thanks to \@farialima!
+-   Fix bug in state group storage (PR \#2649)
+-   Fix 500 on invalid utf-8 in request (PR \#2663)
+
+Changes in synapse v0.24.1 (2017-10-24)
+=======================================
+
+Bug fixes:
+
+-   Fix updating group profiles over federation (PR \#2567)
+
+Changes in synapse v0.24.0 (2017-10-23)
+=======================================
+
+No changes since v0.24.0-rc1
+
+Changes in synapse v0.24.0-rc1 (2017-10-19)
+===========================================
+
+Features:
+
+-   Add Group Server (PR \#2352, \#2363, \#2374, \#2377, \#2378, \#2382, \#2410, \#2426, \#2430, \#2454, \#2471, \#2472, \#2544)
+-   Add support for channel notifications (PR \#2501)
+-   Add basic implementation of backup media store (PR \#2538)
+-   Add config option to auto-join new users to rooms (PR \#2545)
+
+Changes:
+
+-   Make the spam checker a module (PR \#2474)
+-   Delete expired url cache data (PR \#2478)
+-   Ignore incoming events for rooms that we have left (PR \#2490)
+-   Allow spam checker to reject invites too (PR \#2492)
+-   Add room creation checks to spam checker (PR \#2495)
+-   Spam checking: add the invitee to user\_may\_invite (PR \#2502)
+-   Process events from federation for different rooms in parallel (PR \#2520)
+-   Allow error strings from spam checker (PR \#2531)
+-   Improve error handling for missing files in config (PR \#2551)
+
+Bug fixes:
+
+-   Fix handling SERVFAILs when doing AAAA lookups for federation (PR \#2477)
+-   Fix incompatibility with newer versions of ujson (PR \#2483) Thanks to \@jeremycline!
+-   Fix notification keywords that start/end with non-word chars (PR \#2500)
+-   Fix stack overflow and logcontexts from linearizer (PR \#2532)
+-   Fix 500 error when fields missing from power\_levels event (PR \#2552)
+-   Fix 500 error when we get an error handling a PDU (PR \#2553)
+
+Changes in synapse v0.23.1 (2017-10-02)
+=======================================
+
+Changes:
+
+-   Make \'affinity\' package optional, as it is not supported on some platforms
+
+Changes in synapse v0.23.0 (2017-10-02)
+=======================================
+
+No changes since v0.23.0-rc2
+
+Changes in synapse v0.23.0-rc2 (2017-09-26)
+===========================================
+
+Bug fixes:
+
+-   Fix regression in performance of syncs (PR \#2470)
+
+Changes in synapse v0.23.0-rc1 (2017-09-25)
+===========================================
+
+Features:
+
+-   Add a frontend proxy worker (PR \#2344)
+-   Add support for event\_id\_only push format (PR \#2450)
+-   Add a PoC for filtering spammy events (PR \#2456)
+-   Add a config option to block all room invites (PR \#2457)
+
+Changes:
+
+-   Use bcrypt module instead of py-bcrypt (PR \#2288) Thanks to \@kyrias!
+-   Improve performance of generating push notifications (PR \#2343, \#2357, \#2365, \#2366, \#2371)
+-   Improve DB performance for device list handling in sync (PR \#2362)
+-   Include a sample prometheus config (PR \#2416)
+-   Document known to work postgres version (PR \#2433) Thanks to \@ptman!
+
+Bug fixes:
+
+-   Fix caching error in the push evaluator (PR \#2332)
+-   Fix bug where pusherpool didn\'t start and broke some rooms (PR \#2342)
+-   Fix port script for user directory tables (PR \#2375)
+-   Fix device lists notifications when user rejoins a room (PR \#2443, \#2449)
+-   Fix sync to always send down current state events in timeline (PR \#2451)
+-   Fix bug where guest users were incorrectly kicked (PR \#2453)
+-   Fix bug talking to IPv6 only servers using SRV records (PR \#2462)
+
+Changes in synapse v0.22.1 (2017-07-06)
+=======================================
+
+Bug fixes:
+
+-   Fix bug where pusher pool didn\'t start and caused issues when interacting with some rooms (PR \#2342)
+
+Changes in synapse v0.22.0 (2017-07-06)
+=======================================
+
+No changes since v0.22.0-rc2
+
+Changes in synapse v0.22.0-rc2 (2017-07-04)
+===========================================
+
+Changes:
+
+-   Improve performance of storing user IPs (PR \#2307, \#2308)
+-   Slightly improve performance of verifying access tokens (PR \#2320)
+-   Slightly improve performance of event persistence (PR \#2321)
+-   Increase default cache factor size from 0.1 to 0.5 (PR \#2330)
+
+Bug fixes:
+
+-   Fix bug with storing registration sessions that caused frequent CPU churn (PR \#2319)
+
+Changes in synapse v0.22.0-rc1 (2017-06-26)
+===========================================
+
+Features:
+
+-   Add a user directory API (PR \#2252, and many more)
+-   Add shutdown room API to remove room from local server (PR \#2291)
+-   Add API to quarantine media (PR \#2292)
+-   Add new config option to not send event contents to push servers (PR \#2301) Thanks to \@cjdelisle!
+
+Changes:
+
+-   Various performance fixes (PR \#2177, \#2233, \#2230, \#2238, \#2248, \#2256, \#2274)
+-   Deduplicate sync filters (PR \#2219) Thanks to \@krombel!
+-   Correct a typo in UPGRADE.rst (PR \#2231) Thanks to \@aaronraimist!
+-   Add count of one time keys to sync stream (PR \#2237)
+-   Only store event\_auth for state events (PR \#2247)
+-   Store URL cache preview downloads separately (PR \#2299)
+
+Bug fixes:
+
+-   Fix users not getting notifications when AS listened to that user\_id (PR \#2216) Thanks to \@slipeer!
+-   Fix users without push set up not getting notifications after joining rooms (PR \#2236)
+-   Fix preview url API to trim long descriptions (PR \#2243)
+-   Fix bug where we used cached but unpersisted state group as prev group, resulting in broken state of restart (PR \#2263)
+-   Fix removing of pushers when using workers (PR \#2267)
+-   Fix CORS headers to allow Authorization header (PR \#2285) Thanks to \@krombel!
+
+Changes in synapse v0.21.1 (2017-06-15)
+=======================================
+
+Bug fixes:
+
+-   Fix bug in anonymous usage statistic reporting (PR \#2281)
+
+Changes in synapse v0.21.0 (2017-05-18)
+=======================================
+
+No changes since v0.21.0-rc3
+
+Changes in synapse v0.21.0-rc3 (2017-05-17)
+===========================================
+
+Features:
+
+-   Add per user rate-limiting overrides (PR \#2208)
+-   Add config option to limit maximum number of events requested by `/sync` and `/messages` (PR \#2221) Thanks to \@psaavedra!
+
+Changes:
+
+-   Various small performance fixes (PR \#2201, \#2202, \#2224, \#2226, \#2227, \#2228, \#2229)
+-   Update username availability checker API (PR \#2209, \#2213)
+-   When purging, don\'t de-delta state groups we\'re about to delete (PR \#2214)
+-   Documentation to check synapse version (PR \#2215) Thanks to \@hamber-dick!
+-   Add an index to event\_search to speed up purge history API (PR \#2218)
+
+Bug fixes:
+
+-   Fix API to allow clients to upload one-time-keys with new sigs (PR \#2206)
+
+Changes in synapse v0.21.0-rc2 (2017-05-08)
+===========================================
+
+Changes:
+
+-   Always mark remotes as up if we receive a signed request from them (PR \#2190)
+
+Bug fixes:
+
+-   Fix bug where users got pushed for rooms they had muted (PR \#2200)
+
+Changes in synapse v0.21.0-rc1 (2017-05-08)
+===========================================
+
+Features:
+
+-   Add username availability checker API (PR \#2183)
+-   Add read marker API (PR \#2120)
+
+Changes:
+
+-   Enable guest access for the 3pl/3pid APIs (PR \#1986)
+-   Add setting to support TURN for guests (PR \#2011)
+-   Various performance improvements (PR \#2075, \#2076, \#2080, \#2083, \#2108, \#2158, \#2176, \#2185)
+-   Make synctl a bit more user friendly (PR \#2078, \#2127) Thanks \@APwhitehat!
+-   Replace HTTP replication with TCP replication (PR \#2082, \#2097, \#2098, \#2099, \#2103, \#2014, \#2016, \#2115, \#2116, \#2117)
+-   Support authenticated SMTP (PR \#2102) Thanks \@DanielDent!
+-   Add a counter metric for successfully-sent transactions (PR \#2121)
+-   Propagate errors sensibly from proxied IS requests (PR \#2147)
+-   Add more granular event send metrics (PR \#2178)
+
+Bug fixes:
+
+-   Fix nuke-room script to work with current schema (PR \#1927) Thanks \@zuckschwerdt!
+-   Fix db port script to not assume postgres tables are in the public schema (PR \#2024) Thanks \@jerrykan!
+-   Fix getting latest device IP for user with no devices (PR \#2118)
+-   Fix rejection of invites to unreachable servers (PR \#2145)
+-   Fix code for reporting old verify keys in synapse (PR \#2156)
+-   Fix invite state to always include all events (PR \#2163)
+-   Fix bug where synapse would always fetch state for any missing event (PR \#2170)
+-   Fix a leak with timed out HTTP connections (PR \#2180)
+-   Fix bug where we didn\'t time out HTTP requests to ASes (PR \#2192)
+
+Docs:
+
+-   Clarify doc for SQLite to PostgreSQL port (PR \#1961) Thanks \@benhylau!
+-   Fix typo in synctl help (PR \#2107) Thanks \@HarHarLinks!
+-   `web_client_location` documentation fix (PR \#2131) Thanks \@matthewjwolff!
+-   Update README.rst with FreeBSD changes (PR \#2132) Thanks \@feld!
+-   Clarify setting up metrics (PR \#2149) Thanks \@encks!
+
+Changes in synapse v0.20.0 (2017-04-11)
+=======================================
+
+Bug fixes:
+
+-   Fix joining rooms over federation where not all servers in the room saw the new server had joined (PR \#2094)
+
+Changes in synapse v0.20.0-rc1 (2017-03-30)
+===========================================
+
+Features:
+
+-   Add delete\_devices API (PR \#1993)
+-   Add phone number registration/login support (PR \#1994, \#2055)
+
+Changes:
+
+-   Use JSONSchema for validation of filters. Thanks \@pik! (PR \#1783)
+-   Reread log config on SIGHUP (PR \#1982)
+-   Speed up public room list (PR \#1989)
+-   Add helpful texts to logger config options (PR \#1990)
+-   Minor `/sync` performance improvements. (PR \#2002, \#2013, \#2022)
+-   Add some debug to help diagnose weird federation issue (PR \#2035)
+-   Correctly limit retries for all federation requests (PR \#2050, \#2061)
+-   Don\'t lock table when persisting new one time keys (PR \#2053)
+-   Reduce some CPU work on DB threads (PR \#2054)
+-   Cache hosts in room (PR \#2060)
+-   Batch sending of device list pokes (PR \#2063)
+-   Speed up persist event path in certain edge cases (PR \#2070)
+
+Bug fixes:
+
+-   Fix bug where current\_state\_events renamed to current\_state\_ids (PR \#1849)
+-   Fix routing loop when fetching remote media (PR \#1992)
+-   Fix current\_state\_events table to not lie (PR \#1996)
+-   Fix CAS login to handle PartialDownloadError (PR \#1997)
+-   Fix assertion to stop transaction queue getting wedged (PR \#2010)
+-   Fix presence to fallback to last\_active\_ts if it beats the last sync time. Thanks \@Half-Shot! (PR \#2014)
+-   Fix bug when federation received a PDU while a room join is in progress (PR \#2016)
+-   Fix resetting state on rejected events (PR \#2025)
+-   Fix installation issues in readme. Thanks \@ricco386 (PR \#2037)
+-   Fix caching of remote servers\' signature keys (PR \#2042)
+-   Fix some leaking log context (PR \#2048, \#2049, \#2057, \#2058)
+-   Fix rejection of invites not reaching sync (PR \#2056)
+
+Changes in synapse v0.19.3 (2017-03-20)
+=======================================
+
+No changes since v0.19.3-rc2
+
+Changes in synapse v0.19.3-rc2 (2017-03-13)
+===========================================
+
+Bug fixes:
+
+-   Fix bug in handling of incoming device list updates over federation.
+
+Changes in synapse v0.19.3-rc1 (2017-03-08)
+===========================================
+
+Features:
+
+-   Add some administration functionalities. Thanks to morteza-araby! (PR \#1784)
+
+Changes:
+
+-   Reduce database table sizes (PR \#1873, \#1916, \#1923, \#1963)
+-   Update contrib/ to not use syutil. Thanks to andrewshadura! (PR \#1907)
+-   Don\'t fetch current state when sending an event in common case (PR \#1955)
+
+Bug fixes:
+
+-   Fix synapse\_port\_db failure. Thanks to Pneumaticat! (PR \#1904)
+-   Fix caching to not cache error responses (PR \#1913)
+-   Fix APIs to make kick & ban reasons work (PR \#1917)
+-   Fix bugs in the /keys/changes api (PR \#1921)
+-   Fix bug where users couldn\'t forget rooms they were banned from (PR \#1922)
+-   Fix issue with long language values in pushers API (PR \#1925)
+-   Fix a race in transaction queue (PR \#1930)
+-   Fix dynamic thumbnailing to preserve aspect ratio. Thanks to jkolo! (PR \#1945)
+-   Fix device list update to not constantly resync (PR \#1964)
+-   Fix potential for huge memory usage when getting device that have changed (PR \#1969)
+
+Changes in synapse v0.19.2 (2017-02-20)
+=======================================
+
+-   Fix bug with event visibility check in /context/ API. Thanks to Tokodomo for pointing it out! (PR \#1929)
+
+Changes in synapse v0.19.1 (2017-02-09)
+=======================================
+
+-   Fix bug where state was incorrectly reset in a room when synapse received an event over federation that did not pass auth checks (PR \#1892)
+
+Changes in synapse v0.19.0 (2017-02-04)
+=======================================
+
+No changes since RC 4.
+
+Changes in synapse v0.19.0-rc4 (2017-02-02)
+===========================================
+
+-   Bump cache sizes for common membership queries (PR \#1879)
+
+Changes in synapse v0.19.0-rc3 (2017-02-02)
+===========================================
+
+-   Fix email push in pusher worker (PR \#1875)
+-   Make presence.get\_new\_events a bit faster (PR \#1876)
+-   Make /keys/changes a bit more performant (PR \#1877)
+
+Changes in synapse v0.19.0-rc2 (2017-02-02)
+===========================================
+
+-   Include newly joined users in /keys/changes API (PR \#1872)
+
+Changes in synapse v0.19.0-rc1 (2017-02-02)
+===========================================
+
+Features:
+
+-   Add support for specifying multiple bind addresses (PR \#1709, \#1712, \#1795, \#1835). Thanks to \@kyrias!
+-   Add /account/3pid/delete endpoint (PR \#1714)
+-   Add config option to configure the Riot URL used in notification emails (PR \#1811). Thanks to \@aperezdc!
+-   Add username and password config options for turn server (PR \#1832). Thanks to \@xsteadfastx!
+-   Implement device lists updates over federation (PR \#1857, \#1861, \#1864)
+-   Implement /keys/changes (PR \#1869, \#1872)
+
+Changes:
+
+-   Improve IPv6 support (PR \#1696). Thanks to \@kyrias and \@glyph!
+-   Log which files we saved attachments to in the media\_repository (PR \#1791)
+-   Linearize updates to membership via PUT /state/ to better handle multiple joins (PR \#1787)
+-   Limit number of entries to prefill from cache on startup (PR \#1792)
+-   Remove full\_twisted\_stacktraces option (PR \#1802)
+-   Measure size of some caches by sum of the size of cached values (PR \#1815)
+-   Measure metrics of string\_cache (PR \#1821)
+-   Reduce logging verbosity (PR \#1822, \#1823, \#1824)
+-   Don\'t clobber a displayname or avatar\_url if provided by an m.room.member event (PR \#1852)
+-   Better handle 401/404 response for federation /send/ (PR \#1866, \#1871)
+
+Fixes:
+
+-   Fix ability to change password to a non-ascii one (PR \#1711)
+-   Fix push getting stuck due to looking at the wrong view of state (PR \#1820)
+-   Fix email address comparison to be case insensitive (PR \#1827)
+-   Fix occasional inconsistencies of room membership (PR \#1836, \#1840)
+
+Performance:
+
+-   Don\'t block messages sending on bumping presence (PR \#1789)
+-   Change device\_inbox stream index to include user (PR \#1793)
+-   Optimise state resolution (PR \#1818)
+-   Use DB cache of joined users for presence (PR \#1862)
+-   Add an index to make membership queries faster (PR \#1867)
+
+Changes in synapse v0.18.7 (2017-01-09)
+=======================================
+
+No changes from v0.18.7-rc2
+
+Changes in synapse v0.18.7-rc2 (2017-01-07)
+===========================================
+
+Bug fixes:
+
+-   Fix error in rc1\'s discarding invalid inbound traffic logic that was incorrectly discarding missing events
+
+Changes in synapse v0.18.7-rc1 (2017-01-06)
+===========================================
+
+Bug fixes:
+
+-   Fix error in \#PR 1764 to actually fix the nightmare \#1753 bug.
+-   Improve deadlock logging further
+-   Discard inbound federation traffic from invalid domains, to immunise against \#1753
+
+Changes in synapse v0.18.6 (2017-01-06)
+=======================================
+
+Bug fixes:
+
+-   Fix bug when checking if a guest user is allowed to join a room (PR \#1772) Thanks to Patrik Oldsberg for diagnosing and the fix!
+
+Changes in synapse v0.18.6-rc3 (2017-01-05)
+===========================================
+
+Bug fixes:
+
+-   Fix bug where we failed to send ban events to the banned server (PR \#1758)
+-   Fix bug where we sent event that didn\'t originate on this server to other servers (PR \#1764)
+-   Fix bug where processing an event from a remote server took a long time because we were making long HTTP requests (PR \#1765, PR \#1744)
+
+Changes:
+
+-   Improve logging for debugging deadlocks (PR \#1766, PR \#1767)
+
+Changes in synapse v0.18.6-rc2 (2016-12-30)
+===========================================
+
+Bug fixes:
+
+-   Fix memory leak in twisted by initialising logging correctly (PR \#1731)
+-   Fix bug where fetching missing events took an unacceptable amount of time in large rooms (PR \#1734)
+
+Changes in synapse v0.18.6-rc1 (2016-12-29)
+===========================================
+
+Bug fixes:
+
+-   Make sure that outbound connections are closed (PR \#1725)
+
+Changes in synapse v0.18.5 (2016-12-16)
+=======================================
+
+Bug fixes:
+
+-   Fix federation /backfill returning events it shouldn\'t (PR \#1700)
+-   Fix crash in url preview (PR \#1701)
+
+Changes in synapse v0.18.5-rc3 (2016-12-13)
+===========================================
+
+Features:
+
+-   Add support for E2E for guests (PR \#1653)
+-   Add new API appservice specific public room list (PR \#1676)
+-   Add new room membership APIs (PR \#1680)
+
+Changes:
+
+-   Enable guest access for private rooms by default (PR \#653)
+-   Limit the number of events that can be created on a given room concurrently (PR \#1620)
+-   Log the args that we have on UI auth completion (PR \#1649)
+-   Stop generating refresh\_tokens (PR \#1654)
+-   Stop putting a time caveat on access tokens (PR \#1656)
+-   Remove unspecced GET endpoints for e2e keys (PR \#1694)
+
+Bug fixes:
+
+-   Fix handling of 500 and 429\'s over federation (PR \#1650)
+-   Fix Content-Type header parsing (PR \#1660)
+-   Fix error when previewing sites that include unicode, thanks to kyrias (PR \#1664)
+-   Fix some cases where we drop read receipts (PR \#1678)
+-   Fix bug where calls to `/sync` didn\'t correctly timeout (PR \#1683)
+-   Fix bug where E2E key query would fail if a single remote host failed (PR \#1686)
+
+Changes in synapse v0.18.5-rc2 (2016-11-24)
+===========================================
+
+Bug fixes:
+
+-   Don\'t send old events over federation, fixes bug in -rc1.
+
+Changes in synapse v0.18.5-rc1 (2016-11-24)
+===========================================
+
+Features:
+
+-   Implement \"event\_fields\" in filters (PR \#1638)
+
+Changes:
+
+-   Use external ldap auth pacakge (PR \#1628)
+-   Split out federation transaction sending to a worker (PR \#1635)
+-   Fail with a coherent error message if /sync?filter= is invalid (PR \#1636)
+-   More efficient notif count queries (PR \#1644)
+
+Changes in synapse v0.18.4 (2016-11-22)
+=======================================
+
+Bug fixes:
+
+-   Add workaround for buggy clients that the fail to register (PR \#1632)
+
+Changes in synapse v0.18.4-rc1 (2016-11-14)
+===========================================
+
+Changes:
+
+-   Various database efficiency improvements (PR \#1188, \#1192)
+-   Update default config to blacklist more internal IPs, thanks to Euan Kemp (PR \#1198)
+-   Allow specifying duration in minutes in config, thanks to Daniel Dent (PR \#1625)
+
+Bug fixes:
+
+-   Fix media repo to set CORs headers on responses (PR \#1190)
+-   Fix registration to not error on non-ascii passwords (PR \#1191)
+-   Fix create event code to limit the number of prev\_events (PR \#1615)
+-   Fix bug in transaction ID deduplication (PR \#1624)
+
+Changes in synapse v0.18.3 (2016-11-08)
+=======================================
+
+SECURITY UPDATE
+
+Explicitly require authentication when using LDAP3. This is the default on versions of `ldap3` above 1.0, but some distributions will package an older version.
+
+If you are using LDAP3 login and have a version of `ldap3` older than 1.0 it is **CRITICAL to updgrade**.
+
+Changes in synapse v0.18.2 (2016-11-01)
+=======================================
+
+No changes since v0.18.2-rc5
+
+Changes in synapse v0.18.2-rc5 (2016-10-28)
+===========================================
+
+Bug fixes:
+
+-   Fix prometheus process metrics in worker processes (PR \#1184)
+
+Changes in synapse v0.18.2-rc4 (2016-10-27)
+===========================================
+
+Bug fixes:
+
+-   Fix `user_threepids` schema delta, which in some instances prevented startup after upgrade (PR \#1183)
+
+Changes in synapse v0.18.2-rc3 (2016-10-27)
+===========================================
+
+Changes:
+
+-   Allow clients to supply access tokens as headers (PR \#1098)
+-   Clarify error codes for GET /filter/, thanks to Alexander Maznev (PR \#1164)
+-   Make password reset email field case insensitive (PR \#1170)
+-   Reduce redundant database work in email pusher (PR \#1174)
+-   Allow configurable rate limiting per AS (PR \#1175)
+-   Check whether to ratelimit sooner to avoid work (PR \#1176)
+-   Standardise prometheus metrics (PR \#1177)
+
+Bug fixes:
+
+-   Fix incredibly slow back pagination query (PR \#1178)
+-   Fix infinite typing bug (PR \#1179)
+
+Changes in synapse v0.18.2-rc2 (2016-10-25)
+===========================================
+
+(This release did not include the changes advertised and was identical to RC1)
+
+Changes in synapse v0.18.2-rc1 (2016-10-17)
+===========================================
+
+Changes:
+
+-   Remove redundant event\_auth index (PR \#1113)
+-   Reduce DB hits for replication (PR \#1141)
+-   Implement pluggable password auth (PR \#1155)
+-   Remove rate limiting from app service senders and fix get\_or\_create\_user requester, thanks to Patrik Oldsberg (PR \#1157)
+-   window.postmessage for Interactive Auth fallback (PR \#1159)
+-   Use sys.executable instead of hardcoded python, thanks to Pedro Larroy (PR \#1162)
+-   Add config option for adding additional TLS fingerprints (PR \#1167)
+-   User-interactive auth on delete device (PR \#1168)
+
+Bug fixes:
+
+-   Fix not being allowed to set your own state\_key, thanks to Patrik Oldsberg (PR \#1150)
+-   Fix interactive auth to return 401 from for incorrect password (PR \#1160, \#1166)
+-   Fix email push notifs being dropped (PR \#1169)
+
+Changes in synapse v0.18.1 (2016-10-05) ======================================
+
+No changes since v0.18.1-rc1
+
+Changes in synapse v0.18.1-rc1 (2016-09-30)
+===========================================
+
+Features:
+
+-   Add total\_room\_count\_estimate to `/publicRooms` (PR \#1133)
+
+Changes:
+
+-   Time out typing over federation (PR \#1140)
+-   Restructure LDAP authentication (PR \#1153)
+
+Bug fixes:
+
+-   Fix 3pid invites when server is already in the room (PR \#1136)
+-   Fix upgrading with SQLite taking lots of CPU for a few days after upgrade (PR \#1144)
+-   Fix upgrading from very old database versions (PR \#1145)
+-   Fix port script to work with recently added tables (PR \#1146)
+
+Changes in synapse v0.18.0 (2016-09-19)
+=======================================
+
+The release includes major changes to the state storage database schemas, which significantly reduce database size. Synapse will attempt to upgrade the current data in the background. Servers with large SQLite database may experience degradation of performance while this upgrade is in progress, therefore you may want to consider migrating to using Postgres before upgrading very large SQLite databases
+
+Changes:
+
+-   Make public room search case insensitive (PR \#1127)
+
+Bug fixes:
+
+-   Fix and clean up publicRooms pagination (PR \#1129)
+
+Changes in synapse v0.18.0-rc1 (2016-09-16)
+===========================================
+
+Features:
+
+-   Add `only=highlight` on `/notifications` (PR \#1081)
+-   Add server param to /publicRooms (PR \#1082)
+-   Allow clients to ask for the whole of a single state event (PR \#1094)
+-   Add is\_direct param to /createRoom (PR \#1108)
+-   Add pagination support to publicRooms (PR \#1121)
+-   Add very basic filter API to /publicRooms (PR \#1126)
+-   Add basic direct to device messaging support for E2E (PR \#1074, \#1084, \#1104, \#1111)
+
+Changes:
+
+-   Move to storing state\_groups\_state as deltas, greatly reducing DB size (PR \#1065)
+-   Reduce amount of state pulled out of the DB during common requests (PR \#1069)
+-   Allow PDF to be rendered from media repo (PR \#1071)
+-   Reindex state\_groups\_state after pruning (PR \#1085)
+-   Clobber EDUs in send queue (PR \#1095)
+-   Conform better to the CAS protocol specification (PR \#1100)
+-   Limit how often we ask for keys from dead servers (PR \#1114)
+
+Bug fixes:
+
+-   Fix /notifications API when used with `from` param (PR \#1080)
+-   Fix backfill when cannot find an event. (PR \#1107)
+
+Changes in synapse v0.17.3 (2016-09-09)
+=======================================
+
+This release fixes a major bug that stopped servers from handling rooms with over 1000 members.
+
+Changes in synapse v0.17.2 (2016-09-08)
+=======================================
+
+This release contains security bug fixes. Please upgrade.
+
+No changes since v0.17.2-rc1
+
+Changes in synapse v0.17.2-rc1 (2016-09-05)
+===========================================
+
+Features:
+
+-   Start adding store-and-forward direct-to-device messaging (PR \#1046, \#1050, \#1062, \#1066)
+
+Changes:
+
+-   Avoid pulling the full state of a room out so often (PR \#1047, \#1049, \#1063, \#1068)
+-   Don\'t notify for online to online presence transitions. (PR \#1054)
+-   Occasionally persist unpersisted presence updates (PR \#1055)
+-   Allow application services to have an optional \'url\' (PR \#1056)
+-   Clean up old sent transactions from DB (PR \#1059)
+
+Bug fixes:
+
+-   Fix None check in backfill (PR \#1043)
+-   Fix membership changes to be idempotent (PR \#1067)
+-   Fix bug in get\_pdu where it would sometimes return events with incorrect signature
+
+Changes in synapse v0.17.1 (2016-08-24)
+=======================================
+
+Changes:
+
+-   Delete old received\_transactions rows (PR \#1038)
+-   Pass through user-supplied content in /join/\$room\_id (PR \#1039)
+
+Bug fixes:
+
+-   Fix bug with backfill (PR \#1040)
+
+Changes in synapse v0.17.1-rc1 (2016-08-22)
+===========================================
+
+Features:
+
+-   Add notification API (PR \#1028)
+
+Changes:
+
+-   Don\'t print stack traces when failing to get remote keys (PR \#996)
+-   Various federation /event/ perf improvements (PR \#998)
+-   Only process one local membership event per room at a time (PR \#1005)
+-   Move default display name push rule (PR \#1011, \#1023)
+-   Fix up preview URL API. Add tests. (PR \#1015)
+-   Set `Content-Security-Policy` on media repo (PR \#1021)
+-   Make notify\_interested\_services faster (PR \#1022)
+-   Add usage stats to prometheus monitoring (PR \#1037)
+
+Bug fixes:
+
+-   Fix token login (PR \#993)
+-   Fix CAS login (PR \#994, \#995)
+-   Fix /sync to not clobber status\_msg (PR \#997)
+-   Fix redacted state events to include prev\_content (PR \#1003)
+-   Fix some bugs in the auth/ldap handler (PR \#1007)
+-   Fix backfill request to limit URI length, so that remotes don\'t reject the requests due to path length limits (PR \#1012)
+-   Fix AS push code to not send duplicate events (PR \#1025)
+
+Changes in synapse v0.17.0 (2016-08-08)
+=======================================
+
+This release contains significant security bug fixes regarding authenticating events received over federation. PLEASE UPGRADE.
+
+This release changes the LDAP configuration format in a backwards incompatible way, see PR \#843 for details.
+
+Changes:
+
+-   Add federation /version API (PR \#990)
+-   Make psutil dependency optional (PR \#992)
+
+Bug fixes:
+
+-   Fix URL preview API to exclude HTML comments in description (PR \#988)
+-   Fix error handling of remote joins (PR \#991)
+
+Changes in synapse v0.17.0-rc4 (2016-08-05)
+===========================================
+
+Changes:
+
+-   Change the way we summarize URLs when previewing (PR \#973)
+-   Add new `/state_ids/` federation API (PR \#979)
+-   Speed up processing of `/state/` response (PR \#986)
+
+Bug fixes:
+
+-   Fix event persistence when event has already been partially persisted (PR \#975, \#983, \#985)
+-   Fix port script to also copy across backfilled events (PR \#982)
+
+Changes in synapse v0.17.0-rc3 (2016-08-02)
+===========================================
+
+Changes:
+
+-   Forbid non-ASes from registering users whose names begin with \'\_\' (PR \#958)
+-   Add some basic admin API docs (PR \#963)
+
+Bug fixes:
+
+-   Send the correct host header when fetching keys (PR \#941)
+-   Fix joining a room that has missing auth events (PR \#964)
+-   Fix various push bugs (PR \#966, \#970)
+-   Fix adding emails on registration (PR \#968)
+
+Changes in synapse v0.17.0-rc2 (2016-08-02)
+===========================================
+
+(This release did not include the changes advertised and was identical to RC1)
+
+Changes in synapse v0.17.0-rc1 (2016-07-28)
+===========================================
+
+This release changes the LDAP configuration format in a backwards incompatible way, see PR \#843 for details.
+
+Features:
+
+-   Add purge\_media\_cache admin API (PR \#902)
+-   Add deactivate account admin API (PR \#903)
+-   Add optional pepper to password hashing (PR \#907, \#910 by KentShikama)
+-   Add an admin option to shared secret registration (breaks backwards compat) (PR \#909)
+-   Add purge local room history API (PR \#911, \#923, \#924)
+-   Add requestToken endpoints (PR \#915)
+-   Add an /account/deactivate endpoint (PR \#921)
+-   Add filter param to /messages. Add \'contains\_url\' to filter. (PR \#922)
+-   Add device\_id support to /login (PR \#929)
+-   Add device\_id support to /v2/register flow. (PR \#937, \#942)
+-   Add GET /devices endpoint (PR \#939, \#944)
+-   Add GET /device/{deviceId} (PR \#943)
+-   Add update and delete APIs for devices (PR \#949)
+
+Changes:
+
+-   Rewrite LDAP Authentication against ldap3 (PR \#843 by mweinelt)
+-   Linearize some federation endpoints based on (origin, room\_id) (PR \#879)
+-   Remove the legacy v0 content upload API. (PR \#888)
+-   Use similar naming we use in email notifs for push (PR \#894)
+-   Optionally include password hash in createUser endpoint (PR \#905 by KentShikama)
+-   Use a query that postgresql optimises better for get\_events\_around (PR \#906)
+-   Fall back to \'username\' if \'user\' is not given for appservice registration. (PR \#927 by Half-Shot)
+-   Add metrics for psutil derived memory usage (PR \#936)
+-   Record device\_id in client\_ips (PR \#938)
+-   Send the correct host header when fetching keys (PR \#941)
+-   Log the hostname the reCAPTCHA was completed on (PR \#946)
+-   Make the device id on e2e key upload optional (PR \#956)
+-   Add r0.2.0 to the \"supported versions\" list (PR \#960)
+-   Don\'t include name of room for invites in push (PR \#961)
+
+Bug fixes:
+
+-   Fix substitution failure in mail template (PR \#887)
+-   Put most recent 20 messages in email notif (PR \#892)
+-   Ensure that the guest user is in the database when upgrading accounts (PR \#914)
+-   Fix various edge cases in auth handling (PR \#919)
+-   Fix 500 ISE when sending alias event without a state\_key (PR \#925)
+-   Fix bug where we stored rejections in the state\_group, persist all rejections (PR \#948)
+-   Fix lack of check of if the user is banned when handling 3pid invites (PR \#952)
+-   Fix a couple of bugs in the transaction and keyring code (PR \#954, \#955)
+
+Changes in synapse v0.16.1-r1 (2016-07-08)
+==========================================
+
+THIS IS A CRITICAL SECURITY UPDATE.
+
+This fixes a bug which allowed users\' accounts to be accessed by unauthorised users.
+
+Changes in synapse v0.16.1 (2016-06-20)
+=======================================
+
+Bug fixes:
+
+-   Fix assorted bugs in `/preview_url` (PR \#872)
+-   Fix TypeError when setting unicode passwords (PR \#873)
+
+Performance improvements:
+
+-   Turn `use_frozen_events` off by default (PR \#877)
+-   Disable responding with canonical json for federation (PR \#878)
+
+Changes in synapse v0.16.1-rc1 (2016-06-15)
+===========================================
+
+Features: None
+
+Changes:
+
+-   Log requester for `/publicRoom` endpoints when possible (PR \#856)
+-   502 on `/thumbnail` when can\'t connect to remote server (PR \#862)
+-   Linearize fetching of gaps on incoming events (PR \#871)
+
+Bugs fixes:
+
+-   Fix bug where rooms where marked as published by default (PR \#857)
+-   Fix bug where joining room with an event with invalid sender (PR \#868)
+-   Fix bug where backfilled events were sent down sync streams (PR \#869)
+-   Fix bug where outgoing connections could wedge indefinitely, causing push notifications to be unreliable (PR \#870)
+
+Performance improvements:
+
+-   Improve `/publicRooms` performance(PR \#859)
+
+Changes in synapse v0.16.0 (2016-06-09)
+=======================================
+
+NB: As of v0.14 all AS config files must have an ID field.
+
+Bug fixes:
+
+-   Don\'t make rooms published by default (PR \#857)
+
+Changes in synapse v0.16.0-rc2 (2016-06-08)
+===========================================
+
+Features:
+
+-   Add configuration option for tuning GC via `gc.set_threshold` (PR \#849)
+
+Changes:
+
+-   Record metrics about GC (PR \#771, \#847, \#852)
+-   Add metric counter for number of persisted events (PR \#841)
+
+Bug fixes:
+
+-   Fix \'From\' header in email notifications (PR \#843)
+-   Fix presence where timeouts were not being fired for the first 8h after restarts (PR \#842)
+-   Fix bug where synapse sent malformed transactions to AS\'s when retrying transactions (Commits 310197b, 8437906)
+
+Performance improvements:
+
+-   Remove event fetching from DB threads (PR \#835)
+-   Change the way we cache events (PR \#836)
+-   Add events to cache when we persist them (PR \#840)
+
+Changes in synapse v0.16.0-rc1 (2016-06-03)
+===========================================
+
+Version 0.15 was not released. See v0.15.0-rc1 below for additional changes.
+
+Features:
+
+-   Add email notifications for missed messages (PR \#759, \#786, \#799, \#810, \#815, \#821)
+-   Add a `url_preview_ip_range_whitelist` config param (PR \#760)
+-   Add /report endpoint (PR \#762)
+-   Add basic ignore user API (PR \#763)
+-   Add an openidish mechanism for proving that you own a given user\_id (PR \#765)
+-   Allow clients to specify a server\_name to avoid \'No known servers\' (PR \#794)
+-   Add secondary\_directory\_servers option to fetch room list from other servers (PR \#808, \#813)
+
+Changes:
+
+-   Report per request metrics for all of the things using request\_handler (PR \#756)
+-   Correctly handle `NULL` password hashes from the database (PR \#775)
+-   Allow receipts for events we haven\'t seen in the db (PR \#784)
+-   Make synctl read a cache factor from config file (PR \#785)
+-   Increment badge count per missed convo, not per msg (PR \#793)
+-   Special case m.room.third\_party\_invite event auth to match invites (PR \#814)
+
+Bug fixes:
+
+-   Fix typo in event\_auth servlet path (PR \#757)
+-   Fix password reset (PR \#758)
+
+Performance improvements:
+
+-   Reduce database inserts when sending transactions (PR \#767)
+-   Queue events by room for persistence (PR \#768)
+-   Add cache to `get_user_by_id` (PR \#772)
+-   Add and use `get_domain_from_id` (PR \#773)
+-   Use tree cache for `get_linearized_receipts_for_room` (PR \#779)
+-   Remove unused indices (PR \#782)
+-   Add caches to `bulk_get_push_rules*` (PR \#804)
+-   Cache `get_event_reference_hashes` (PR \#806)
+-   Add `get_users_with_read_receipts_in_room` cache (PR \#809)
+-   Use state to calculate `get_users_in_room` (PR \#811)
+-   Load push rules in storage layer so that they get cached (PR \#825)
+-   Make `get_joined_hosts_for_room` use get\_users\_in\_room (PR \#828)
+-   Poke notifier on next reactor tick (PR \#829)
+-   Change CacheMetrics to be quicker (PR \#830)
+
+Changes in synapse v0.15.0-rc1 (2016-04-26)
+===========================================
+
+Features:
+
+-   Add login support for Javascript Web Tokens, thanks to Niklas Riekenbrauck (PR \#671,\#687)
+-   Add URL previewing support (PR \#688)
+-   Add login support for LDAP, thanks to Christoph Witzany (PR \#701)
+-   Add GET endpoint for pushers (PR \#716)
+
+Changes:
+
+-   Never notify for member events (PR \#667)
+-   Deduplicate identical `/sync` requests (PR \#668)
+-   Require user to have left room to forget room (PR \#673)
+-   Use DNS cache if within TTL (PR \#677)
+-   Let users see their own leave events (PR \#699)
+-   Deduplicate membership changes (PR \#700)
+-   Increase performance of pusher code (PR \#705)
+-   Respond with error status 504 if failed to talk to remote server (PR \#731)
+-   Increase search performance on postgres (PR \#745)
+
+Bug fixes:
+
+-   Fix bug where disabling all notifications still resulted in push (PR \#678)
+-   Fix bug where users couldn\'t reject remote invites if remote refused (PR \#691)
+-   Fix bug where synapse attempted to backfill from itself (PR \#693)
+-   Fix bug where profile information was not correctly added when joining remote rooms (PR \#703)
+-   Fix bug where register API required incorrect key name for AS registration (PR \#727)
+
+Changes in synapse v0.14.0 (2016-03-30)
+=======================================
+
+No changes from v0.14.0-rc2
+
+Changes in synapse v0.14.0-rc2 (2016-03-23)
+===========================================
+
+Features:
+
+-   Add published room list API (PR \#657)
+
+Changes:
+
+-   Change various caches to consume less memory (PR \#656, \#658, \#660, \#662, \#663, \#665)
+-   Allow rooms to be published without requiring an alias (PR \#664)
+-   Intern common strings in caches to reduce memory footprint (\#666)
+
+Bug fixes:
+
+-   Fix reject invites over federation (PR \#646)
+-   Fix bug where registration was not idempotent (PR \#649)
+-   Update aliases event after deleting aliases (PR \#652)
+-   Fix unread notification count, which was sometimes wrong (PR \#661)
+
+Changes in synapse v0.14.0-rc1 (2016-03-14)
+===========================================
+
+Features:
+
+-   Add event\_id to response to state event PUT (PR \#581)
+-   Allow guest users access to messages in rooms they have joined (PR \#587)
+-   Add config for what state is included in a room invite (PR \#598)
+-   Send the inviter\'s member event in room invite state (PR \#607)
+-   Add error codes for malformed/bad JSON in /login (PR \#608)
+-   Add support for changing the actions for default rules (PR \#609)
+-   Add environment variable SYNAPSE\_CACHE\_FACTOR, default it to 0.1 (PR \#612)
+-   Add ability for alias creators to delete aliases (PR \#614)
+-   Add profile information to invites (PR \#624)
+
+Changes:
+
+-   Enforce user\_id exclusivity for AS registrations (PR \#572)
+-   Make adding push rules idempotent (PR \#587)
+-   Improve presence performance (PR \#582, \#586)
+-   Change presence semantics for `last_active_ago` (PR \#582, \#586)
+-   Don\'t allow `m.room.create` to be changed (PR \#596)
+-   Add 800x600 to default list of valid thumbnail sizes (PR \#616)
+-   Always include kicks and bans in full /sync (PR \#625)
+-   Send history visibility on boundary changes (PR \#626)
+-   Register endpoint now returns a refresh\_token (PR \#637)
+
+Bug fixes:
+
+-   Fix bug where we returned incorrect state in /sync (PR \#573)
+-   Always return a JSON object from push rule API (PR \#606)
+-   Fix bug where registering without a user id sometimes failed (PR \#610)
+-   Report size of ExpiringCache in cache size metrics (PR \#611)
+-   Fix rejection of invites to empty rooms (PR \#615)
+-   Fix usage of `bcrypt` to not use `checkpw` (PR \#619)
+-   Pin `pysaml2` dependency (PR \#634)
+-   Fix bug in `/sync` where timeline order was incorrect for backfilled events (PR \#635)
+
+Changes in synapse v0.13.3 (2016-02-11)
+=======================================
+
+-   Fix bug where `/sync` would occasionally return events in the wrong room.
+
+Changes in synapse v0.13.2 (2016-02-11)
+=======================================
+
+-   Fix bug where `/events` would fail to skip some events if there had been more events than the limit specified since the last request (PR \#570)
+
+Changes in synapse v0.13.1 (2016-02-10)
+=======================================
+
+-   Bump matrix-angular-sdk (matrix web console) dependency to 0.6.8 to pull in the fix for SYWEB-361 so that the default client can display HTML messages again(!)
+
+Changes in synapse v0.13.0 (2016-02-10)
+=======================================
+
+This version includes an upgrade of the schema, specifically adding an index to the `events` table. This may cause synapse to pause for several minutes the first time it is started after the upgrade.
+
+Changes:
+
+-   Improve general performance (PR \#540, \#543. \#544, \#54, \#549, \#567)
+-   Change guest user ids to be incrementing integers (PR \#550)
+-   Improve performance of public room list API (PR \#552)
+-   Change profile API to omit keys rather than return null (PR \#557)
+-   Add `/media/r0` endpoint prefix, which is equivalent to `/media/v1/` (PR \#595)
+
+Bug fixes:
+
+-   Fix bug with upgrading guest accounts where it would fail if you opened the registration email on a different device (PR \#547)
+-   Fix bug where unread count could be wrong (PR \#568)
+
+Changes in synapse v0.12.1-rc1 (2016-01-29)
+===========================================
+
+Features:
+
+-   Add unread notification counts in `/sync` (PR \#456)
+-   Add support for inviting 3pids in `/createRoom` (PR \#460)
+-   Add ability for guest accounts to upgrade (PR \#462)
+-   Add `/versions` API (PR \#468)
+-   Add `event` to `/context` API (PR \#492)
+-   Add specific error code for invalid user names in `/register` (PR \#499)
+-   Add support for push badge counts (PR \#507)
+-   Add support for non-guest users to peek in rooms using `/events` (PR \#510)
+
+Changes:
+
+-   Change `/sync` so that guest users only get rooms they\'ve joined (PR \#469)
+-   Change to require unbanning before other membership changes (PR \#501)
+-   Change default push rules to notify for all messages (PR \#486)
+-   Change default push rules to not notify on membership changes (PR \#514)
+-   Change default push rules in one to one rooms to only notify for events that are messages (PR \#529)
+-   Change `/sync` to reject requests with a `from` query param (PR \#512)
+-   Change server manhole to use SSH rather than telnet (PR \#473)
+-   Change server to require AS users to be registered before use (PR \#487)
+-   Change server not to start when ASes are invalidly configured (PR \#494)
+-   Change server to require ID and `as_token` to be unique for AS\'s (PR \#496)
+-   Change maximum pagination limit to 1000 (PR \#497)
+
+Bug fixes:
+
+-   Fix bug where `/sync` didn\'t return when something under the leave key changed (PR \#461)
+-   Fix bug where we returned smaller rather than larger than requested thumbnails when `method=crop` (PR \#464)
+-   Fix thumbnails API to only return cropped thumbnails when asking for a cropped thumbnail (PR \#475)
+-   Fix bug where we occasionally still logged access tokens (PR \#477)
+-   Fix bug where `/events` would always return immediately for guest users (PR \#480)
+-   Fix bug where `/sync` unexpectedly returned old left rooms (PR \#481)
+-   Fix enabling and disabling push rules (PR \#498)
+-   Fix bug where `/register` returned 500 when given unicode username (PR \#513)
+
+Changes in synapse v0.12.0 (2016-01-04)
+=======================================
+
+-   Expose `/login` under `r0` (PR \#459)
+
+Changes in synapse v0.12.0-rc3 (2015-12-23)
+===========================================
+
+-   Allow guest accounts access to `/sync` (PR \#455)
+-   Allow filters to include/exclude rooms at the room level rather than just from the components of the sync for each room. (PR \#454)
+-   Include urls for room avatars in the response to `/publicRooms` (PR \#453)
+-   Don\'t set a identicon as the avatar for a user when they register (PR \#450)
+-   Add a `display_name` to third-party invites (PR \#449)
+-   Send more information to the identity server for third-party invites so that it can send richer messages to the invitee (PR \#446)
+-   Cache the responses to `/initialSync` for 5 minutes. If a client retries a request to `/initialSync` before the a response was computed to the first request then the same response is used for both requests (PR \#457)
+-   Fix a bug where synapse would always request the signing keys of remote servers even when the key was cached locally (PR \#452)
+-   Fix 500 when pagination search results (PR \#447)
+-   Fix a bug where synapse was leaking raw email address in third-party invites (PR \#448)
+
+Changes in synapse v0.12.0-rc2 (2015-12-14)
+===========================================
+
+-   Add caches for whether rooms have been forgotten by a user (PR \#434)
+-   Remove instructions to use `--process-dependency-link` since all of the dependencies of synapse are on PyPI (PR \#436)
+-   Parallelise the processing of `/sync` requests (PR \#437)
+-   Fix race updating presence in `/events` (PR \#444)
+-   Fix bug back-populating search results (PR \#441)
+-   Fix bug calculating state in `/sync` requests (PR \#442)
+
+Changes in synapse v0.12.0-rc1 (2015-12-10)
+===========================================
+
+-   Host the client APIs released as r0 by <https://matrix.org/docs/spec/r0.0.0/client_server.html> on paths prefixed by `/_matrix/client/r0`. (PR \#430, PR \#415, PR \#400)
+-   Updates the client APIs to match r0 of the matrix specification.
+    -   All APIs return events in the new event format, old APIs also include the fields needed to parse the event using the old format for compatibility. (PR \#402)
+    -   Search results are now given as a JSON array rather than a JSON object (PR \#405)
+    -   Miscellaneous changes to search (PR \#403, PR \#406, PR \#412)
+    -   Filter JSON objects may now be passed as query parameters to `/sync` (PR \#431)
+    -   Fix implementation of `/admin/whois` (PR \#418)
+    -   Only include the rooms that user has left in `/sync` if the client requests them in the filter (PR \#423)
+    -   Don\'t push for `m.room.message` by default (PR \#411)
+    -   Add API for setting per account user data (PR \#392)
+    -   Allow users to forget rooms (PR \#385)
+-   Performance improvements and monitoring:
+    -   Add per-request counters for CPU time spent on the main python thread. (PR \#421, PR \#420)
+    -   Add per-request counters for time spent in the database (PR \#429)
+    -   Make state updates in the C+S API idempotent (PR \#416)
+    -   Only fire `user_joined_room` if the user has actually joined. (PR \#410)
+    -   Reuse a single http client, rather than creating new ones (PR \#413)
+-   Fixed a bug upgrading from older versions of synapse on postgresql (PR \#417)
+
+Changes in synapse v0.11.1 (2015-11-20)
+=======================================
+
+-   Add extra options to search API (PR \#394)
+-   Fix bug where we did not correctly cap federation retry timers. This meant it could take several hours for servers to start talking to ressurected servers, even when they were receiving traffic from them (PR \#393)
+-   Don\'t advertise login token flow unless CAS is enabled. This caused issues where some clients would always use the fallback API if they did not recognize all login flows (PR \#391)
+-   Change /v2 sync API to rename `private_user_data` to `account_data` (PR \#386)
+-   Change /v2 sync API to remove the `event_map` and rename keys in `rooms` object (PR \#389)
+
+Changes in synapse v0.11.0-r2 (2015-11-19)
+==========================================
+
+-   Fix bug in database port script (PR \#387)
+
+Changes in synapse v0.11.0-r1 (2015-11-18)
+==========================================
+
+-   Retry and fail federation requests more aggressively for requests that block client side requests (PR \#384)
+
+Changes in synapse v0.11.0 (2015-11-17)
+=======================================
+
+-   Change CAS login API (PR \#349)
+
+Changes in synapse v0.11.0-rc2 (2015-11-13)
+===========================================
+
+-   Various changes to /sync API response format (PR \#373)
+-   Fix regression when setting display name in newly joined room over federation (PR \#368)
+-   Fix problem where /search was slow when using SQLite (PR \#366)
+
+Changes in synapse v0.11.0-rc1 (2015-11-11)
+===========================================
+
+-   Add Search API (PR \#307, \#324, \#327, \#336, \#350, \#359)
+-   Add \'archived\' state to v2 /sync API (PR \#316)
+-   Add ability to reject invites (PR \#317)
+-   Add config option to disable password login (PR \#322)
+-   Add the login fallback API (PR \#330)
+-   Add room context API (PR \#334)
+-   Add room tagging support (PR \#335)
+-   Update v2 /sync API to match spec (PR \#305, \#316, \#321, \#332, \#337, \#341)
+-   Change retry schedule for application services (PR \#320)
+-   Change retry schedule for remote servers (PR \#340)
+-   Fix bug where we hosted static content in the incorrect place (PR \#329)
+-   Fix bug where we didn\'t increment retry interval for remote servers (PR \#343)
+
+Changes in synapse v0.10.1-rc1 (2015-10-15)
+===========================================
+
+-   Add support for CAS, thanks to Steven Hammerton (PR \#295, \#296)
+-   Add support for using macaroons for `access_token` (PR \#256, \#229)
+-   Add support for `m.room.canonical_alias` (PR \#287)
+-   Add support for viewing the history of rooms that they have left. (PR \#276, \#294)
+-   Add support for refresh tokens (PR \#240)
+-   Add flag on creation which disables federation of the room (PR \#279)
+-   Add some room state to invites. (PR \#275)
+-   Atomically persist events when joining a room over federation (PR \#283)
+-   Change default history visibility for private rooms (PR \#271)
+-   Allow users to redact their own sent events (PR \#262)
+-   Use tox for tests (PR \#247)
+-   Split up syutil into separate libraries (PR \#243)
+
+Changes in synapse v0.10.0-r2 (2015-09-16)
+==========================================
+
+-   Fix bug where we always fetched remote server signing keys instead of using ones in our cache.
+-   Fix adding threepids to an existing account.
+-   Fix bug with invinting over federation where remote server was already in the room. (PR \#281, SYN-392)
+
+Changes in synapse v0.10.0-r1 (2015-09-08)
+==========================================
+
+-   Fix bug with python packaging
+
+Changes in synapse v0.10.0 (2015-09-03)
+=======================================
+
+No change from release candidate.
+
+Changes in synapse v0.10.0-rc6 (2015-09-02)
+===========================================
+
+-   Remove some of the old database upgrade scripts.
+-   Fix database port script to work with newly created sqlite databases.
+
+Changes in synapse v0.10.0-rc5 (2015-08-27)
+===========================================
+
+-   Fix bug that broke downloading files with ascii filenames across federation.
+
+Changes in synapse v0.10.0-rc4 (2015-08-27)
+===========================================
+
+-   Allow UTF-8 filenames for upload. (PR \#259)
+
+Changes in synapse v0.10.0-rc3 (2015-08-25)
+===========================================
+
+-   Add `--keys-directory` config option to specify where files such as certs and signing keys should be stored in, when using `--generate-config` or `--generate-keys`. (PR \#250)
+-   Allow `--config-path` to specify a directory, causing synapse to use all \*.yaml files in the directory as config files. (PR \#249)
+-   Add `web_client_location` config option to specify static files to be hosted by synapse under `/_matrix/client`. (PR \#245)
+-   Add helper utility to synapse to read and parse the config files and extract the value of a given key. For example:
+
+        $ python -m synapse.config read server_name -c homeserver.yaml
+        localhost
+
+    (PR \#246)
+
+Changes in synapse v0.10.0-rc2 (2015-08-24)
+===========================================
+
+-   Fix bug where we incorrectly populated the `event_forward_extremities` table, resulting in problems joining large remote rooms (e.g. `#matrix:matrix.org`)
+-   Reduce the number of times we wake up pushers by not listening for presence or typing events, reducing the CPU cost of each pusher.
+
+Changes in synapse v0.10.0-rc1 (2015-08-21)
+===========================================
+
+Also see v0.9.4-rc1 changelog, which has been amalgamated into this release.
+
+General:
+
+-   Upgrade to Twisted 15 (PR \#173)
+-   Add support for serving and fetching encryption keys over federation. (PR \#208)
+-   Add support for logging in with email address (PR \#234)
+-   Add support for new `m.room.canonical_alias` event. (PR \#233)
+-   Change synapse to treat user IDs case insensitively during registration and login. (If two users already exist with case insensitive matching user ids, synapse will continue to require them to specify their user ids exactly.)
+-   Error if a user tries to register with an email already in use. (PR \#211)
+-   Add extra and improve existing caches (PR \#212, \#219, \#226, \#228)
+-   Batch various storage request (PR \#226, \#228)
+-   Fix bug where we didn\'t correctly log the entity that triggered the request if the request came in via an application service (PR \#230)
+-   Fix bug where we needlessly regenerated the full list of rooms an AS is interested in. (PR \#232)
+-   Add support for AS\'s to use v2\_alpha registration API (PR \#210)
+
+Configuration:
+
+-   Add `--generate-keys` that will generate any missing cert and key files in the configuration files. This is equivalent to running `--generate-config` on an existing configuration file. (PR \#220)
+-   `--generate-config` now no longer requires a `--server-name` parameter when used on existing configuration files. (PR \#220)
+-   Add `--print-pidfile` flag that controls the printing of the pid to stdout of the demonised process. (PR \#213)
+
+Media Repository:
+
+-   Fix bug where we picked a lower resolution image than requested. (PR \#205)
+-   Add support for specifying if a the media repository should dynamically thumbnail images or not. (PR \#206)
+
+Metrics:
+
+-   Add statistics from the reactor to the metrics API. (PR \#224, \#225)
+
+Demo Homeservers:
+
+-   Fix starting the demo homeservers without rate-limiting enabled. (PR \#182)
+-   Fix enabling registration on demo homeservers (PR \#223)
+
+Changes in synapse v0.9.4-rc1 (2015-07-21)
+==========================================
+
+General:
+
+-   Add basic implementation of receipts. (SPEC-99)
+-   Add support for configuration presets in room creation API. (PR \#203)
+-   Add auth event that limits the visibility of history for new users. (SPEC-134)
+-   Add SAML2 login/registration support. (PR \#201. Thanks Muthu Subramanian!)
+-   Add client side key management APIs for end to end encryption. (PR \#198)
+-   Change power level semantics so that you cannot kick, ban or change power levels of users that have equal or greater power level than you. (SYN-192)
+-   Improve performance by bulk inserting events where possible. (PR \#193)
+-   Improve performance by bulk verifying signatures where possible. (PR \#194)
+
+Configuration:
+
+-   Add support for including TLS certificate chains.
+
+Media Repository:
+
+-   Add Content-Disposition headers to content repository responses. (SYN-150)
+
+Changes in synapse v0.9.3 (2015-07-01)
+======================================
+
+No changes from v0.9.3 Release Candidate 1.
+
+Changes in synapse v0.9.3-rc1 (2015-06-23)
+==========================================
+
+General:
+
+-   Fix a memory leak in the notifier. (SYN-412)
+-   Improve performance of room initial sync. (SYN-418)
+-   General improvements to logging.
+-   Remove `access_token` query params from `INFO` level logging.
+
+Configuration:
+
+-   Add support for specifying and configuring multiple listeners. (SYN-389)
+
+Application services:
+
+-   Fix bug where synapse failed to send user queries to application services.
+
+Changes in synapse v0.9.2-r2 (2015-06-15)
+=========================================
+
+Fix packaging so that schema delta python files get included in the package.
+
+Changes in synapse v0.9.2 (2015-06-12)
+======================================
+
+General:
+
+-   Use ultrajson for json (de)serialisation when a canonical encoding is not required. Ultrajson is significantly faster than simplejson in certain circumstances.
+-   Use connection pools for outgoing HTTP connections.
+-   Process thumbnails on separate threads.
+
+Configuration:
+
+-   Add option, `gzip_responses`, to disable HTTP response compression.
+
+Federation:
+
+-   Improve resilience of backfill by ensuring we fetch any missing auth events.
+-   Improve performance of backfill and joining remote rooms by removing unnecessary computations. This included handling events we\'d previously handled as well as attempting to compute the current state for outliers.
+
+Changes in synapse v0.9.1 (2015-05-26)
+======================================
+
+General:
+
+-   Add support for backfilling when a client paginates. This allows servers to request history for a room from remote servers when a client tries to paginate history the server does not have - SYN-36
+-   Fix bug where you couldn\'t disable non-default pushrules - SYN-378
+-   Fix `register_new_user` script - SYN-359
+-   Improve performance of fetching events from the database, this improves both initialSync and sending of events.
+-   Improve performance of event streams, allowing synapse to handle more simultaneous connected clients.
+
+Federation:
+
+-   Fix bug with existing backfill implementation where it returned the wrong selection of events in some circumstances.
+-   Improve performance of joining remote rooms.
+
+Configuration:
+
+-   Add support for changing the bind host of the metrics listener via the `metrics_bind_host` option.
+
+Changes in synapse v0.9.0-r5 (2015-05-21)
+=========================================
+
+-   Add more database caches to reduce amount of work done for each pusher. This radically reduces CPU usage when multiple pushers are set up in the same room.
+
+Changes in synapse v0.9.0 (2015-05-07)
+======================================
+
+General:
+
+-   Add support for using a PostgreSQL database instead of SQLite. See [docs/postgres.rst](docs/postgres.rst) for details.
+-   Add password change and reset APIs. See [Registration](https://github.com/matrix-org/matrix-doc/blob/master/specification/10_client_server_api.rst#registration) in the spec.
+-   Fix memory leak due to not releasing stale notifiers - SYN-339.
+-   Fix race in caches that occasionally caused some presence updates to be dropped - SYN-369.
+-   Check server name has not changed on restart.
+-   Add a sample systemd unit file and a logger configuration in contrib/systemd. Contributed Ivan Shapovalov.
+
+Federation:
+
+-   Add key distribution mechanisms for fetching public keys of unavailable remote home servers. See [Retrieving Server Keys](https://github.com/matrix-org/matrix-doc/blob/6f2698/specification/30_server_server_api.rst#retrieving-server-keys) in the spec.
+
+Configuration:
+
+-   Add support for multiple config files.
+-   Add support for dictionaries in config files.
+-   Remove support for specifying config options on the command line, except for:
+    -   `--daemonize` - Daemonize the home server.
+    -   `--manhole` - Turn on the twisted telnet manhole service on the given port.
+    -   `--database-path` - The path to a sqlite database to use.
+    -   `--verbose` - The verbosity level.
+    -   `--log-file` - File to log to.
+    -   `--log-config` - Python logging config file.
+    -   `--enable-registration` - Enable registration for new users.
+
+Application services:
+
+-   Reliably retry sending of events from Synapse to application services, as per [Application Services](https://github.com/matrix-org/matrix-doc/blob/0c6bd9/specification/25_application_service_api.rst#home-server---application-service-api) spec.
+-   Application services can no longer register via the `/register` API, instead their configuration should be saved to a file and listed in the synapse `app_service_config_files` config option. The AS configuration file has the same format as the old `/register` request. See [docs/application\_services.rst](docs/application_services.rst) for more information.
+
+Changes in synapse v0.8.1 (2015-03-18)
+======================================
+
+-   Disable registration by default. New users can be added using the command `register_new_matrix_user` or by enabling registration in the config.
+-   Add metrics to synapse. To enable metrics use config options `enable_metrics` and `metrics_port`.
+-   Fix bug where banning only kicked the user.
+
+Changes in synapse v0.8.0 (2015-03-06)
+======================================
+
+General:
+
+-   Add support for registration fallback. This is a page hosted on the server which allows a user to register for an account, regardless of what client they are using (e.g. mobile devices).
+-   Added new default push rules and made them configurable by clients:
+    -   Suppress all notice messages.
+    -   Notify when invited to a new room.
+    -   Notify for messages that don\'t match any rule.
+    -   Notify on incoming call.
+
+Federation:
+
+-   Added per host server side rate-limiting of incoming federation requests.
+-   Added a `/get_missing_events/` API to federation to reduce number of `/events/` requests.
+
+Configuration:
+
+-   Added configuration option to disable registration: `disable_registration`.
+-   Added configuration option to change soft limit of number of open file descriptors: `soft_file_limit`.
+-   Make `tls_private_key_path` optional when running with `no_tls`.
+
+Application services:
+
+-   Application services can now poll on the CS API `/events` for their events, by providing their application service `access_token`.
+-   Added exclusive namespace support to application services API.
+
+Changes in synapse v0.7.1 (2015-02-19)
+======================================
+
+-   Initial alpha implementation of parts of the Application Services API. Including:
+    -   AS Registration / Unregistration
+    -   User Query API
+    -   Room Alias Query API
+    -   Push transport for receiving events.
+    -   User/Alias namespace admin control
+-   Add cache when fetching events from remote servers to stop repeatedly fetching events with bad signatures.
+-   Respect the per remote server retry scheme when fetching both events and server keys to reduce the number of times we send requests to dead servers.
+-   Inform remote servers when the local server fails to handle a received event.
+-   Turn off python bytecode generation due to problems experienced when upgrading from previous versions.
+
+Changes in synapse v0.7.0 (2015-02-12)
+======================================
+
+-   Add initial implementation of the query auth federation API, allowing servers to agree on whether an event should be allowed or rejected.
+-   Persist events we have rejected from federation, fixing the bug where servers would keep requesting the same events.
+-   Various federation performance improvements, including:
+    -   Add in memory caches on queries such as:
+
+        > -   Computing the state of a room at a point in time, used for authorization on federation requests.
+        > -   Fetching events from the database.
+        > -   User\'s room membership, used for authorizing presence updates.
+
+    -   Upgraded JSON library to improve parsing and serialisation speeds.
+
+-   Add default avatars to new user accounts using pydenticon library.
+-   Correctly time out federation requests.
+-   Retry federation requests against different servers.
+-   Add support for push and push rules.
+-   Add alpha versions of proposed new CSv2 APIs, including `/sync` API.
+
+Changes in synapse 0.6.1 (2015-01-07)
+=====================================
+
+-   Major optimizations to improve performance of initial sync and event sending in large rooms (by up to 10x)
+-   Media repository now includes a Content-Length header on media downloads.
+-   Improve quality of thumbnails by changing resizing algorithm.
+
+Changes in synapse 0.6.0 (2014-12-16)
+=====================================
+
+-   Add new API for media upload and download that supports thumbnailing.
+-   Replicate media uploads over multiple homeservers so media is always served to clients from their local homeserver. This obsoletes the \--content-addr parameter and confusion over accessing content directly from remote homeservers.
+-   Implement exponential backoff when retrying federation requests when sending to remote homeservers which are offline.
+-   Implement typing notifications.
+-   Fix bugs where we sent events with invalid signatures due to bugs where we incorrectly persisted events.
+-   Improve performance of database queries involving retrieving events.
+
+Changes in synapse 0.5.4a (2014-12-13)
+======================================
+
+-   Fix bug while generating the error message when a file path specified in the config doesn\'t exist.
+
+Changes in synapse 0.5.4 (2014-12-03)
+=====================================
+
+-   Fix presence bug where some rooms did not display presence updates for remote users.
+-   Do not log SQL timing log lines when started with \"-v\"
+-   Fix potential memory leak.
+
+Changes in synapse 0.5.3c (2014-12-02)
+======================================
+
+-   Change the default value for the content\_addr option to use the HTTP listener, as by default the HTTPS listener will be using a self-signed certificate.
+
+Changes in synapse 0.5.3 (2014-11-27)
+=====================================
+
+-   Fix bug that caused joining a remote room to fail if a single event was not signed correctly.
+-   Fix bug which caused servers to continuously try and fetch events from other servers.
+
+Changes in synapse 0.5.2 (2014-11-26)
+=====================================
+
+Fix major bug that caused rooms to disappear from peoples initial sync.
+
+Changes in synapse 0.5.1 (2014-11-26)
+=====================================
+
+See UPGRADES.rst for specific instructions on how to upgrade.
+
+> -   Fix bug where we served up an Event that did not match its signatures.
+> -   Fix regression where we no longer correctly handled the case where a homeserver receives an event for a room it doesn\'t recognise (but is in.)
+
+Changes in synapse 0.5.0 (2014-11-19)
+=====================================
+
+This release includes changes to the federation protocol and client-server API that is not backwards compatible.
+
+This release also changes the internal database schemas and so requires servers to drop their current history. See UPGRADES.rst for details.
+
+Homeserver:
+
+:   -   Add authentication and authorization to the federation protocol. Events are now signed by their originating homeservers.
+    -   Implement the new authorization model for rooms.
+    -   Split out web client into a seperate repository: matrix-angular-sdk.
+    -   Change the structure of PDUs.
+    -   Fix bug where user could not join rooms via an alias containing 4-byte UTF-8 characters.
+    -   Merge concept of PDUs and Events internally.
+    -   Improve logging by adding request ids to log lines.
+    -   Implement a very basic room initial sync API.
+    -   Implement the new invite/join federation APIs.
+
+Webclient:
+
+:   -   The webclient has been moved to a seperate repository.
+
+Changes in synapse 0.4.2 (2014-10-31)
+=====================================
+
+Homeserver:
+
+:   -   Fix bugs where we did not notify users of correct presence updates.
+    -   Fix bug where we did not handle sub second event stream timeouts.
+
+Webclient:
+
+:   -   Add ability to click on messages to see JSON.
+    -   Add ability to redact messages.
+    -   Add ability to view and edit all room state JSON.
+    -   Handle incoming redactions.
+    -   Improve feedback on errors.
+    -   Fix bugs in mobile CSS.
+    -   Fix bugs with desktop notifications.
+
+Changes in synapse 0.4.1 (2014-10-17)
+=====================================
+
+Webclient:
+
+:   -   Fix bug with display of timestamps.
+
+Changes in synpase 0.4.0 (2014-10-17)
+=====================================
+
+This release includes changes to the federation protocol and client-server API that is not backwards compatible.
+
+The Matrix specification has been moved to a separate git repository: <http://github.com/matrix-org/matrix-doc>
+
+You will also need an updated syutil and config. See UPGRADES.rst.
+
+Homeserver:
+
+:   -   Sign federation transactions to assert strong identity over federation.
+    -   Rename timestamp keys in PDUs and events from \'ts\' and \'hsob\_ts\' to \'origin\_server\_ts\'.
+
+Changes in synapse 0.3.4 (2014-09-25)
+=====================================
+
+This version adds support for using a TURN server. See docs/turn-howto.rst on how to set one up.
+
+Homeserver:
+
+:   -   Add support for redaction of messages.
+    -   Fix bug where inviting a user on a remote home server could take up to 20-30s.
+    -   Implement a get current room state API.
+    -   Add support specifying and retrieving turn server configuration.
+
+Webclient:
+
+:   -   Add button to send messages to users from the home page.
+    -   Add support for using TURN for VoIP calls.
+    -   Show display name change messages.
+    -   Fix bug where the client didn\'t get the state of a newly joined room until after it has been refreshed.
+    -   Fix bugs with tab complete.
+    -   Fix bug where holding down the down arrow caused chrome to chew 100% CPU.
+    -   Fix bug where desktop notifications occasionally used \"Undefined\" as the display name.
+    -   Fix more places where we sometimes saw room IDs incorrectly.
+    -   Fix bug which caused lag when entering text in the text box.
+
+Changes in synapse 0.3.3 (2014-09-22)
+=====================================
+
+Homeserver:
+
+:   -   Fix bug where you continued to get events for rooms you had left.
+
+Webclient:
+
+:   -   Add support for video calls with basic UI.
+    -   Fix bug where one to one chats were named after your display name rather than the other person\'s.
+    -   Fix bug which caused lag when typing in the textarea.
+    -   Refuse to run on browsers we know won\'t work.
+    -   Trigger pagination when joining new rooms.
+    -   Fix bug where we sometimes didn\'t display invitations in recents.
+    -   Automatically join room when accepting a VoIP call.
+    -   Disable outgoing and reject incoming calls on browsers we don\'t support VoIP in.
+    -   Don\'t display desktop notifications for messages in the room you are non-idle and speaking in.
+
+Changes in synapse 0.3.2 (2014-09-18)
+=====================================
+
+Webclient:
+
+:   -   Fix bug where an empty \"bing words\" list in old accounts didn\'t send notifications when it should have done.
+
+Changes in synapse 0.3.1 (2014-09-18)
+=====================================
+
+This is a release to hotfix v0.3.0 to fix two regressions.
+
+Webclient:
+
+:   -   Fix a regression where we sometimes displayed duplicate events.
+    -   Fix a regression where we didn\'t immediately remove rooms you were banned in from the recents list.
+
+Changes in synapse 0.3.0 (2014-09-18)
+=====================================
+
+See UPGRADE for information about changes to the client server API, including breaking backwards compatibility with VoIP calls and registration API.
+
+Homeserver:
+
+:   -   When a user changes their displayname or avatar the server will now update all their join states to reflect this.
+    -   The server now adds \"age\" key to events to indicate how old they are. This is clock independent, so at no point does any server or webclient have to assume their clock is in sync with everyone else.
+    -   Fix bug where we didn\'t correctly pull in missing PDUs.
+    -   Fix bug where prev\_content key wasn\'t always returned.
+    -   Add support for password resets.
+
+Webclient:
+
+:   -   Improve page content loading.
+    -   Join/parts now trigger desktop notifications.
+    -   Always show room aliases in the UI if one is present.
+    -   No longer show user-count in the recents side panel.
+    -   Add up & down arrow support to the text box for message sending to step through your sent history.
+    -   Don\'t display notifications for our own messages.
+    -   Emotes are now formatted correctly in desktop notifications.
+    -   The recents list now differentiates between public & private rooms.
+    -   Fix bug where when switching between rooms the pagination flickered before the view jumped to the bottom of the screen.
+    -   Add bing word support.
+
+Registration API:
+
+:   -   The registration API has been overhauled to function like the login API. In practice, this means registration requests must now include the following: \'type\':\'m.login.password\'. See UPGRADE for more information on this.
+    -   The \'user\_id\' key has been renamed to \'user\' to better match the login API.
+    -   There is an additional login type: \'m.login.email.identity\'.
+    -   The command client and web client have been updated to reflect these changes.
+
+Changes in synapse 0.2.3 (2014-09-12)
+=====================================
+
+Homeserver:
+
+:   -   Fix bug where we stopped sending events to remote home servers if a user from that home server left, even if there were some still in the room.
+    -   Fix bugs in the state conflict resolution where it was incorrectly rejecting events.
+
+Webclient:
+
+:   -   Display room names and topics.
+    -   Allow setting/editing of room names and topics.
+    -   Display information about rooms on the main page.
+    -   Handle ban and kick events in real time.
+    -   VoIP UI and reliability improvements.
+    -   Add glare support for VoIP.
+    -   Improvements to initial startup speed.
+    -   Don\'t display duplicate join events.
+    -   Local echo of messages.
+    -   Differentiate sending and sent of local echo.
+    -   Various minor bug fixes.
+
+Changes in synapse 0.2.2 (2014-09-06)
+=====================================
+
+Homeserver:
+
+:   -   When the server returns state events it now also includes the previous content.
+    -   Add support for inviting people when creating a new room.
+    -   Make the homeserver inform the room via m.room.aliases when a new alias is added for a room.
+    -   Validate m.room.power\_level events.
+
+Webclient:
+
+:   -   Add support for captchas on registration.
+    -   Handle m.room.aliases events.
+    -   Asynchronously send messages and show a local echo.
+    -   Inform the UI when a message failed to send.
+    -   Only autoscroll on receiving a new message if the user was already at the bottom of the screen.
+    -   Add support for ban/kick reasons.
+
+Changes in synapse 0.2.1 (2014-09-03)
+=====================================
+
+Homeserver:
+
+:   -   Added support for signing up with a third party id.
+    -   Add synctl scripts.
+    -   Added rate limiting.
+    -   Add option to change the external address the content repo uses.
+    -   Presence bug fixes.
+
+Webclient:
+
+:   -   Added support for signing up with a third party id.
+    -   Added support for banning and kicking users.
+    -   Added support for displaying and setting ops.
+    -   Added support for room names.
+    -   Fix bugs with room membership event display.
+
+Changes in synapse 0.2.0 (2014-09-02)
+=====================================
+
+This update changes many configuration options, updates the database schema and mandates SSL for server-server connections.
+
+Homeserver:
+
+:   -   Require SSL for server-server connections.
+    -   Add SSL listener for client-server connections.
+    -   Add ability to use config files.
+    -   Add support for kicking/banning and power levels.
+    -   Allow setting of room names and topics on creation.
+    -   Change presence to include last seen time of the user.
+    -   Change url path prefix to /\_matrix/\...
+    -   Bug fixes to presence.
+
+Webclient:
+
+:   -   Reskin the CSS for registration and login.
+    -   Various improvements to rooms CSS.
+    -   Support changes in client-server API.
+    -   Bug fixes to VOIP UI.
+    -   Various bug fixes to handling of changes to room member list.
+
+Changes in synapse 0.1.2 (2014-08-29)
+=====================================
+
+Webclient:
+
+:   -   Add basic call state UI for VoIP calls.
+
+Changes in synapse 0.1.1 (2014-08-29)
+=====================================
+
+Homeserver:
+
+:   -   Fix bug that caused the event stream to not notify some clients about changes.
+
+Changes in synapse 0.1.0 (2014-08-29)
+=====================================
+
+Presence has been reenabled in this release.
+
+Homeserver:
+
+:   -   
+
+        Update client to server API, including:
+
+        :   -   Use a more consistent url scheme.
+            -   Provide more useful information in the initial sync api.
+
+    -   Change the presence handling to be much more efficient.
+    -   Change the presence server to server API to not require explicit polling of all users who share a room with a user.
+    -   Fix races in the event streaming logic.
+
+Webclient:
+
+:   -   Update to use new client to server API.
+    -   Add basic VOIP support.
+    -   Add idle timers that change your status to away.
+    -   Add recent rooms column when viewing a room.
+    -   Various network efficiency improvements.
+    -   Add basic mobile browser support.
+    -   Add a settings page.
+
+Changes in synapse 0.0.1 (2014-08-22)
+=====================================
+
+Presence has been disabled in this release due to a bug that caused the homeserver to spam other remote homeservers.
+
+Homeserver:
+
+:   -   Completely change the database schema to support generic event types.
+    -   Improve presence reliability.
+    -   Improve reliability of joining remote rooms.
+    -   Fix bug where room join events were duplicated.
+    -   Improve initial sync API to return more information to the client.
+    -   Stop generating fake messages for room membership events.
+
+Webclient:
+
+:   -   Add tab completion of names.
+    -   Add ability to upload and send images.
+    -   Add profile pages.
+    -   Improve CSS layout of room.
+    -   Disambiguate identical display names.
+    -   Don\'t get remote users display names and avatars individually.
+    -   Use the new initial sync API to reduce number of round trips to the homeserver.
+    -   Change url scheme to use room aliases instead of room ids where known.
+    -   Increase longpoll timeout.
+
+Changes in synapse 0.0.0 (2014-08-13)
+=====================================
+
+> -   Initial alpha release
diff --git a/CHANGES.rst b/CHANGES.rst
deleted file mode 100644
index afc4fee88277..000000000000
--- a/CHANGES.rst
+++ /dev/null
@@ -1,2867 +0,0 @@
-Synapse 0.33.0 (2018-07-19)
-===========================
-
-Features
---------
-
-- Enforce the specified API for report_event (`#3316 <https://github.com/matrix-org/synapse/issues/3316>`_)
-- Include CPU time from database threads in request/block metrics. (`#3496 <https://github.com/matrix-org/synapse/issues/3496>`_, `#3501 <https://github.com/matrix-org/synapse/issues/3501>`_)
-- Add CPU metrics for _fetch_event_list (`#3497 <https://github.com/matrix-org/synapse/issues/3497>`_)
-- optimisation for /sync (`#3505 <https://github.com/matrix-org/synapse/issues/3505>`_, `#3521 <https://github.com/matrix-org/synapse/issues/3521>`_)
-- Optimisation to make handling incoming federation requests more efficient. (`#3541 <https://github.com/matrix-org/synapse/issues/3541>`_)
-
-
-Bugfixes
---------
-
-- Use more portable syntax in our use of the attrs package, widening the supported versions (`#3498 <https://github.com/matrix-org/synapse/issues/3498>`_)
-- Fix queued federation requests being processed in the wrong order (`#3533 <https://github.com/matrix-org/synapse/issues/3533>`_)
-- Ensure that erasure requests are correctly honoured for publicly accessible rooms when accessed over federation. (`#3546 <https://github.com/matrix-org/synapse/issues/3546>`_)
-- Disable a noisy warning about logcontexts (`#3561 <https://github.com/matrix-org/synapse/issues/3561>`_)
-
-
-Misc
-----
-
-- `#3351 <https://github.com/matrix-org/synapse/issues/3351>`_, `#3463 <https://github.com/matrix-org/synapse/issues/3463>`_, `#3464 <https://github.com/matrix-org/synapse/issues/3464>`_, `#3499 <https://github.com/matrix-org/synapse/issues/3499>`_, `#3530 <https://github.com/matrix-org/synapse/issues/3530>`_, `#3534 <https://github.com/matrix-org/synapse/issues/3534>`_, `#3535 <https://github.com/matrix-org/synapse/issues/3535>`_, `#3540 <https://github.com/matrix-org/synapse/issues/3540>`_, `#3544 <https://github.com/matrix-org/synapse/issues/3544>`_
-
-
-Synapse 0.32.2 (2018-07-07)
-===========================
-
-Bugfixes
---------
-
-- Amend the Python dependencies to depend on attrs from PyPI, not attr (`#3492 <https://github.com/matrix-org/synapse/issues/3492>`_)
-
-
-Synapse 0.32.1 (2018-07-06)
-===========================
-
-Bugfixes
---------
-
-- Add explicit dependency on netaddr (`#3488 <https://github.com/matrix-org/synapse/issues/3488>`_)
-
-
-Changes in synapse v0.32.0 (2018-07-06)
-===========================================
-No changes since 0.32.0rc1
-
-Synapse 0.32.0rc1 (2018-07-05)
-==============================
-
-Features
---------
-
-- Add blacklist & whitelist of servers allowed to send events to a room via ``m.room.server_acl`` event.
-- Cache factor override system for specific caches (`#3334 <https://github.com/matrix-org/synapse/issues/3334>`_)
-- Add metrics to track appservice transactions (`#3344 <https://github.com/matrix-org/synapse/issues/3344>`_)
-- Try to log more helpful info when a sig verification fails (`#3372 <https://github.com/matrix-org/synapse/issues/3372>`_)
-- Synapse now uses the best performing JSON encoder/decoder according to your runtime (simplejson on CPython, stdlib json on PyPy). (`#3462 <https://github.com/matrix-org/synapse/issues/3462>`_)
-- Add optional ip_range_whitelist param to AS registration files to lock AS IP access (`#3465 <https://github.com/matrix-org/synapse/issues/3465>`_)
-- Reject invalid server names in federation requests (`#3480 <https://github.com/matrix-org/synapse/issues/3480>`_)
-- Reject invalid server names in homeserver.yaml (`#3483 <https://github.com/matrix-org/synapse/issues/3483>`_)
-
-
-Bugfixes
---------
-
-- Strip access_token from outgoing requests (`#3327 <https://github.com/matrix-org/synapse/issues/3327>`_)
-- Redact AS tokens in logs (`#3349 <https://github.com/matrix-org/synapse/issues/3349>`_)
-- Fix federation backfill from SQLite servers (`#3355 <https://github.com/matrix-org/synapse/issues/3355>`_)
-- Fix event-purge-by-ts admin API (`#3363 <https://github.com/matrix-org/synapse/issues/3363>`_)
-- Fix event filtering in get_missing_events handler (`#3371 <https://github.com/matrix-org/synapse/issues/3371>`_)
-- Synapse is now stricter regarding accepting events which it cannot retrieve the prev_events for. (`#3456 <https://github.com/matrix-org/synapse/issues/3456>`_)
-- Fix bug where synapse would explode when receiving unicode in HTTP User-Agent header (`#3470 <https://github.com/matrix-org/synapse/issues/3470>`_)
-- Invalidate cache on correct thread to avoid race (`#3473 <https://github.com/matrix-org/synapse/issues/3473>`_)
-
-
-Improved Documentation
-----------------------
-
-- ``doc/postgres.rst``: fix display of the last command block. Thanks to @ArchangeGabriel! (`#3340 <https://github.com/matrix-org/synapse/issues/3340>`_)
-
-
-Deprecations and Removals
--------------------------
-
-- Remove was_forgotten_at (`#3324 <https://github.com/matrix-org/synapse/issues/3324>`_)
-
-
-Misc
-----
-
-- `#3332 <https://github.com/matrix-org/synapse/issues/3332>`_, `#3341 <https://github.com/matrix-org/synapse/issues/3341>`_, `#3347 <https://github.com/matrix-org/synapse/issues/3347>`_, `#3348 <https://github.com/matrix-org/synapse/issues/3348>`_, `#3356 <https://github.com/matrix-org/synapse/issues/3356>`_, `#3385 <https://github.com/matrix-org/synapse/issues/3385>`_, `#3446 <https://github.com/matrix-org/synapse/issues/3446>`_, `#3447 <https://github.com/matrix-org/synapse/issues/3447>`_, `#3467 <https://github.com/matrix-org/synapse/issues/3467>`_, `#3474 <https://github.com/matrix-org/synapse/issues/3474>`_
-
-
-Changes in synapse v0.31.2 (2018-06-14)
-=======================================
-
-SECURITY UPDATE: Prevent unauthorised users from setting state events in a room
-when there is no ``m.room.power_levels`` event in force in the room. (PR #3397)
-
-Discussion around the Matrix Spec change proposal for this change can be
-followed at https://github.com/matrix-org/matrix-doc/issues/1304.
-
-Changes in synapse v0.31.1 (2018-06-08)
-=======================================
-
-v0.31.1 fixes a security bug in the ``get_missing_events`` federation API
-where event visibility rules were not applied correctly.
-
-We are not aware of it being actively exploited but please upgrade asap.
-
-Bug Fixes:
-
-* Fix event filtering in get_missing_events handler (PR #3371)
-
-Changes in synapse v0.31.0 (2018-06-06)
-=======================================
-
-Most notable change from v0.30.0 is to switch to the python prometheus library to improve system
-stats reporting. WARNING: this changes a number of prometheus metrics in a
-backwards-incompatible manner. For more details, see
-`docs/metrics-howto.rst <docs/metrics-howto.rst#removal-of-deprecated-metrics--time-based-counters-becoming-histograms-in-0310>`_.
-
-Bug Fixes:
-
-* Fix metric documentation tables (PR #3341)
-* Fix LaterGauge error handling (694968f)
-* Fix replication metrics (b7e7fd2)
-
-Changes in synapse v0.31.0-rc1 (2018-06-04)
-==========================================
-
-Features:
-
-* Switch to the Python Prometheus library (PR #3256, #3274)
-* Let users leave the server notice room after joining (PR #3287)
-
-
-Changes:
-
-* daily user type phone home stats (PR #3264)
-* Use iter* methods for _filter_events_for_server (PR #3267)
-* Docs on consent bits (PR #3268)
-* Remove users from user directory on deactivate (PR #3277)
-* Avoid sending consent notice to guest users (PR #3288)
-* disable CPUMetrics if no /proc/self/stat (PR #3299)
-* Consistently use six's iteritems and wrap lazy keys/values in list() if they're not meant to be lazy (PR #3307)
-* Add private IPv6 addresses to example config for url preview blacklist (PR #3317) Thanks to @thegcat!
-* Reduce stuck read-receipts: ignore depth when updating (PR #3318)
-* Put python's logs into Trial when running unit tests (PR #3319)
-
-Changes, python 3 migration:
-
-* Replace some more comparisons with six (PR #3243) Thanks to @NotAFile!
-* replace some iteritems with six (PR #3244) Thanks to @NotAFile!
-* Add batch_iter to utils (PR #3245) Thanks to @NotAFile!
-* use repr, not str (PR #3246) Thanks to @NotAFile!
-* Misc Python3 fixes (PR #3247) Thanks to @NotAFile!
-* Py3 storage/_base.py (PR #3278) Thanks to @NotAFile!
-* more six iteritems (PR #3279) Thanks to @NotAFile!
-* More Misc. py3 fixes (PR #3280) Thanks to @NotAFile!
-* remaining isintance fixes (PR #3281) Thanks to @NotAFile!
-* py3-ize state.py (PR #3283) Thanks to @NotAFile!
-* extend tox testing for py3 to avoid regressions (PR #3302) Thanks to @krombel!
-* use memoryview in py3 (PR #3303) Thanks to @NotAFile!
-
-Bugs:
-
-* Fix federation backfill bugs (PR #3261)
-* federation: fix LaterGauge usage (PR #3328) Thanks to @intelfx!
-
-
-Changes in synapse v0.30.0 (2018-05-24)
-==========================================
-
-'Server Notices' are a new feature introduced in Synapse 0.30. They provide a
-channel whereby server administrators can send messages to users on the server.
-
-They are used as part of communication of the server policies (see ``docs/consent_tracking.md``),
-however the intention is that they may also find a use for features such
-as "Message of the day".
-
-This feature is specific to Synapse, but uses standard Matrix communication mechanisms,
-so should work with any Matrix client. For more details see ``docs/server_notices.md``
-
-Further Server Notices/Consent Tracking Support:
-
-* Allow overriding the server_notices user's avatar (PR #3273)
-* Use the localpart in the consent uri (PR #3272)
-* Support for putting %(consent_uri)s in messages (PR #3271)
-* Block attempts to send server notices to remote users (PR #3270)
-* Docs on consent bits (PR #3268)
-
-
-
-Changes in synapse v0.30.0-rc1 (2018-05-23)
-==========================================
-
-Server Notices/Consent Tracking Support:
-
-* ConsentResource to gather policy consent from users (PR #3213)
-* Move RoomCreationHandler out of synapse.handlers.Handlers (PR #3225)
-* Infrastructure for a server notices room (PR #3232)
-* Send users a server notice about consent (PR #3236)
-* Reject attempts to send event before privacy consent is given (PR #3257)
-* Add a 'has_consented' template var to consent forms (PR #3262)
-* Fix dependency on jinja2 (PR #3263)
-
-Features:
-
-* Cohort analytics (PR #3163, #3241, #3251)
-* Add lxml to docker image for web previews (PR #3239) Thanks to @ptman!
-* Add in flight request metrics (PR #3252)
-
-Changes:
-
-* Remove unused `update_external_syncs` (PR #3233)
-* Use stream rather depth ordering for push actions (PR #3212)
-* Make purge_history operate on tokens (PR #3221)
-* Don't support limitless pagination (PR #3265)
-
-Bug Fixes:
-
-* Fix logcontext resource usage tracking (PR #3258)
-* Fix error in handling receipts (PR #3235)
-* Stop the transaction cache caching failures (PR #3255)
-
-
-Changes in synapse v0.29.1 (2018-05-17)
-==========================================
-Changes:
-
-* Update docker documentation (PR #3222)
-
-Changes in synapse v0.29.0 (2018-05-16)
-===========================================
-Not changes since v0.29.0-rc1
-
-Changes in synapse v0.29.0-rc1 (2018-05-14)
-===========================================
-
-Notable changes, a docker file for running Synapse (Thanks to @kaiyou!) and a
-closed spec bug in the Client Server API. Additionally further prep for Python 3
-migration.
-
-Potentially breaking change:
-
-* Make Client-Server API return 401 for invalid token (PR #3161).
-
-  This changes the Client-server spec to return a 401 error code instead of 403
-  when the access token is unrecognised. This is the behaviour required by the
-  specification, but some clients may be relying on the old, incorrect
-  behaviour.
-
-  Thanks to @NotAFile for fixing this.
-
-Features:
-
-* Add a Dockerfile for synapse (PR #2846) Thanks to @kaiyou!
-
-Changes - General:
-
-* nuke-room-from-db.sh: added postgresql option and help (PR #2337) Thanks to @rubo77!
-* Part user from rooms on account deactivate (PR #3201)
-* Make 'unexpected logging context' into warnings (PR #3007)
-* Set Server header in SynapseRequest (PR #3208)
-* remove duplicates from groups tables (PR #3129)
-* Improve exception handling for background processes (PR #3138)
-* Add missing consumeErrors to improve exception handling (PR #3139)
-* reraise exceptions more carefully (PR #3142)
-* Remove redundant call to preserve_fn (PR #3143)
-* Trap exceptions thrown within run_in_background (PR #3144)
-
-Changes - Refactors:
-
-* Refactor /context to reuse pagination storage functions (PR #3193)
-* Refactor recent events func to use pagination func (PR #3195)
-* Refactor pagination DB API to return concrete type (PR #3196)
-* Refactor get_recent_events_for_room return type (PR #3198)
-* Refactor sync APIs to reuse pagination API (PR #3199)
-* Remove unused code path from member change DB func (PR #3200)
-* Refactor request handling wrappers (PR #3203)
-* transaction_id, destination defined twice (PR #3209) Thanks to @damir-manapov!
-* Refactor event storage to prepare for changes in state calculations (PR #3141)
-* Set Server header in SynapseRequest (PR #3208)
-* Use deferred.addTimeout instead of time_bound_deferred (PR #3127, #3178)
-* Use run_in_background in preference to preserve_fn (PR #3140)
-
-Changes - Python 3 migration:
-
-* Construct HMAC as bytes on py3 (PR #3156) Thanks to @NotAFile!
-* run config tests on py3 (PR #3159) Thanks to @NotAFile!
-* Open certificate files as bytes (PR #3084) Thanks to @NotAFile!
-* Open config file in non-bytes mode (PR #3085) Thanks to @NotAFile!
-* Make event properties raise AttributeError instead (PR #3102) Thanks to @NotAFile!
-* Use six.moves.urlparse (PR #3108) Thanks to @NotAFile!
-* Add py3 tests to tox with folders that work (PR #3145) Thanks to @NotAFile!
-* Don't yield in list comprehensions (PR #3150) Thanks to @NotAFile!
-* Move more xrange to six (PR #3151) Thanks to @NotAFile!
-* make imports local (PR #3152) Thanks to @NotAFile!
-* move httplib import to six (PR #3153) Thanks to @NotAFile!
-* Replace stringIO imports with six (PR #3154, #3168) Thanks to @NotAFile!
-* more bytes strings (PR #3155) Thanks to @NotAFile!
-
-Bug Fixes:
-
-* synapse fails to start under Twisted >= 18.4 (PR #3157)
-* Fix a class of logcontext leaks (PR #3170)
-* Fix a couple of logcontext leaks in unit tests (PR #3172)
-* Fix logcontext leak in media repo (PR #3174)
-* Escape label values in prometheus metrics (PR #3175, #3186)
-* Fix 'Unhandled Error' logs with Twisted 18.4 (PR #3182) Thanks to @Half-Shot!
-* Fix logcontext leaks in rate limiter (PR #3183)
-* notifications: Convert next_token to string according to the spec (PR #3190) Thanks to @mujx!
-* nuke-room-from-db.sh: fix deletion from search table (PR #3194) Thanks to @rubo77!
-* add guard for None on purge_history api (PR #3160) Thanks to @krombel!
-
-Changes in synapse v0.28.1 (2018-05-01)
-=======================================
-
-SECURITY UPDATE
-
-* Clamp the allowed values of event depth received over federation to be
-  [0, 2^63 - 1].  This mitigates an attack where malicious events
-  injected with depth = 2^63 - 1 render rooms unusable.  Depth is used to
-  determine the cosmetic ordering of events within a room, and so the ordering
-  of events in such a room will default to using stream_ordering rather than depth
-  (topological_ordering).
-
-  This is a temporary solution to mitigate abuse in the wild, whilst a long term solution
-  is being implemented to improve how the depth parameter is used.
-
-  Full details at
-  https://docs.google.com/document/d/1I3fi2S-XnpO45qrpCsowZv8P8dHcNZ4fsBsbOW7KABI
-
-* Pin Twisted to <18.4 until we stop using the private _OpenSSLECCurve API.
-
-
-Changes in synapse v0.28.0 (2018-04-26)
-=======================================
-
-Bug Fixes:
-
-* Fix quarantine media admin API and search reindex (PR #3130)
-* Fix media admin APIs (PR #3134)
-
-
-Changes in synapse v0.28.0-rc1 (2018-04-24)
-===========================================
-
-Minor performance improvement to federation sending and bug fixes.
-
-(Note: This release does not include the delta state resolution implementation discussed in matrix live)
-
-
-Features:
-
-* Add metrics for event processing lag (PR #3090)
-* Add metrics for ResponseCache (PR #3092)
-
-Changes:
-
-* Synapse on PyPy (PR #2760) Thanks to @Valodim!
-* move handling of auto_join_rooms to RegisterHandler (PR #2996) Thanks to @krombel!
-* Improve handling of SRV records for federation connections (PR #3016) Thanks to @silkeh!
-* Document the behaviour of ResponseCache (PR #3059)
-* Preparation for py3 (PR #3061, #3073, #3074, #3075, #3103, #3104, #3106, #3107, #3109, #3110) Thanks to @NotAFile!
-* update prometheus dashboard to use new metric names (PR #3069) Thanks to @krombel!
-* use python3-compatible prints (PR #3074) Thanks to @NotAFile!
-* Send federation events concurrently (PR #3078)
-* Limit concurrent event sends for a room (PR #3079)
-* Improve R30 stat definition (PR #3086)
-* Send events to ASes concurrently (PR #3088)
-* Refactor ResponseCache usage (PR #3093)
-* Clarify that SRV may not point to a CNAME (PR #3100) Thanks to @silkeh!
-* Use str(e) instead of e.message (PR #3103) Thanks to @NotAFile!
-* Use six.itervalues in some places (PR #3106) Thanks to @NotAFile!
-* Refactor store.have_events (PR #3117)
-
-Bug Fixes:
-
-* Return 401 for invalid access_token on logout (PR #2938) Thanks to @dklug!
-* Return a 404 rather than a 500 on rejoining empty rooms (PR #3080)
-* fix federation_domain_whitelist (PR #3099)
-* Avoid creating events with huge numbers of prev_events (PR #3113)
-* Reject events which have lots of prev_events (PR #3118)
-
-
-Changes in synapse v0.27.4 (2018-04-13)
-======================================
-
-Changes:
-
-* Update canonicaljson dependency (#3095)
-
-
-Changes in synapse v0.27.3 (2018-04-11)
-======================================
-
-Bug fixes:
-
-* URL quote path segments over federation (#3082)
-
-Changes in synapse v0.27.3-rc2 (2018-04-09)
-==========================================
-
-v0.27.3-rc1 used a stale version of the develop branch so the changelog overstates
-the functionality. v0.27.3-rc2 is up to date, rc1 should be ignored.
-
-Changes in synapse v0.27.3-rc1 (2018-04-09)
-=======================================
-
-Notable changes include API support for joinability of groups. Also new metrics
-and phone home stats. Phone home stats include better visibility of system usage
-so we can tweak synpase to work better for all users rather than our own experience
-with matrix.org. Also, recording 'r30' stat which is the measure we use to track
-overal growth of the Matrix ecosystem. It is defined as:-
-
-Counts the number of native 30 day retained users, defined as:-
-         * Users who have created their accounts more than 30 days
-         * Where last seen at most 30 days ago
-         * Where account creation and last_seen are > 30 days"
-
-
-Features:
-
-* Add joinability for groups (PR #3045)
-* Implement group join API (PR #3046)
-* Add counter metrics for calculating state delta (PR #3033)
-* R30 stats (PR #3041)
-* Measure time it takes to calculate state group ID (PR #3043)
-* Add basic performance statistics to phone home (PR #3044)
-* Add response size metrics (PR #3071)
-* phone home cache size configurations (PR #3063)
-
-Changes:
-
-* Add a blurb explaining the main synapse worker (PR #2886) Thanks to @turt2live!
-* Replace old style error catching with 'as' keyword (PR #3000) Thanks to @NotAFile!
-* Use .iter* to avoid copies in StateHandler (PR #3006)
-* Linearize calls to _generate_user_id (PR #3029)
-* Remove last usage of ujson (PR #3030)
-* Use simplejson throughout (PR #3048)
-* Use static JSONEncoders (PR #3049)
-* Remove uses of events.content (PR #3060)
-* Improve database cache performance (PR #3068)
-
-Bug fixes:
-
-* Add room_id to the response of `rooms/{roomId}/join` (PR #2986) Thanks to @jplatte!
-* Fix replication after switch to simplejson (PR #3015)
-* 404 correctly on missing paths via NoResource (PR #3022)
-* Fix error when claiming e2e keys from offline servers (PR #3034)
-* fix tests/storage/test_user_directory.py (PR #3042)
-* use PUT instead of POST for federating groups/m.join_policy (PR #3070) Thanks to @krombel!
-* postgres port script: fix state_groups_pkey error (PR #3072)
-
-
-Changes in synapse v0.27.2 (2018-03-26)
-=======================================
-
-Bug fixes:
-
-* Fix bug which broke TCP replication between workers (PR #3015)
-
-
-Changes in synapse v0.27.1 (2018-03-26)
-=======================================
-
-Meta release as v0.27.0 temporarily pointed to the wrong commit
-
-
-Changes in synapse v0.27.0 (2018-03-26)
-=======================================
-
-No changes since v0.27.0-rc2
-
-
-Changes in synapse v0.27.0-rc2 (2018-03-19)
-===========================================
-
-Pulls in v0.26.1
-
-Bug fixes:
-
-* Fix bug introduced in v0.27.0-rc1 that causes much increased memory usage in state cache (PR #3005)
-
-
-Changes in synapse v0.26.1 (2018-03-15)
-=======================================
-
-Bug fixes:
-
-* Fix bug where an invalid event caused server to stop functioning correctly,
-  due to parsing and serializing bugs in ujson library (PR #3008)
-
-
-Changes in synapse v0.27.0-rc1 (2018-03-14)
-===========================================
-
-The common case for running Synapse is not to run separate workers, but for those that do, be aware that synctl no longer starts the main synapse when using ``-a`` option with workers. A new worker file should be added with ``worker_app: synapse.app.homeserver``.
-
-This release also begins the process of renaming a number of the metrics
-reported to prometheus. See `docs/metrics-howto.rst <docs/metrics-howto.rst#block-and-response-metrics-renamed-for-0-27-0>`_.
-Note that the v0.28.0 release will remove the deprecated metric names.
-
-Features:
-
-* Add ability for ASes to override message send time (PR #2754)
-* Add support for custom storage providers for media repository (PR #2867, #2777, #2783, #2789, #2791, #2804, #2812, #2814, #2857, #2868, #2767)
-* Add purge API features, see `docs/admin_api/purge_history_api.rst <docs/admin_api/purge_history_api.rst>`_ for full details (PR #2858, #2867, #2882, #2946, #2962, #2943)
-* Add support for whitelisting 3PIDs that users can register. (PR #2813)
-* Add ``/room/{id}/event/{id}`` API (PR #2766)
-* Add an admin API to get all the media in a room (PR #2818) Thanks to @turt2live!
-* Add ``federation_domain_whitelist`` option (PR #2820, #2821)
-
-
-Changes:
-
-* Continue to factor out processing from main process and into worker processes. See updated `docs/workers.rst <docs/workers.rst>`_ (PR #2892 - #2904, #2913, #2920 - #2926, #2947, #2847, #2854, #2872, #2873, #2874, #2928, #2929, #2934, #2856, #2976 - #2984, #2987 - #2989, #2991 - #2993, #2995, #2784)
-* Ensure state cache is used when persisting events (PR #2864, #2871, #2802, #2835, #2836, #2841, #2842, #2849)
-* Change the default config to bind on both IPv4 and IPv6 on all platforms (PR #2435) Thanks to @silkeh!
-* No longer require a specific version of saml2 (PR #2695) Thanks to @okurz!
-* Remove ``verbosity``/``log_file`` from generated config (PR #2755)
-* Add and improve metrics and logging (PR #2770, #2778, #2785, #2786, #2787, #2793, #2794, #2795, #2809, #2810, #2833, #2834, #2844, #2965, #2927, #2975, #2790, #2796, #2838)
-* When using synctl with workers, don't start the main synapse automatically (PR #2774)
-* Minor performance improvements (PR #2773, #2792)
-* Use a connection pool for non-federation outbound connections (PR #2817)
-* Make it possible to run unit tests against postgres (PR #2829)
-* Update pynacl dependency to 1.2.1 or higher (PR #2888) Thanks to @bachp!
-* Remove ability for AS users to call /events and /sync (PR #2948)
-* Use bcrypt.checkpw (PR #2949) Thanks to @krombel!
-
-Bug fixes:
-
-* Fix broken ``ldap_config`` config option (PR #2683) Thanks to @seckrv!
-* Fix error message when user is not allowed to unban (PR #2761) Thanks to @turt2live!
-* Fix publicised groups GET API (singular) over federation (PR #2772)
-* Fix user directory when using ``user_directory_search_all_users`` config option (PR #2803, #2831)
-* Fix error on ``/publicRooms`` when no rooms exist (PR #2827)
-* Fix bug in quarantine_media (PR #2837)
-* Fix url_previews when no Content-Type is returned from URL (PR #2845)
-* Fix rare race in sync API when joining room (PR #2944)
-* Fix slow event search, switch back from GIST to GIN indexes (PR #2769, #2848)
-
-
-Changes in synapse v0.26.0 (2018-01-05)
-=======================================
-
-No changes since v0.26.0-rc1
-
-
-Changes in synapse v0.26.0-rc1 (2017-12-13)
-===========================================
-
-Features:
-
-* Add ability for ASes to publicise groups for their users (PR #2686)
-* Add all local users to the user_directory and optionally search them (PR
-  #2723)
-* Add support for custom login types for validating users (PR #2729)
-
-
-Changes:
-
-* Update example Prometheus config to new format (PR #2648) Thanks to
-  @krombel!
-* Rename redact_content option to include_content in Push API (PR #2650)
-* Declare support for r0.3.0 (PR #2677)
-* Improve upserts (PR #2684, #2688, #2689, #2713)
-* Improve documentation of workers (PR #2700)
-* Improve tracebacks on exceptions (PR #2705)
-* Allow guest access to group APIs for reading (PR #2715)
-* Support for posting content in federation_client script (PR #2716)
-* Delete devices and pushers on logouts etc (PR #2722)
-
-
-Bug fixes:
-
-* Fix database port script (PR #2673)
-* Fix internal server error on login with ldap_auth_provider (PR #2678) Thanks
-  to @jkolo!
-* Fix error on sqlite 3.7 (PR #2697)
-* Fix OPTIONS on preview_url (PR #2707)
-* Fix error handling on dns lookup (PR #2711)
-* Fix wrong avatars when inviting multiple users when creating room (PR #2717)
-* Fix 500 when joining matrix-dev (PR #2719)
-
-
-Changes in synapse v0.25.1 (2017-11-17)
-=======================================
-
-Bug fixes:
-
-* Fix login with LDAP and other password provider modules (PR #2678). Thanks to
-  @jkolo!
-
-Changes in synapse v0.25.0 (2017-11-15)
-=======================================
-
-Bug fixes:
-
-* Fix port script (PR #2673)
-
-
-Changes in synapse v0.25.0-rc1 (2017-11-14)
-===========================================
-
-Features:
-
-* Add is_public to groups table to allow for private groups (PR #2582)
-* Add a route for determining who you are (PR #2668) Thanks to @turt2live!
-* Add more features to the password providers (PR #2608, #2610, #2620, #2622,
-  #2623, #2624, #2626, #2628, #2629)
-* Add a hook for custom rest endpoints (PR #2627)
-* Add API to update group room visibility (PR #2651)
-
-
-Changes:
-
-* Ignore <noscript> tags when generating URL preview descriptions (PR #2576)
-  Thanks to @maximevaillancourt!
-* Register some /unstable endpoints in /r0 as well (PR #2579) Thanks to
-  @krombel!
-* Support /keys/upload on /r0 as well as /unstable (PR #2585)
-* Front-end proxy: pass through auth header (PR #2586)
-* Allow ASes to deactivate their own users (PR #2589)
-* Remove refresh tokens (PR #2613)
-* Automatically set default displayname on register (PR #2617)
-* Log login requests (PR #2618)
-* Always return `is_public` in the `/groups/:group_id/rooms` API (PR #2630)
-* Avoid no-op media deletes (PR #2637) Thanks to @spantaleev!
-* Fix various embarrassing typos around user_directory and add some doc. (PR
-  #2643)
-* Return whether a user is an admin within a group (PR #2647)
-* Namespace visibility options for groups (PR #2657)
-* Downcase UserIDs on registration (PR #2662)
-* Cache failures when fetching URL previews (PR #2669)
-
-
-Bug fixes:
-
-* Fix port script (PR #2577)
-* Fix error when running synapse with no logfile (PR #2581)
-* Fix UI auth when deleting devices (PR #2591)
-* Fix typo when checking if user is invited to group (PR #2599)
-* Fix the port script to drop NUL values in all tables (PR #2611)
-* Fix appservices being backlogged and not receiving new events due to a bug in
-  notify_interested_services (PR #2631) Thanks to @xyzz!
-* Fix updating rooms avatar/display name when modified by admin (PR #2636)
-  Thanks to @farialima!
-* Fix bug in state group storage (PR #2649)
-* Fix 500 on invalid utf-8 in request (PR #2663)
-
-
-Changes in synapse v0.24.1 (2017-10-24)
-=======================================
-
-Bug fixes:
-
-* Fix updating group profiles over federation (PR #2567)
-
-
-Changes in synapse v0.24.0 (2017-10-23)
-=======================================
-
-No changes since v0.24.0-rc1
-
-
-Changes in synapse v0.24.0-rc1 (2017-10-19)
-===========================================
-
-Features:
-
-* Add Group Server (PR #2352, #2363, #2374, #2377, #2378, #2382, #2410, #2426,
-  #2430, #2454, #2471, #2472, #2544)
-* Add support for channel notifications (PR #2501)
-* Add basic implementation of backup media store (PR #2538)
-* Add config option to auto-join new users to rooms (PR #2545)
-
-
-Changes:
-
-* Make the spam checker a module (PR #2474)
-* Delete expired url cache data (PR #2478)
-* Ignore incoming events for rooms that we have left (PR #2490)
-* Allow spam checker to reject invites too (PR #2492)
-* Add room creation checks to spam checker (PR #2495)
-* Spam checking: add the invitee to user_may_invite (PR #2502)
-* Process events from federation for different rooms in parallel (PR #2520)
-* Allow error strings from spam checker (PR #2531)
-* Improve error handling for missing files in config (PR #2551)
-
-
-Bug fixes:
-
-* Fix handling SERVFAILs when doing AAAA lookups for federation (PR #2477)
-* Fix incompatibility with newer versions of ujson (PR #2483) Thanks to
-  @jeremycline!
-* Fix notification keywords that start/end with non-word chars (PR #2500)
-* Fix stack overflow and logcontexts from linearizer (PR #2532)
-* Fix 500 error when fields missing from power_levels event (PR #2552)
-* Fix 500 error when we get an error handling a PDU (PR #2553)
-
-
-Changes in synapse v0.23.1 (2017-10-02)
-=======================================
-
-Changes:
-
-* Make 'affinity' package optional, as it is not supported on some platforms
-
-
-Changes in synapse v0.23.0 (2017-10-02)
-=======================================
-
-No changes since v0.23.0-rc2
-
-
-Changes in synapse v0.23.0-rc2 (2017-09-26)
-===========================================
-
-Bug fixes:
-
-* Fix regression in performance of syncs (PR #2470)
-
-
-Changes in synapse v0.23.0-rc1 (2017-09-25)
-===========================================
-
-Features:
-
-* Add a frontend proxy worker (PR #2344)
-* Add support for event_id_only push format (PR #2450)
-* Add a PoC for filtering spammy events (PR #2456)
-* Add a config option to block all room invites (PR #2457)
-
-
-Changes:
-
-* Use bcrypt module instead of py-bcrypt (PR #2288) Thanks to @kyrias!
-* Improve performance of generating push notifications (PR #2343, #2357, #2365,
-  #2366, #2371)
-* Improve DB performance for device list handling in sync (PR #2362)
-* Include a sample prometheus config (PR #2416)
-* Document known to work postgres version (PR #2433) Thanks to @ptman!
-
-
-Bug fixes:
-
-* Fix caching error in the push evaluator (PR #2332)
-* Fix bug where pusherpool didn't start and broke some rooms (PR #2342)
-* Fix port script for user directory tables (PR #2375)
-* Fix device lists notifications when user rejoins a room (PR #2443, #2449)
-* Fix sync to always send down current state events in timeline (PR #2451)
-* Fix bug where guest users were incorrectly kicked (PR #2453)
-* Fix bug talking to IPv6 only servers using SRV records (PR #2462)
-
-
-Changes in synapse v0.22.1 (2017-07-06)
-=======================================
-
-Bug fixes:
-
-* Fix bug where pusher pool didn't start and caused issues when
-  interacting with some rooms (PR #2342)
-
-
-Changes in synapse v0.22.0 (2017-07-06)
-=======================================
-
-No changes since v0.22.0-rc2
-
-
-Changes in synapse v0.22.0-rc2 (2017-07-04)
-===========================================
-
-Changes:
-
-* Improve performance of storing user IPs (PR #2307, #2308)
-* Slightly improve performance of verifying access tokens (PR #2320)
-* Slightly improve performance of event persistence (PR #2321)
-* Increase default cache factor size from 0.1 to 0.5 (PR #2330)
-
-Bug fixes:
-
-* Fix bug with storing registration sessions that caused frequent CPU churn
-  (PR #2319)
-
-
-Changes in synapse v0.22.0-rc1 (2017-06-26)
-===========================================
-
-Features:
-
-* Add a user directory API (PR #2252, and many more)
-* Add shutdown room API to remove room from local server (PR #2291)
-* Add API to quarantine media (PR #2292)
-* Add new config option to not send event contents to push servers (PR #2301)
-  Thanks to @cjdelisle!
-
-Changes:
-
-* Various performance fixes (PR #2177, #2233, #2230, #2238, #2248, #2256,
-  #2274)
-* Deduplicate sync filters (PR #2219) Thanks to @krombel!
-* Correct a typo in UPGRADE.rst (PR #2231) Thanks to @aaronraimist!
-* Add count of one time keys to sync stream (PR #2237)
-* Only store event_auth for state events (PR #2247)
-* Store URL cache preview downloads separately (PR #2299)
-
-Bug fixes:
-
-* Fix users not getting notifications when AS listened to that user_id (PR
-  #2216) Thanks to @slipeer!
-* Fix users without push set up not getting notifications after joining rooms
-  (PR #2236)
-* Fix preview url API to trim long descriptions (PR #2243)
-* Fix bug where we used cached but unpersisted state group as prev group,
-  resulting in broken state of restart (PR #2263)
-* Fix removing of pushers when using workers (PR #2267)
-* Fix CORS headers to allow Authorization header (PR #2285) Thanks to @krombel!
-
-
-Changes in synapse v0.21.1 (2017-06-15)
-=======================================
-
-Bug fixes:
-
-* Fix bug in anonymous usage statistic reporting (PR #2281)
-
-
-Changes in synapse v0.21.0 (2017-05-18)
-=======================================
-
-No changes since v0.21.0-rc3
-
-
-Changes in synapse v0.21.0-rc3 (2017-05-17)
-===========================================
-
-Features:
-
-* Add per user rate-limiting overrides (PR #2208)
-* Add config option to limit maximum number of events requested by ``/sync``
-  and ``/messages`` (PR #2221) Thanks to @psaavedra!
-
-
-Changes:
-
-* Various small performance fixes (PR #2201, #2202, #2224, #2226, #2227, #2228,
-  #2229)
-* Update username availability checker API (PR #2209, #2213)
-* When purging, don't de-delta state groups we're about to delete (PR #2214)
-* Documentation to check synapse version (PR #2215) Thanks to @hamber-dick!
-* Add an index to event_search to speed up purge history API (PR #2218)
-
-
-Bug fixes:
-
-* Fix API to allow clients to upload one-time-keys with new sigs (PR #2206)
-
-
-Changes in synapse v0.21.0-rc2 (2017-05-08)
-===========================================
-
-Changes:
-
-* Always mark remotes as up if we receive a signed request from them (PR #2190)
-
-
-Bug fixes:
-
-* Fix bug where users got pushed for rooms they had muted (PR #2200)
-
-
-Changes in synapse v0.21.0-rc1 (2017-05-08)
-===========================================
-
-Features:
-
-* Add username availability checker API (PR #2183)
-* Add read marker API (PR #2120)
-
-
-Changes:
-
-* Enable guest access for the 3pl/3pid APIs (PR #1986)
-* Add setting to support TURN for guests (PR #2011)
-* Various performance improvements (PR #2075, #2076, #2080, #2083, #2108,
-  #2158, #2176, #2185)
-* Make synctl a bit more user friendly (PR #2078, #2127) Thanks @APwhitehat!
-* Replace HTTP replication with TCP replication (PR #2082, #2097, #2098,
-  #2099, #2103, #2014, #2016, #2115, #2116, #2117)
-* Support authenticated SMTP (PR #2102) Thanks @DanielDent!
-* Add a counter metric for successfully-sent transactions (PR #2121)
-* Propagate errors sensibly from proxied IS requests (PR #2147)
-* Add more granular event send metrics (PR #2178)
-
-
-
-Bug fixes:
-
-* Fix nuke-room script to work with current schema (PR #1927) Thanks
-  @zuckschwerdt!
-* Fix db port script to not assume postgres tables are in the public schema
-  (PR #2024) Thanks @jerrykan!
-* Fix getting latest device IP for user with no devices (PR #2118)
-* Fix rejection of invites to unreachable servers (PR #2145)
-* Fix code for reporting old verify keys in synapse (PR #2156)
-* Fix invite state to always include all events (PR #2163)
-* Fix bug where synapse would always fetch state for any missing event (PR #2170)
-* Fix a leak with timed out HTTP connections (PR #2180)
-* Fix bug where we didn't time out HTTP requests to ASes  (PR #2192)
-
-
-Docs:
-
-* Clarify doc for SQLite to PostgreSQL port (PR #1961) Thanks @benhylau!
-* Fix typo in synctl help (PR #2107) Thanks @HarHarLinks!
-* ``web_client_location`` documentation fix (PR #2131) Thanks @matthewjwolff!
-* Update README.rst with FreeBSD changes (PR #2132) Thanks @feld!
-* Clarify setting up metrics (PR #2149) Thanks @encks!
-
-
-Changes in synapse v0.20.0 (2017-04-11)
-=======================================
-
-Bug fixes:
-
-* Fix joining rooms over federation where not all servers in the room saw the
-  new server had joined (PR #2094)
-
-
-Changes in synapse v0.20.0-rc1 (2017-03-30)
-===========================================
-
-Features:
-
-* Add delete_devices API (PR #1993)
-* Add phone number registration/login support (PR #1994, #2055)
-
-
-Changes:
-
-* Use JSONSchema for validation of filters. Thanks @pik! (PR #1783)
-* Reread log config on SIGHUP (PR #1982)
-* Speed up public room list (PR #1989)
-* Add helpful texts to logger config options (PR #1990)
-* Minor ``/sync`` performance improvements. (PR #2002, #2013, #2022)
-* Add some debug to help diagnose weird federation issue (PR #2035)
-* Correctly limit retries for all federation requests (PR #2050, #2061)
-* Don't lock table when persisting new one time keys (PR #2053)
-* Reduce some CPU work on DB threads (PR #2054)
-* Cache hosts in room (PR #2060)
-* Batch sending of device list pokes (PR #2063)
-* Speed up persist event path in certain edge cases (PR #2070)
-
-
-Bug fixes:
-
-* Fix bug where current_state_events renamed to current_state_ids (PR #1849)
-* Fix routing loop when fetching remote media (PR #1992)
-* Fix current_state_events table to not lie (PR #1996)
-* Fix CAS login to handle PartialDownloadError (PR #1997)
-* Fix assertion to stop transaction queue getting wedged (PR #2010)
-* Fix presence to fallback to last_active_ts if it beats the last sync time.
-  Thanks @Half-Shot! (PR #2014)
-* Fix bug when federation received a PDU while a room join is in progress (PR
-  #2016)
-* Fix resetting state on rejected events (PR #2025)
-* Fix installation issues in readme. Thanks @ricco386 (PR #2037)
-* Fix caching of remote servers' signature keys (PR #2042)
-* Fix some leaking log context (PR #2048, #2049, #2057, #2058)
-* Fix rejection of invites not reaching sync (PR #2056)
-
-
-
-Changes in synapse v0.19.3 (2017-03-20)
-=======================================
-
-No changes since v0.19.3-rc2
-
-
-Changes in synapse v0.19.3-rc2 (2017-03-13)
-===========================================
-
-Bug fixes:
-
-* Fix bug in handling of incoming device list updates over federation.
-
-
-
-Changes in synapse v0.19.3-rc1 (2017-03-08)
-===========================================
-
-Features:
-
-* Add some administration functionalities. Thanks to morteza-araby! (PR #1784)
-
-
-Changes:
-
-* Reduce database table sizes (PR #1873, #1916, #1923, #1963)
-* Update contrib/ to not use syutil. Thanks to andrewshadura! (PR #1907)
-* Don't fetch current state when sending an event in common case (PR #1955)
-
-
-Bug fixes:
-
-* Fix synapse_port_db failure. Thanks to Pneumaticat! (PR #1904)
-* Fix caching to not cache error responses (PR #1913)
-* Fix APIs to make kick & ban reasons work (PR #1917)
-* Fix bugs in the /keys/changes api (PR #1921)
-* Fix bug where users couldn't forget rooms they were banned from (PR #1922)
-* Fix issue with long language values in pushers API (PR #1925)
-* Fix a race in transaction queue (PR #1930)
-* Fix dynamic thumbnailing to preserve aspect ratio. Thanks to jkolo! (PR
-  #1945)
-* Fix device list update to not constantly resync (PR #1964)
-* Fix potential for huge memory usage when getting device that have
-  changed (PR #1969)
-
-
-
-Changes in synapse v0.19.2 (2017-02-20)
-=======================================
-
-* Fix bug with event visibility check in /context/ API. Thanks to Tokodomo for
-  pointing it out! (PR #1929)
-
-
-Changes in synapse v0.19.1 (2017-02-09)
-=======================================
-
-* Fix bug where state was incorrectly reset in a room when synapse received an
-  event over federation that did not pass auth checks (PR #1892)
-
-
-Changes in synapse v0.19.0 (2017-02-04)
-=======================================
-
-No changes since RC 4.
-
-
-Changes in synapse v0.19.0-rc4 (2017-02-02)
-===========================================
-
-* Bump cache sizes for common membership queries (PR #1879)
-
-
-Changes in synapse v0.19.0-rc3 (2017-02-02)
-===========================================
-
-* Fix email push in pusher worker (PR #1875)
-* Make presence.get_new_events a bit faster (PR #1876)
-* Make /keys/changes a bit more performant (PR #1877)
-
-
-Changes in synapse v0.19.0-rc2 (2017-02-02)
-===========================================
-
-* Include newly joined users in /keys/changes API (PR #1872)
-
-
-Changes in synapse v0.19.0-rc1 (2017-02-02)
-===========================================
-
-Features:
-
-* Add support for specifying multiple bind addresses (PR #1709, #1712, #1795,
-  #1835). Thanks to @kyrias!
-* Add /account/3pid/delete endpoint (PR #1714)
-* Add config option to configure the Riot URL used in notification emails (PR
-  #1811). Thanks to @aperezdc!
-* Add username and password config options for turn server (PR #1832). Thanks
-  to @xsteadfastx!
-* Implement device lists updates over federation (PR #1857, #1861, #1864)
-* Implement /keys/changes (PR #1869, #1872)
-
-
-Changes:
-
-* Improve IPv6 support (PR #1696). Thanks to @kyrias and @glyph!
-* Log which files we saved attachments to in the media_repository (PR #1791)
-* Linearize updates to membership via PUT /state/ to better handle multiple
-  joins (PR #1787)
-* Limit number of entries to prefill from cache on startup (PR #1792)
-* Remove full_twisted_stacktraces option (PR #1802)
-* Measure size of some caches by sum of the size of cached values (PR #1815)
-* Measure metrics of string_cache (PR #1821)
-* Reduce logging verbosity (PR #1822, #1823, #1824)
-* Don't clobber a displayname or avatar_url if provided by an m.room.member
-  event (PR #1852)
-* Better handle 401/404 response for federation /send/ (PR #1866, #1871)
-
-
-Fixes:
-
-* Fix ability to change password to a non-ascii one (PR #1711)
-* Fix push getting stuck due to looking at the wrong view of state (PR #1820)
-* Fix email address comparison to be case insensitive (PR #1827)
-* Fix occasional inconsistencies of room membership (PR #1836, #1840)
-
-
-Performance:
-
-* Don't block messages sending on bumping presence (PR #1789)
-* Change device_inbox stream index to include user (PR #1793)
-* Optimise state resolution (PR #1818)
-* Use DB cache of joined users for presence (PR #1862)
-* Add an index to make membership queries faster (PR #1867)
-
-
-Changes in synapse v0.18.7 (2017-01-09)
-=======================================
-
-No changes from v0.18.7-rc2
-
-
-Changes in synapse v0.18.7-rc2 (2017-01-07)
-===========================================
-
-Bug fixes:
-
-* Fix error in rc1's discarding invalid inbound traffic logic that was
-  incorrectly discarding missing events
-
-
-Changes in synapse v0.18.7-rc1 (2017-01-06)
-===========================================
-
-Bug fixes:
-
-* Fix error in #PR 1764 to actually fix the nightmare #1753 bug.
-* Improve deadlock logging further
-* Discard inbound federation traffic from invalid domains, to immunise
-  against #1753
-
-
-Changes in synapse v0.18.6 (2017-01-06)
-=======================================
-
-Bug fixes:
-
-* Fix bug when checking if a guest user is allowed to join a room (PR #1772)
-  Thanks to Patrik Oldsberg for diagnosing and the fix!
-
-
-Changes in synapse v0.18.6-rc3 (2017-01-05)
-===========================================
-
-Bug fixes:
-
-* Fix bug where we failed to send ban events to the banned server (PR #1758)
-* Fix bug where we sent event that didn't originate on this server to
-  other servers (PR #1764)
-* Fix bug where processing an event from a remote server took a long time
-  because we were making long HTTP requests (PR #1765, PR #1744)
-
-Changes:
-
-* Improve logging for debugging deadlocks (PR #1766, PR #1767)
-
-
-Changes in synapse v0.18.6-rc2 (2016-12-30)
-===========================================
-
-Bug fixes:
-
-* Fix memory leak in twisted by initialising logging correctly (PR #1731)
-* Fix bug where fetching missing events took an unacceptable amount of time in
-  large rooms (PR #1734)
-
-
-Changes in synapse v0.18.6-rc1 (2016-12-29)
-===========================================
-
-Bug fixes:
-
-* Make sure that outbound connections are closed (PR #1725)
-
-
-Changes in synapse v0.18.5 (2016-12-16)
-=======================================
-
-Bug fixes:
-
-* Fix federation /backfill returning events it shouldn't (PR #1700)
-* Fix crash in url preview (PR #1701)
-
-
-Changes in synapse v0.18.5-rc3 (2016-12-13)
-===========================================
-
-Features:
-
-* Add support for E2E for guests (PR #1653)
-* Add new API appservice specific public room list (PR #1676)
-* Add new room membership APIs (PR #1680)
-
-
-Changes:
-
-* Enable guest access for private rooms by default (PR #653)
-* Limit the number of events that can be created on a given room concurrently
-  (PR #1620)
-* Log the args that we have on UI auth completion (PR #1649)
-* Stop generating refresh_tokens (PR #1654)
-* Stop putting a time caveat on access tokens (PR #1656)
-* Remove unspecced GET endpoints for e2e keys (PR #1694)
-
-
-Bug fixes:
-
-* Fix handling of 500 and 429's over federation (PR #1650)
-* Fix Content-Type header parsing (PR #1660)
-* Fix error when previewing sites that include unicode, thanks to kyrias (PR
-  #1664)
-* Fix some cases where we drop read receipts (PR #1678)
-* Fix bug where calls to ``/sync`` didn't correctly timeout (PR #1683)
-* Fix bug where E2E key query would fail if a single remote host failed (PR
-  #1686)
-
-
-
-Changes in synapse v0.18.5-rc2 (2016-11-24)
-===========================================
-
-Bug fixes:
-
-* Don't send old events over federation, fixes bug in -rc1.
-
-Changes in synapse v0.18.5-rc1 (2016-11-24)
-===========================================
-
-Features:
-
-* Implement "event_fields" in filters (PR #1638)
-
-Changes:
-
-* Use external ldap auth pacakge (PR #1628)
-* Split out federation transaction sending to a worker (PR #1635)
-* Fail with a coherent error message if `/sync?filter=` is invalid (PR #1636)
-* More efficient notif count queries (PR #1644)
-
-
-Changes in synapse v0.18.4 (2016-11-22)
-=======================================
-
-Bug fixes:
-
-* Add workaround for buggy clients that the fail to register (PR #1632)
-
-
-Changes in synapse v0.18.4-rc1 (2016-11-14)
-===========================================
-
-Changes:
-
-* Various database efficiency improvements (PR #1188, #1192)
-* Update default config to blacklist more internal IPs, thanks to Euan Kemp (PR
-  #1198)
-* Allow specifying duration in minutes in config, thanks to Daniel Dent (PR
-  #1625)
-
-
-Bug fixes:
-
-* Fix media repo to set CORs headers on responses (PR #1190)
-* Fix registration to not error on non-ascii passwords (PR #1191)
-* Fix create event code to limit the number of prev_events (PR #1615)
-* Fix bug in transaction ID deduplication (PR #1624)
-
-
-Changes in synapse v0.18.3 (2016-11-08)
-=======================================
-
-SECURITY UPDATE
-
-Explicitly require authentication when using LDAP3. This is the default on
-versions of ``ldap3`` above 1.0, but some distributions will package an older
-version.
-
-If you are using LDAP3 login and have a version of ``ldap3`` older than 1.0 it
-is **CRITICAL to updgrade**.
-
-
-Changes in synapse v0.18.2 (2016-11-01)
-=======================================
-
-No changes since v0.18.2-rc5
-
-
-Changes in synapse v0.18.2-rc5 (2016-10-28)
-===========================================
-
-Bug fixes:
-
-* Fix prometheus process metrics in worker processes (PR #1184)
-
-
-Changes in synapse v0.18.2-rc4 (2016-10-27)
-===========================================
-
-Bug fixes:
-
-* Fix ``user_threepids`` schema delta, which in some instances prevented
-  startup after upgrade (PR #1183)
-
-
-Changes in synapse v0.18.2-rc3 (2016-10-27)
-===========================================
-
-Changes:
-
-* Allow clients to supply access tokens as headers (PR #1098)
-* Clarify error codes for GET /filter/, thanks to Alexander Maznev (PR #1164)
-* Make password reset email field case insensitive (PR #1170)
-* Reduce redundant database work in email pusher (PR #1174)
-* Allow configurable rate limiting per AS (PR #1175)
-* Check whether to ratelimit sooner to avoid work (PR #1176)
-* Standardise prometheus metrics (PR #1177)
-
-
-Bug fixes:
-
-* Fix incredibly slow back pagination query (PR #1178)
-* Fix infinite typing bug (PR #1179)
-
-
-Changes in synapse v0.18.2-rc2 (2016-10-25)
-===========================================
-
-(This release did not include the changes advertised and was identical to RC1)
-
-
-Changes in synapse v0.18.2-rc1 (2016-10-17)
-===========================================
-
-Changes:
-
-* Remove redundant event_auth index (PR #1113)
-* Reduce DB hits for replication (PR #1141)
-* Implement pluggable password auth (PR #1155)
-* Remove rate limiting from app service senders and fix get_or_create_user
-  requester, thanks to Patrik Oldsberg (PR #1157)
-* window.postmessage for Interactive Auth fallback (PR #1159)
-* Use sys.executable instead of hardcoded python, thanks to Pedro Larroy
-  (PR #1162)
-* Add config option for adding additional TLS fingerprints (PR #1167)
-* User-interactive auth on delete device (PR #1168)
-
-
-Bug fixes:
-
-* Fix not being allowed to set your own state_key, thanks to Patrik Oldsberg
-  (PR #1150)
-* Fix interactive auth to return 401 from for incorrect password (PR #1160,
-  #1166)
-* Fix email push notifs being dropped (PR #1169)
-
-
-
-Changes in synapse v0.18.1 (2016-10-05)
-======================================
-
-No changes since v0.18.1-rc1
-
-
-Changes in synapse v0.18.1-rc1 (2016-09-30)
-===========================================
-
-Features:
-
-* Add total_room_count_estimate to ``/publicRooms`` (PR #1133)
-
-
-Changes:
-
-* Time out typing over federation (PR #1140)
-* Restructure LDAP authentication (PR #1153)
-
-
-Bug fixes:
-
-* Fix 3pid invites when server is already in the room (PR #1136)
-* Fix upgrading with SQLite taking lots of CPU for a few days
-  after upgrade (PR #1144)
-* Fix upgrading from very old database versions (PR #1145)
-* Fix port script to work with recently added tables (PR #1146)
-
-
-Changes in synapse v0.18.0 (2016-09-19)
-=======================================
-
-The release includes major changes to the state storage database schemas, which
-significantly reduce database size. Synapse will attempt to upgrade the current
-data in the background. Servers with large SQLite database may experience
-degradation of performance while this upgrade is in progress, therefore you may
-want to consider migrating to using Postgres before upgrading very large SQLite
-databases
-
-
-Changes:
-
-* Make public room search case insensitive (PR #1127)
-
-
-Bug fixes:
-
-* Fix and clean up publicRooms pagination (PR #1129)
-
-
-Changes in synapse v0.18.0-rc1 (2016-09-16)
-===========================================
-
-Features:
-
-* Add ``only=highlight`` on ``/notifications`` (PR #1081)
-* Add server param to /publicRooms (PR #1082)
-* Allow clients to ask for the whole of a single state event (PR #1094)
-* Add is_direct param to /createRoom (PR #1108)
-* Add pagination support to publicRooms (PR #1121)
-* Add very basic filter API to /publicRooms (PR #1126)
-* Add basic direct to device messaging support for E2E (PR #1074, #1084, #1104,
-  #1111)
-
-
-Changes:
-
-* Move to storing state_groups_state as deltas, greatly reducing DB size (PR
-  #1065)
-* Reduce amount of state pulled out of the DB during common requests (PR #1069)
-* Allow PDF to be rendered from media repo (PR #1071)
-* Reindex state_groups_state after pruning (PR #1085)
-* Clobber EDUs in send queue (PR #1095)
-* Conform better to the CAS protocol specification (PR #1100)
-* Limit how often we ask for keys from dead servers (PR #1114)
-
-
-Bug fixes:
-
-* Fix /notifications API when used with ``from`` param (PR #1080)
-* Fix backfill when cannot find an event. (PR #1107)
-
-
-Changes in synapse v0.17.3 (2016-09-09)
-=======================================
-
-This release fixes a major bug that stopped servers from handling rooms with
-over 1000 members.
-
-
-Changes in synapse v0.17.2 (2016-09-08)
-=======================================
-
-This release contains security bug fixes. Please upgrade.
-
-
-No changes since v0.17.2-rc1
-
-
-Changes in synapse v0.17.2-rc1 (2016-09-05)
-===========================================
-
-Features:
-
-* Start adding store-and-forward direct-to-device messaging (PR #1046, #1050,
-  #1062, #1066)
-
-
-Changes:
-
-* Avoid pulling the full state of a room out so often (PR #1047, #1049, #1063,
-  #1068)
-* Don't notify for online to online presence transitions. (PR #1054)
-* Occasionally persist unpersisted presence updates (PR #1055)
-* Allow application services to have an optional 'url' (PR #1056)
-* Clean up old sent transactions from DB (PR #1059)
-
-
-Bug fixes:
-
-* Fix None check in backfill (PR #1043)
-* Fix membership changes to be idempotent (PR #1067)
-* Fix bug in get_pdu where it would sometimes return events with incorrect
-  signature
-
-
-
-Changes in synapse v0.17.1 (2016-08-24)
-=======================================
-
-Changes:
-
-* Delete old received_transactions rows (PR #1038)
-* Pass through user-supplied content in /join/$room_id (PR #1039)
-
-
-Bug fixes:
-
-* Fix bug with backfill (PR #1040)
-
-
-Changes in synapse v0.17.1-rc1 (2016-08-22)
-===========================================
-
-Features:
-
-* Add notification API (PR #1028)
-
-
-Changes:
-
-* Don't print stack traces when failing to get remote keys (PR #996)
-* Various federation /event/ perf improvements (PR #998)
-* Only process one local membership event per room at a time (PR #1005)
-* Move default display name push rule (PR #1011, #1023)
-* Fix up preview URL API. Add tests. (PR #1015)
-* Set ``Content-Security-Policy`` on media repo (PR #1021)
-* Make notify_interested_services faster (PR #1022)
-* Add usage stats to prometheus monitoring (PR #1037)
-
-
-Bug fixes:
-
-* Fix token login (PR #993)
-* Fix CAS login (PR #994, #995)
-* Fix /sync to not clobber status_msg (PR #997)
-* Fix redacted state events to include prev_content (PR #1003)
-* Fix some bugs in the auth/ldap handler (PR #1007)
-* Fix backfill request to limit URI length, so that remotes don't reject the
-  requests due to path length limits (PR #1012)
-* Fix AS push code to not send duplicate events (PR #1025)
-
-
-
-Changes in synapse v0.17.0 (2016-08-08)
-=======================================
-
-This release contains significant security bug fixes regarding authenticating
-events received over federation. PLEASE UPGRADE.
-
-This release changes the LDAP configuration format in a backwards incompatible
-way, see PR #843 for details.
-
-
-Changes:
-
-* Add federation /version API (PR #990)
-* Make psutil dependency optional (PR #992)
-
-
-Bug fixes:
-
-* Fix URL preview API to exclude HTML comments in description (PR #988)
-* Fix error handling of remote joins (PR #991)
-
-
-Changes in synapse v0.17.0-rc4 (2016-08-05)
-===========================================
-
-Changes:
-
-* Change the way we summarize URLs when previewing (PR #973)
-* Add new ``/state_ids/`` federation API (PR #979)
-* Speed up processing of ``/state/`` response (PR #986)
-
-Bug fixes:
-
-* Fix event persistence when event has already been partially persisted
-  (PR #975, #983, #985)
-* Fix port script to also copy across backfilled events (PR #982)
-
-
-Changes in synapse v0.17.0-rc3 (2016-08-02)
-===========================================
-
-Changes:
-
-* Forbid non-ASes from registering users whose names begin with '_' (PR #958)
-* Add some basic admin API docs (PR #963)
-
-
-Bug fixes:
-
-* Send the correct host header when fetching keys (PR #941)
-* Fix joining a room that has missing auth events (PR #964)
-* Fix various push bugs (PR #966, #970)
-* Fix adding emails on registration (PR #968)
-
-
-Changes in synapse v0.17.0-rc2 (2016-08-02)
-===========================================
-
-(This release did not include the changes advertised and was identical to RC1)
-
-
-Changes in synapse v0.17.0-rc1 (2016-07-28)
-===========================================
-
-This release changes the LDAP configuration format in a backwards incompatible
-way, see PR #843 for details.
-
-
-Features:
-
-* Add purge_media_cache admin API (PR #902)
-* Add deactivate account admin API (PR #903)
-* Add optional pepper to password hashing (PR #907, #910 by KentShikama)
-* Add an admin option to shared secret registration (breaks backwards compat)
-  (PR #909)
-* Add purge local room history API (PR #911, #923, #924)
-* Add requestToken endpoints (PR #915)
-* Add an /account/deactivate endpoint (PR #921)
-* Add filter param to /messages. Add 'contains_url' to filter. (PR #922)
-* Add device_id support to /login (PR #929)
-* Add device_id support to /v2/register flow. (PR #937, #942)
-* Add GET /devices endpoint (PR #939, #944)
-* Add GET /device/{deviceId} (PR #943)
-* Add update and delete APIs for devices (PR #949)
-
-
-Changes:
-
-* Rewrite LDAP Authentication against ldap3 (PR #843 by mweinelt)
-* Linearize some federation endpoints based on (origin, room_id) (PR #879)
-* Remove the legacy v0 content upload API. (PR #888)
-* Use similar naming we use in email notifs for push (PR #894)
-* Optionally include password hash in createUser endpoint (PR #905 by
-  KentShikama)
-* Use a query that postgresql optimises better for get_events_around (PR #906)
-* Fall back to 'username' if 'user' is not given for appservice registration.
-  (PR #927 by Half-Shot)
-* Add metrics for psutil derived memory usage (PR #936)
-* Record device_id in client_ips (PR #938)
-* Send the correct host header when fetching keys (PR #941)
-* Log the hostname the reCAPTCHA was completed on (PR #946)
-* Make the device id on e2e key upload optional (PR #956)
-* Add r0.2.0 to the "supported versions" list (PR #960)
-* Don't include name of room for invites in push (PR #961)
-
-
-Bug fixes:
-
-* Fix substitution failure in mail template (PR #887)
-* Put most recent 20 messages in email notif (PR #892)
-* Ensure that the guest user is in the database when upgrading accounts
-  (PR #914)
-* Fix various edge cases in auth handling (PR #919)
-* Fix 500 ISE when sending alias event without a state_key (PR #925)
-* Fix bug where we stored rejections in the state_group, persist all
-  rejections (PR #948)
-* Fix lack of check of if the user is banned when handling 3pid invites
-  (PR #952)
-* Fix a couple of bugs in the transaction and keyring code (PR #954, #955)
-
-
-
-Changes in synapse v0.16.1-r1 (2016-07-08)
-==========================================
-
-THIS IS A CRITICAL SECURITY UPDATE.
-
-This fixes a bug which allowed users' accounts to be accessed by unauthorised
-users.
-
-Changes in synapse v0.16.1 (2016-06-20)
-=======================================
-
-Bug fixes:
-
-* Fix assorted bugs in ``/preview_url`` (PR #872)
-* Fix TypeError when setting unicode passwords (PR #873)
-
-
-Performance improvements:
-
-* Turn ``use_frozen_events`` off by default (PR #877)
-* Disable responding with canonical json for federation (PR #878)
-
-
-Changes in synapse v0.16.1-rc1 (2016-06-15)
-===========================================
-
-Features: None
-
-Changes:
-
-* Log requester for ``/publicRoom`` endpoints when possible (PR #856)
-* 502 on ``/thumbnail`` when can't connect to remote server (PR #862)
-* Linearize fetching of gaps on incoming events (PR #871)
-
-
-Bugs fixes:
-
-* Fix bug where rooms where marked as published by default (PR #857)
-* Fix bug where joining room with an event with invalid sender (PR #868)
-* Fix bug where backfilled events were sent down sync streams (PR #869)
-* Fix bug where outgoing connections could wedge indefinitely, causing push
-  notifications to be unreliable (PR #870)
-
-
-Performance improvements:
-
-* Improve ``/publicRooms`` performance(PR #859)
-
-
-Changes in synapse v0.16.0 (2016-06-09)
-=======================================
-
-NB: As of v0.14 all AS config files must have an ID field.
-
-
-Bug fixes:
-
-* Don't make rooms published by default (PR #857)
-
-Changes in synapse v0.16.0-rc2 (2016-06-08)
-===========================================
-
-Features:
-
-* Add configuration option for tuning GC via ``gc.set_threshold`` (PR #849)
-
-Changes:
-
-* Record metrics about GC (PR #771, #847, #852)
-* Add metric counter for number of persisted events (PR #841)
-
-Bug fixes:
-
-* Fix 'From' header in email notifications (PR #843)
-* Fix presence where timeouts were not being fired for the first 8h after
-  restarts (PR #842)
-* Fix bug where synapse sent malformed transactions to AS's when retrying
-  transactions (Commits 310197b, 8437906)
-
-Performance improvements:
-
-* Remove event fetching from DB threads (PR #835)
-* Change the way we cache events (PR #836)
-* Add events to cache when we persist them (PR #840)
-
-
-Changes in synapse v0.16.0-rc1 (2016-06-03)
-===========================================
-
-Version 0.15 was not released. See v0.15.0-rc1 below for additional changes.
-
-Features:
-
-* Add email notifications for missed messages (PR #759, #786, #799, #810, #815,
-  #821)
-* Add a ``url_preview_ip_range_whitelist`` config param (PR #760)
-* Add /report endpoint (PR #762)
-* Add basic ignore user API (PR #763)
-* Add an openidish mechanism for proving that you own a given user_id (PR #765)
-* Allow clients to specify a server_name to avoid 'No known servers' (PR #794)
-* Add secondary_directory_servers option to fetch room list from other servers
-  (PR #808, #813)
-
-Changes:
-
-* Report per request metrics for all of the things using request_handler (PR
-  #756)
-* Correctly handle ``NULL`` password hashes from the database (PR #775)
-* Allow receipts for events we haven't seen in the db (PR #784)
-* Make synctl read a cache factor from config file (PR #785)
-* Increment badge count per missed convo, not per msg (PR #793)
-* Special case m.room.third_party_invite event auth to match invites (PR #814)
-
-
-Bug fixes:
-
-* Fix typo in event_auth servlet path (PR #757)
-* Fix password reset (PR #758)
-
-
-Performance improvements:
-
-* Reduce database inserts when sending transactions (PR #767)
-* Queue events by room for persistence (PR #768)
-* Add cache to ``get_user_by_id`` (PR #772)
-* Add and use ``get_domain_from_id`` (PR #773)
-* Use tree cache for ``get_linearized_receipts_for_room`` (PR #779)
-* Remove unused indices (PR #782)
-* Add caches to ``bulk_get_push_rules*`` (PR #804)
-* Cache ``get_event_reference_hashes`` (PR #806)
-* Add ``get_users_with_read_receipts_in_room`` cache (PR #809)
-* Use state to calculate ``get_users_in_room`` (PR #811)
-* Load push rules in storage layer so that they get cached (PR #825)
-* Make ``get_joined_hosts_for_room`` use get_users_in_room (PR #828)
-* Poke notifier on next reactor tick (PR #829)
-* Change CacheMetrics to be quicker (PR #830)
-
-
-Changes in synapse v0.15.0-rc1 (2016-04-26)
-===========================================
-
-Features:
-
-* Add login support for Javascript Web Tokens, thanks to Niklas Riekenbrauck
-  (PR #671,#687)
-* Add URL previewing support (PR #688)
-* Add login support for LDAP, thanks to Christoph Witzany (PR #701)
-* Add GET endpoint for pushers (PR #716)
-
-Changes:
-
-* Never notify for member events (PR #667)
-* Deduplicate identical ``/sync`` requests (PR #668)
-* Require user to have left room to forget room (PR #673)
-* Use DNS cache if within TTL (PR #677)
-* Let users see their own leave events (PR #699)
-* Deduplicate membership changes (PR #700)
-* Increase performance of pusher code (PR #705)
-* Respond with error status 504 if failed to talk to remote server (PR #731)
-* Increase search performance on postgres (PR #745)
-
-Bug fixes:
-
-* Fix bug where disabling all notifications still resulted in push (PR #678)
-* Fix bug where users couldn't reject remote invites if remote refused (PR #691)
-* Fix bug where synapse attempted to backfill from itself (PR #693)
-* Fix bug where profile information was not correctly added when joining remote
-  rooms (PR #703)
-* Fix bug where register API required incorrect key name for AS registration
-  (PR #727)
-
-
-Changes in synapse v0.14.0 (2016-03-30)
-=======================================
-
-No changes from v0.14.0-rc2
-
-Changes in synapse v0.14.0-rc2 (2016-03-23)
-===========================================
-
-Features:
-
-* Add published room list API (PR #657)
-
-Changes:
-
-* Change various caches to consume less memory (PR #656, #658, #660, #662,
-  #663, #665)
-* Allow rooms to be published without requiring an alias (PR #664)
-* Intern common strings in caches to reduce memory footprint (#666)
-
-Bug fixes:
-
-* Fix reject invites over federation (PR #646)
-* Fix bug where registration was not idempotent (PR #649)
-* Update aliases event after deleting aliases (PR #652)
-* Fix unread notification count, which was sometimes wrong (PR #661)
-
-Changes in synapse v0.14.0-rc1 (2016-03-14)
-===========================================
-
-Features:
-
-* Add event_id to response to state event PUT (PR #581)
-* Allow guest users access to messages in rooms they have joined (PR #587)
-* Add config for what state is included in a room invite (PR #598)
-* Send the inviter's member event in room invite state (PR #607)
-* Add error codes for malformed/bad JSON in /login (PR #608)
-* Add support for changing the actions for default rules (PR #609)
-* Add environment variable SYNAPSE_CACHE_FACTOR, default it to 0.1 (PR #612)
-* Add ability for alias creators to delete aliases (PR #614)
-* Add profile information to invites (PR #624)
-
-Changes:
-
-* Enforce user_id exclusivity for AS registrations (PR #572)
-* Make adding push rules idempotent (PR #587)
-* Improve presence performance (PR #582, #586)
-* Change presence semantics for ``last_active_ago`` (PR #582, #586)
-* Don't allow ``m.room.create`` to be changed (PR #596)
-* Add 800x600 to default list of valid thumbnail sizes (PR #616)
-* Always include kicks and bans in full /sync (PR #625)
-* Send history visibility on boundary changes (PR #626)
-* Register endpoint now returns a refresh_token (PR #637)
-
-Bug fixes:
-
-* Fix bug where we returned incorrect state in /sync (PR #573)
-* Always return a JSON object from push rule API (PR #606)
-* Fix bug where registering without a user id sometimes failed (PR #610)
-* Report size of ExpiringCache in cache size metrics (PR #611)
-* Fix rejection of invites to empty rooms (PR #615)
-* Fix usage of ``bcrypt`` to not use ``checkpw`` (PR #619)
-* Pin ``pysaml2`` dependency (PR #634)
-* Fix bug in ``/sync`` where timeline order was incorrect for backfilled events
-  (PR #635)
-
-Changes in synapse v0.13.3 (2016-02-11)
-=======================================
-
-* Fix bug where ``/sync`` would occasionally return events in the wrong room.
-
-Changes in synapse v0.13.2 (2016-02-11)
-=======================================
-
-* Fix bug where ``/events`` would fail to skip some events if there had been
-  more events than the limit specified since the last request (PR #570)
-
-Changes in synapse v0.13.1 (2016-02-10)
-=======================================
-
-* Bump matrix-angular-sdk (matrix web console) dependency to 0.6.8 to
-  pull in the fix for SYWEB-361 so that the default client can display
-  HTML messages again(!)
-
-Changes in synapse v0.13.0 (2016-02-10)
-=======================================
-
-This version includes an upgrade of the schema, specifically adding an index to
-the ``events`` table. This may cause synapse to pause for several minutes the
-first time it is started after the upgrade.
-
-Changes:
-
-* Improve general performance (PR #540, #543. #544, #54, #549, #567)
-* Change guest user ids to be incrementing integers (PR #550)
-* Improve performance of public room list API (PR #552)
-* Change profile API to omit keys rather than return null (PR #557)
-* Add ``/media/r0`` endpoint prefix, which is equivalent to ``/media/v1/``
-  (PR #595)
-
-Bug fixes:
-
-* Fix bug with upgrading guest accounts where it would fail if you opened the
-  registration email on a different device (PR #547)
-* Fix bug where unread count could be wrong (PR #568)
-
-
-
-Changes in synapse v0.12.1-rc1 (2016-01-29)
-===========================================
-
-Features:
-
-* Add unread notification counts in ``/sync`` (PR #456)
-* Add support for inviting 3pids in ``/createRoom`` (PR #460)
-* Add ability for guest accounts to upgrade (PR #462)
-* Add ``/versions`` API (PR #468)
-* Add ``event`` to ``/context`` API (PR #492)
-* Add specific error code for invalid user names in ``/register`` (PR #499)
-* Add support for push badge counts (PR #507)
-* Add support for non-guest users to peek in rooms using ``/events`` (PR #510)
-
-Changes:
-
-* Change ``/sync`` so that guest users only get rooms they've joined (PR #469)
-* Change to require unbanning before other membership changes (PR #501)
-* Change default push rules to notify for all messages (PR #486)
-* Change default push rules to not notify on membership changes (PR #514)
-* Change default push rules in one to one rooms to only notify for events that
-  are messages (PR #529)
-* Change ``/sync`` to reject requests with a ``from`` query param (PR #512)
-* Change server manhole to use SSH rather than telnet (PR #473)
-* Change server to require AS users to be registered before use (PR #487)
-* Change server not to start when ASes are invalidly configured (PR #494)
-* Change server to require ID and ``as_token`` to be unique for AS's (PR #496)
-* Change maximum pagination limit to 1000 (PR #497)
-
-Bug fixes:
-
-* Fix bug where ``/sync`` didn't return when something under the leave key
-  changed (PR #461)
-* Fix bug where we returned smaller rather than larger than requested
-  thumbnails when ``method=crop`` (PR #464)
-* Fix thumbnails API to only return cropped thumbnails when asking for a
-  cropped thumbnail (PR #475)
-* Fix bug where we occasionally still logged access tokens (PR #477)
-* Fix bug where ``/events`` would always return immediately for guest users
-  (PR #480)
-* Fix bug where ``/sync`` unexpectedly returned old left rooms (PR #481)
-* Fix enabling and disabling push rules (PR #498)
-* Fix bug where ``/register`` returned 500 when given unicode username
-  (PR #513)
-
-Changes in synapse v0.12.0 (2016-01-04)
-=======================================
-
-* Expose ``/login`` under ``r0`` (PR #459)
-
-Changes in synapse v0.12.0-rc3 (2015-12-23)
-===========================================
-
-* Allow guest accounts access to ``/sync`` (PR #455)
-* Allow filters to include/exclude rooms at the room level
-  rather than just from the components of the sync for each
-  room. (PR #454)
-* Include urls for room avatars in the response to ``/publicRooms`` (PR #453)
-* Don't set a identicon as the avatar for a user when they register (PR #450)
-* Add a ``display_name`` to third-party invites (PR #449)
-* Send more information to the identity server for third-party invites so that
-  it can send richer messages to the invitee (PR #446)
-* Cache the responses to ``/initialSync`` for 5 minutes. If a client
-  retries a request to ``/initialSync`` before the a response was computed
-  to the first request then the same response is used for both requests
-  (PR #457)
-* Fix a bug where synapse would always request the signing keys of
-  remote servers even when the key was cached locally (PR #452)
-* Fix 500 when pagination search results (PR #447)
-* Fix a bug where synapse was leaking raw email address in third-party invites
-  (PR #448)
-
-Changes in synapse v0.12.0-rc2 (2015-12-14)
-===========================================
-
-* Add caches for whether rooms have been forgotten by a user (PR #434)
-* Remove instructions to use ``--process-dependency-link`` since all of the
-  dependencies of synapse are on PyPI (PR #436)
-* Parallelise the processing of ``/sync`` requests (PR #437)
-* Fix race updating presence in ``/events`` (PR #444)
-* Fix bug back-populating search results (PR #441)
-* Fix bug calculating state in ``/sync`` requests (PR #442)
-
-Changes in synapse v0.12.0-rc1 (2015-12-10)
-===========================================
-
-* Host the client APIs released as r0 by
-  https://matrix.org/docs/spec/r0.0.0/client_server.html
-  on paths prefixed by ``/_matrix/client/r0``. (PR #430, PR #415, PR #400)
-* Updates the client APIs to match r0 of the matrix specification.
-
-  * All APIs return events in the new event format, old APIs also include
-    the fields needed to parse the event using the old format for
-    compatibility. (PR #402)
-  * Search results are now given as a JSON array rather than
-    a JSON object (PR #405)
-  * Miscellaneous changes to search (PR #403, PR #406, PR #412)
-  * Filter JSON objects may now be passed as query parameters to ``/sync``
-    (PR #431)
-  * Fix implementation of ``/admin/whois`` (PR #418)
-  * Only include the rooms that user has left in ``/sync`` if the client
-    requests them in the filter (PR #423)
-  * Don't push for ``m.room.message`` by default (PR #411)
-  * Add API for setting per account user data (PR #392)
-  * Allow users to forget rooms (PR #385)
-
-* Performance improvements and monitoring:
-
-  * Add per-request counters for CPU time spent on the main python thread.
-    (PR #421, PR #420)
-  * Add per-request counters for time spent in the database (PR #429)
-  * Make state updates in the C+S API idempotent (PR #416)
-  * Only fire ``user_joined_room`` if the user has actually joined. (PR #410)
-  * Reuse a single http client, rather than creating new ones (PR #413)
-
-* Fixed a bug upgrading from older versions of synapse on postgresql (PR #417)
-
-Changes in synapse v0.11.1 (2015-11-20)
-=======================================
-
-* Add extra options to search API (PR #394)
-* Fix bug where we did not correctly cap federation retry timers. This meant it
-  could take several hours for servers to start talking to ressurected servers,
-  even when they were receiving traffic from them (PR #393)
-* Don't advertise login token flow unless CAS is enabled. This caused issues
-  where some clients would always use the fallback API if they did not
-  recognize all login flows (PR #391)
-* Change /v2 sync API to rename ``private_user_data`` to ``account_data``
-  (PR #386)
-* Change /v2 sync API to remove the ``event_map`` and rename keys in ``rooms``
-  object (PR #389)
-
-Changes in synapse v0.11.0-r2 (2015-11-19)
-==========================================
-
-* Fix bug in database port script (PR #387)
-
-Changes in synapse v0.11.0-r1 (2015-11-18)
-==========================================
-
-* Retry and fail federation requests more aggressively for requests that block
-  client side requests (PR #384)
-
-Changes in synapse v0.11.0 (2015-11-17)
-=======================================
-
-* Change CAS login API (PR #349)
-
-Changes in synapse v0.11.0-rc2 (2015-11-13)
-===========================================
-
-* Various changes to /sync API response format (PR #373)
-* Fix regression when setting display name in newly joined room over
-  federation (PR #368)
-* Fix problem where /search was slow when using SQLite (PR #366)
-
-Changes in synapse v0.11.0-rc1 (2015-11-11)
-===========================================
-
-* Add Search API (PR #307, #324, #327, #336, #350, #359)
-* Add 'archived' state to v2 /sync API (PR #316)
-* Add ability to reject invites (PR #317)
-* Add config option to disable password login (PR #322)
-* Add the login fallback API (PR #330)
-* Add room context API (PR #334)
-* Add room tagging support (PR #335)
-* Update v2 /sync API to match spec (PR #305, #316, #321, #332, #337, #341)
-* Change retry schedule for application services (PR #320)
-* Change retry schedule for remote servers (PR #340)
-* Fix bug where we hosted static content in the incorrect place (PR #329)
-* Fix bug where we didn't increment retry interval for remote servers (PR #343)
-
-Changes in synapse v0.10.1-rc1 (2015-10-15)
-===========================================
-
-* Add support for CAS, thanks to Steven Hammerton (PR #295, #296)
-* Add support for using macaroons for ``access_token`` (PR #256, #229)
-* Add support for ``m.room.canonical_alias`` (PR #287)
-* Add support for viewing the history of rooms that they have left. (PR #276,
-  #294)
-* Add support for refresh tokens (PR #240)
-* Add flag on creation which disables federation of the room (PR #279)
-* Add some room state to invites. (PR #275)
-* Atomically persist events when joining a room over federation (PR #283)
-* Change default history visibility for private rooms (PR #271)
-* Allow users to redact their own sent events (PR #262)
-* Use tox for tests (PR #247)
-* Split up syutil into separate libraries (PR #243)
-
-Changes in synapse v0.10.0-r2 (2015-09-16)
-==========================================
-
-* Fix bug where we always fetched remote server signing keys instead of using
-  ones in our cache.
-* Fix adding threepids to an existing account.
-* Fix bug with invinting over federation where remote server was already in
-  the room. (PR #281, SYN-392)
-
-Changes in synapse v0.10.0-r1 (2015-09-08)
-==========================================
-
-* Fix bug with python packaging
-
-Changes in synapse v0.10.0 (2015-09-03)
-=======================================
-
-No change from release candidate.
-
-Changes in synapse v0.10.0-rc6 (2015-09-02)
-===========================================
-
-* Remove some of the old database upgrade scripts.
-* Fix database port script to work with newly created sqlite databases.
-
-Changes in synapse v0.10.0-rc5 (2015-08-27)
-===========================================
-
-* Fix bug that broke downloading files with ascii filenames across federation.
-
-Changes in synapse v0.10.0-rc4 (2015-08-27)
-===========================================
-
-* Allow UTF-8 filenames for upload. (PR #259)
-
-Changes in synapse v0.10.0-rc3 (2015-08-25)
-===========================================
-
-* Add ``--keys-directory`` config option to specify where files such as
-  certs and signing keys should be stored in, when using ``--generate-config``
-  or ``--generate-keys``. (PR #250)
-* Allow ``--config-path`` to specify a directory, causing synapse to use all
-  \*.yaml files in the directory as config files. (PR #249)
-* Add ``web_client_location`` config option to specify static files to be
-  hosted by synapse under ``/_matrix/client``. (PR #245)
-* Add helper utility to synapse to read and parse the config files and extract
-  the value of a given key. For example::
-
-    $ python -m synapse.config read server_name -c homeserver.yaml
-    localhost
-
-  (PR #246)
-
-
-Changes in synapse v0.10.0-rc2 (2015-08-24)
-===========================================
-
-* Fix bug where we incorrectly populated the ``event_forward_extremities``
-  table, resulting in problems joining large remote rooms (e.g.
-  ``#matrix:matrix.org``)
-* Reduce the number of times we wake up pushers by not listening for presence
-  or typing events, reducing the CPU cost of each pusher.
-
-
-Changes in synapse v0.10.0-rc1 (2015-08-21)
-===========================================
-
-Also see v0.9.4-rc1 changelog, which has been amalgamated into this release.
-
-General:
-
-* Upgrade to Twisted 15 (PR #173)
-* Add support for serving and fetching encryption keys over federation.
-  (PR #208)
-* Add support for logging in with email address (PR #234)
-* Add support for new ``m.room.canonical_alias`` event. (PR #233)
-* Change synapse to treat user IDs case insensitively during registration and
-  login. (If two users already exist with case insensitive matching user ids,
-  synapse will continue to require them to specify their user ids exactly.)
-* Error if a user tries to register with an email already in use. (PR #211)
-* Add extra and improve existing caches  (PR #212, #219, #226, #228)
-* Batch various storage request (PR #226, #228)
-* Fix bug where we didn't correctly log the entity that triggered the request
-  if the request came in via an application service (PR #230)
-* Fix bug where we needlessly regenerated the full list of rooms an AS is
-  interested in. (PR #232)
-* Add support for AS's to use v2_alpha registration API (PR #210)
-
-
-Configuration:
-
-* Add ``--generate-keys`` that will generate any missing cert and key files in
-  the configuration files. This is equivalent to running ``--generate-config``
-  on an existing configuration file. (PR #220)
-* ``--generate-config`` now no longer requires a ``--server-name`` parameter
-  when used on existing configuration files. (PR #220)
-* Add ``--print-pidfile`` flag that controls the printing of the pid to stdout
-  of the demonised process. (PR #213)
-
-Media Repository:
-
-* Fix bug where we picked a lower resolution image than requested. (PR #205)
-* Add support for specifying if a the media repository should dynamically
-  thumbnail images or not. (PR #206)
-
-Metrics:
-
-* Add statistics from the reactor to the metrics API. (PR #224, #225)
-
-Demo Homeservers:
-
-* Fix starting the demo homeservers without rate-limiting enabled. (PR #182)
-* Fix enabling registration on demo homeservers (PR #223)
-
-
-Changes in synapse v0.9.4-rc1 (2015-07-21)
-==========================================
-
-General:
-
-* Add basic implementation of receipts. (SPEC-99)
-* Add support for configuration presets in room creation API. (PR  #203)
-* Add auth event that limits the visibility of history for new users.
-  (SPEC-134)
-* Add SAML2 login/registration support. (PR  #201. Thanks Muthu Subramanian!)
-* Add client side key management APIs for end to end encryption. (PR #198)
-* Change power level semantics so that you cannot kick, ban or change power
-  levels of users that have equal or greater power level than you. (SYN-192)
-* Improve performance by bulk inserting events where possible. (PR #193)
-* Improve performance by bulk verifying signatures where possible. (PR #194)
-
-
-Configuration:
-
-* Add support for including TLS certificate chains.
-
-Media Repository:
-
-* Add Content-Disposition headers to content repository responses. (SYN-150)
-
-
-Changes in synapse v0.9.3 (2015-07-01)
-======================================
-
-No changes from v0.9.3 Release Candidate 1.
-
-Changes in synapse v0.9.3-rc1 (2015-06-23)
-==========================================
-
-General:
-
-* Fix a memory leak in the notifier. (SYN-412)
-* Improve performance of room initial sync. (SYN-418)
-* General improvements to logging.
-* Remove ``access_token`` query params from ``INFO`` level logging.
-
-Configuration:
-
-* Add support for specifying and configuring multiple listeners. (SYN-389)
-
-Application services:
-
-* Fix bug where synapse failed to send user queries to application services.
-
-Changes in synapse v0.9.2-r2 (2015-06-15)
-=========================================
-
-Fix packaging so that schema delta python files get included in the package.
-
-Changes in synapse v0.9.2 (2015-06-12)
-======================================
-
-General:
-
-* Use ultrajson for json (de)serialisation when a canonical encoding is not
-  required. Ultrajson is significantly faster than simplejson in certain
-  circumstances.
-* Use connection pools for outgoing HTTP connections.
-* Process thumbnails on separate threads.
-
-Configuration:
-
-* Add option, ``gzip_responses``, to disable HTTP response compression.
-
-Federation:
-
-* Improve resilience of backfill by ensuring we fetch any missing auth events.
-* Improve performance of backfill and joining remote rooms by removing
-  unnecessary computations. This included handling events we'd previously
-  handled as well as attempting to compute the current state for outliers.
-
-
-Changes in synapse v0.9.1 (2015-05-26)
-======================================
-
-General:
-
-* Add support for backfilling when a client paginates. This allows servers to
-  request history for a room from remote servers when a client tries to
-  paginate history the server does not have - SYN-36
-* Fix bug where you couldn't disable non-default pushrules - SYN-378
-* Fix ``register_new_user`` script - SYN-359
-* Improve performance of fetching events from the database, this improves both
-  initialSync and sending of events.
-* Improve performance of event streams, allowing synapse to handle more
-  simultaneous connected clients.
-
-Federation:
-
-* Fix bug with existing backfill implementation where it returned the wrong
-  selection of events in some circumstances.
-* Improve performance of joining remote rooms.
-
-Configuration:
-
-* Add support for changing the bind host of the metrics listener via the
-  ``metrics_bind_host`` option.
-
-
-Changes in synapse v0.9.0-r5 (2015-05-21)
-=========================================
-
-* Add more database caches to reduce amount of work done for each pusher. This
-  radically reduces CPU usage when multiple pushers are set up in the same room.
-
-Changes in synapse v0.9.0 (2015-05-07)
-======================================
-
-General:
-
-* Add support for using a PostgreSQL database instead of SQLite. See
-  `docs/postgres.rst`_ for details.
-* Add password change and reset APIs. See `Registration`_ in the spec.
-* Fix memory leak due to not releasing stale notifiers - SYN-339.
-* Fix race in caches that occasionally caused some presence updates to be
-  dropped - SYN-369.
-* Check server name has not changed on restart.
-* Add a sample systemd unit file and a logger configuration in
-  contrib/systemd. Contributed Ivan Shapovalov.
-
-Federation:
-
-* Add key distribution mechanisms for fetching public keys of unavailable
-  remote home servers. See `Retrieving Server Keys`_ in the spec.
-
-Configuration:
-
-* Add support for multiple config files.
-* Add support for dictionaries in config files.
-* Remove support for specifying config options on the command line, except
-  for:
-
-  * ``--daemonize`` - Daemonize the home server.
-  * ``--manhole`` - Turn on the twisted telnet manhole service on the given
-    port.
-  * ``--database-path`` - The path to a sqlite database to use.
-  * ``--verbose`` - The verbosity level.
-  * ``--log-file`` - File to log to.
-  * ``--log-config`` - Python logging config file.
-  * ``--enable-registration`` - Enable registration for new users.
-
-Application services:
-
-* Reliably retry sending of events from Synapse to application services, as per
-  `Application Services`_ spec.
-* Application services can no longer register via the ``/register`` API,
-  instead their configuration should be saved to a file and listed in the
-  synapse ``app_service_config_files`` config option. The AS configuration file
-  has the same format as the old ``/register`` request.
-  See `docs/application_services.rst`_ for more information.
-
-.. _`docs/postgres.rst`: docs/postgres.rst
-.. _`docs/application_services.rst`: docs/application_services.rst
-.. _`Registration`: https://github.com/matrix-org/matrix-doc/blob/master/specification/10_client_server_api.rst#registration
-.. _`Retrieving Server Keys`: https://github.com/matrix-org/matrix-doc/blob/6f2698/specification/30_server_server_api.rst#retrieving-server-keys
-.. _`Application Services`: https://github.com/matrix-org/matrix-doc/blob/0c6bd9/specification/25_application_service_api.rst#home-server---application-service-api
-
-Changes in synapse v0.8.1 (2015-03-18)
-======================================
-
-* Disable registration by default. New users can be added using the command
-  ``register_new_matrix_user`` or by enabling registration in the config.
-* Add metrics to synapse. To enable metrics use config options
-  ``enable_metrics`` and ``metrics_port``.
-* Fix bug where banning only kicked the user.
-
-Changes in synapse v0.8.0 (2015-03-06)
-======================================
-
-General:
-
-* Add support for registration fallback. This is a page hosted on the server
-  which allows a user to register for an account, regardless of what client
-  they are using (e.g. mobile devices).
-
-* Added new default push rules and made them configurable by clients:
-
-  * Suppress all notice messages.
-  * Notify when invited to a new room.
-  * Notify for messages that don't match any rule.
-  * Notify on incoming call.
-
-Federation:
-
-* Added per host server side rate-limiting of incoming federation requests.
-* Added a ``/get_missing_events/`` API to federation to reduce number of
-  ``/events/`` requests.
-
-Configuration:
-
-* Added configuration option to disable registration:
-  ``disable_registration``.
-* Added configuration option to change soft limit of number of open file
-  descriptors: ``soft_file_limit``.
-* Make ``tls_private_key_path`` optional when running with ``no_tls``.
-
-Application services:
-
-* Application services can now poll on the CS API ``/events`` for their events,
-  by providing their application service ``access_token``.
-* Added exclusive namespace support to application services API.
-
-
-Changes in synapse v0.7.1 (2015-02-19)
-======================================
-
-* Initial alpha implementation of parts of the Application Services API.
-  Including:
-
-  - AS Registration / Unregistration
-  - User Query API
-  - Room Alias Query API
-  - Push transport for receiving events.
-  - User/Alias namespace admin control
-
-* Add cache when fetching events from remote servers to stop repeatedly
-  fetching events with bad signatures.
-* Respect the per remote server retry scheme when fetching both events and
-  server keys to reduce the number of times we send requests to dead servers.
-* Inform remote servers when the local server fails to handle a received event.
-* Turn off python bytecode generation due to problems experienced when
-  upgrading from previous versions.
-
-Changes in synapse v0.7.0 (2015-02-12)
-======================================
-
-* Add initial implementation of the query auth federation API, allowing
-  servers to agree on whether an event should be allowed or rejected.
-* Persist events we have rejected from federation, fixing the bug where
-  servers would keep requesting the same events.
-* Various federation performance improvements, including:
-
-  - Add in memory caches on queries such as:
-
-     * Computing the state of a room at a point in time, used for
-       authorization on federation requests.
-     * Fetching events from the database.
-     * User's room membership, used for authorizing presence updates.
-
-  - Upgraded JSON library to improve parsing and serialisation speeds.
-
-* Add default avatars to new user accounts using pydenticon library.
-* Correctly time out federation requests.
-* Retry federation requests against different servers.
-* Add support for push and push rules.
-* Add alpha versions of proposed new CSv2 APIs, including ``/sync`` API.
-
-Changes in synapse 0.6.1 (2015-01-07)
-=====================================
-
-* Major optimizations to improve performance of initial sync and event sending
-  in large rooms (by up to 10x)
-* Media repository now includes a Content-Length header on media downloads.
-* Improve quality of thumbnails by changing resizing algorithm.
-
-Changes in synapse 0.6.0 (2014-12-16)
-=====================================
-
-* Add new API for media upload and download that supports thumbnailing.
-* Replicate media uploads over multiple homeservers so media is always served
-  to clients from their local homeserver.  This obsoletes the
-  --content-addr parameter and confusion over accessing content directly
-  from remote homeservers.
-* Implement exponential backoff when retrying federation requests when
-  sending to remote homeservers which are offline.
-* Implement typing notifications.
-* Fix bugs where we sent events with invalid signatures due to bugs where
-  we incorrectly persisted events.
-* Improve performance of database queries involving retrieving events.
-
-Changes in synapse 0.5.4a (2014-12-13)
-======================================
-
-* Fix bug while generating the error message when a file path specified in
-  the config doesn't exist.
-
-Changes in synapse 0.5.4 (2014-12-03)
-=====================================
-
-* Fix presence bug where some rooms did not display presence updates for
-  remote users.
-* Do not log SQL timing log lines when started with "-v"
-* Fix potential memory leak.
-
-Changes in synapse 0.5.3c (2014-12-02)
-======================================
-
-* Change the default value for the `content_addr` option to use the HTTP
-  listener, as by default the HTTPS listener will be using a self-signed
-  certificate.
-
-Changes in synapse 0.5.3 (2014-11-27)
-=====================================
-
-* Fix bug that caused joining a remote room to fail if a single event was not
-  signed correctly.
-* Fix bug which caused servers to continuously try and fetch events from other
-  servers.
-
-Changes in synapse 0.5.2 (2014-11-26)
-=====================================
-
-Fix major bug that caused rooms to disappear from peoples initial sync.
-
-Changes in synapse 0.5.1 (2014-11-26)
-=====================================
-See UPGRADES.rst for specific instructions on how to upgrade.
-
- * Fix bug where we served up an Event that did not match its signatures.
- * Fix regression where we no longer correctly handled the case where a
-   homeserver receives an event for a room it doesn't recognise (but is in.)
-
-Changes in synapse 0.5.0 (2014-11-19)
-=====================================
-This release includes changes to the federation protocol and client-server API
-that is not backwards compatible.
-
-This release also changes the internal database schemas and so requires servers to
-drop their current history. See UPGRADES.rst for details.
-
-Homeserver:
- * Add authentication and authorization to the federation protocol. Events are
-   now signed by their originating homeservers.
- * Implement the new authorization model for rooms.
- * Split out web client into a seperate repository: matrix-angular-sdk.
- * Change the structure of PDUs.
- * Fix bug where user could not join rooms via an alias containing 4-byte
-   UTF-8 characters.
- * Merge concept of PDUs and Events internally.
- * Improve logging by adding request ids to log lines.
- * Implement a very basic room initial sync API.
- * Implement the new invite/join federation APIs.
-
-Webclient:
- * The webclient has been moved to a seperate repository.
-
-Changes in synapse 0.4.2 (2014-10-31)
-=====================================
-
-Homeserver:
- * Fix bugs where we did not notify users of correct presence updates.
- * Fix bug where we did not handle sub second event stream timeouts.
-
-Webclient:
- * Add ability to click on messages to see JSON.
- * Add ability to redact messages.
- * Add ability to view and edit all room state JSON.
- * Handle incoming redactions.
- * Improve feedback on errors.
- * Fix bugs in mobile CSS.
- * Fix bugs with desktop notifications.
-
-Changes in synapse 0.4.1 (2014-10-17)
-=====================================
-Webclient:
- * Fix bug with display of timestamps.
-
-Changes in synpase 0.4.0 (2014-10-17)
-=====================================
-This release includes changes to the federation protocol and client-server API
-that is not backwards compatible.
-
-The Matrix specification has been moved to a separate git repository:
-http://github.com/matrix-org/matrix-doc
-
-You will also need an updated syutil and config. See UPGRADES.rst.
-
-Homeserver:
- * Sign federation transactions to assert strong identity over federation.
- * Rename timestamp keys in PDUs and events from 'ts' and 'hsob_ts' to 'origin_server_ts'.
-
-
-Changes in synapse 0.3.4 (2014-09-25)
-=====================================
-This version adds support for using a TURN server. See docs/turn-howto.rst on
-how to set one up.
-
-Homeserver:
- * Add support for redaction of messages.
- * Fix bug where inviting a user on a remote home server could take up to
-   20-30s.
- * Implement a get current room state API.
- * Add support specifying and retrieving turn server configuration.
-
-Webclient:
- * Add button to send messages to users from the home page.
- * Add support for using TURN for VoIP calls.
- * Show display name change messages.
- * Fix bug where the client didn't get the state of a newly joined room
-   until after it has been refreshed.
- * Fix bugs with tab complete.
- * Fix bug where holding down the down arrow caused chrome to chew 100% CPU.
- * Fix bug where desktop notifications occasionally used "Undefined" as the
-   display name.
- * Fix more places where we sometimes saw room IDs incorrectly.
- * Fix bug which caused lag when entering text in the text box.
-
-Changes in synapse 0.3.3 (2014-09-22)
-=====================================
-
-Homeserver:
- * Fix bug where you continued to get events for rooms you had left.
-
-Webclient:
- * Add support for video calls with basic UI.
- * Fix bug where one to one chats were named after your display name rather
-   than the other person's.
- * Fix bug which caused lag when typing in the textarea.
- * Refuse to run on browsers we know won't work.
- * Trigger pagination when joining new rooms.
- * Fix bug where we sometimes didn't display invitations in recents.
- * Automatically join room when accepting a VoIP call.
- * Disable outgoing and reject incoming calls on browsers we don't support
-   VoIP in.
- * Don't display desktop notifications for messages in the room you are
-   non-idle and speaking in.
-
-Changes in synapse 0.3.2 (2014-09-18)
-=====================================
-
-Webclient:
- * Fix bug where an empty "bing words" list in old accounts didn't send
-   notifications when it should have done.
-
-Changes in synapse 0.3.1 (2014-09-18)
-=====================================
-This is a release to hotfix v0.3.0 to fix two regressions.
-
-Webclient:
- * Fix a regression where we sometimes displayed duplicate events.
- * Fix a regression where we didn't immediately remove rooms you were
-   banned in from the recents list.
-
-Changes in synapse 0.3.0 (2014-09-18)
-=====================================
-See UPGRADE for information about changes to the client server API, including
-breaking backwards compatibility with VoIP calls and registration API.
-
-Homeserver:
- * When a user changes their displayname or avatar the server will now update
-   all their join states to reflect this.
- * The server now adds "age" key to events to indicate how old they are. This
-   is clock independent, so at no point does any server or webclient have to
-   assume their clock is in sync with everyone else.
- * Fix bug where we didn't correctly pull in missing PDUs.
- * Fix bug where prev_content key wasn't always returned.
- * Add support for password resets.
-
-Webclient:
- * Improve page content loading.
- * Join/parts now trigger desktop notifications.
- * Always show room aliases in the UI if one is present.
- * No longer show user-count in the recents side panel.
- * Add up & down arrow support to the text box for message sending to step
-   through your sent history.
- * Don't display notifications for our own messages.
- * Emotes are now formatted correctly in desktop notifications.
- * The recents list now differentiates between public & private rooms.
- * Fix bug where when switching between rooms the pagination flickered before
-   the view jumped to the bottom of the screen.
- * Add bing word support.
-
-Registration API:
- * The registration API has been overhauled to function like the login API. In
-   practice, this means registration requests must now include the following:
-   'type':'m.login.password'. See UPGRADE for more information on this.
- * The 'user_id' key has been renamed to 'user' to better match the login API.
- * There is an additional login type: 'm.login.email.identity'.
- * The command client and web client have been updated to reflect these changes.
-
-Changes in synapse 0.2.3 (2014-09-12)
-=====================================
-
-Homeserver:
- * Fix bug where we stopped sending events to remote home servers if a
-   user from that home server left, even if there were some still in the
-   room.
- * Fix bugs in the state conflict resolution where it was incorrectly
-   rejecting events.
-
-Webclient:
- * Display room names and topics.
- * Allow setting/editing of room names and topics.
- * Display information about rooms on the main page.
- * Handle ban and kick events in real time.
- * VoIP UI and reliability improvements.
- * Add glare support for VoIP.
- * Improvements to initial startup speed.
- * Don't display duplicate join events.
- * Local echo of messages.
- * Differentiate sending and sent of local echo.
- * Various minor bug fixes.
-
-Changes in synapse 0.2.2 (2014-09-06)
-=====================================
-
-Homeserver:
- * When the server returns state events it now also includes the previous
-   content.
- * Add support for inviting people when creating a new room.
- * Make the homeserver inform the room via `m.room.aliases` when a new alias
-   is added for a room.
- * Validate `m.room.power_level` events.
-
-Webclient:
- * Add support for captchas on registration.
- * Handle `m.room.aliases` events.
- * Asynchronously send messages and show a local echo.
- * Inform the UI when a message failed to send.
- * Only autoscroll on receiving a new message if the user was already at the
-   bottom of the screen.
- * Add support for ban/kick reasons.
-
-Changes in synapse 0.2.1 (2014-09-03)
-=====================================
-
-Homeserver:
- * Added support for signing up with a third party id.
- * Add synctl scripts.
- * Added rate limiting.
- * Add option to change the external address the content repo uses.
- * Presence bug fixes.
-
-Webclient:
- * Added support for signing up with a third party id.
- * Added support for banning and kicking users.
- * Added support for displaying and setting ops.
- * Added support for room names.
- * Fix bugs with room membership event display.
-
-Changes in synapse 0.2.0 (2014-09-02)
-=====================================
-This update changes many configuration options, updates the
-database schema and mandates SSL for server-server connections.
-
-Homeserver:
- * Require SSL for server-server connections.
- * Add SSL listener for client-server connections.
- * Add ability to use config files.
- * Add support for kicking/banning and power levels.
- * Allow setting of room names and topics on creation.
- * Change presence to include last seen time of the user.
- * Change url path prefix to /_matrix/...
- * Bug fixes to presence.
-
-Webclient:
- * Reskin the CSS for registration and login.
- * Various improvements to rooms CSS.
- * Support changes in client-server API.
- * Bug fixes to VOIP UI.
- * Various bug fixes to handling of changes to room member list.
-
-Changes in synapse 0.1.2 (2014-08-29)
-=====================================
-
-Webclient:
- * Add basic call state UI for VoIP calls.
-
-Changes in synapse 0.1.1 (2014-08-29)
-=====================================
-
-Homeserver:
-    * Fix bug that caused the event stream to not notify some clients about
-      changes.
-
-Changes in synapse 0.1.0 (2014-08-29)
-=====================================
-Presence has been reenabled in this release.
-
-Homeserver:
- * Update client to server API, including:
-    - Use a more consistent url scheme.
-    - Provide more useful information in the initial sync api.
- * Change the presence handling to be much more efficient.
- * Change the presence server to server API to not require explicit polling of
-   all users who share a room with a user.
- * Fix races in the event streaming logic.
-
-Webclient:
- * Update to use new client to server API.
- * Add basic VOIP support.
- * Add idle timers that change your status to away.
- * Add recent rooms column when viewing a room.
- * Various network efficiency improvements.
- * Add basic mobile browser support.
- * Add a settings page.
-
-Changes in synapse 0.0.1 (2014-08-22)
-=====================================
-Presence has been disabled in this release due to a bug that caused the
-homeserver to spam other remote homeservers.
-
-Homeserver:
- * Completely change the database schema to support generic event types.
- * Improve presence reliability.
- * Improve reliability of joining remote rooms.
- * Fix bug where room join events were duplicated.
- * Improve initial sync API to return more information to the client.
- * Stop generating fake messages for room membership events.
-
-Webclient:
- * Add tab completion of names.
- * Add ability to upload and send images.
- * Add profile pages.
- * Improve CSS layout of room.
- * Disambiguate identical display names.
- * Don't get remote users display names and avatars individually.
- * Use the new initial sync API to reduce number of round trips to the homeserver.
- * Change url scheme to use room aliases instead of room ids where known.
- * Increase longpoll timeout.
-
-Changes in synapse 0.0.0 (2014-08-13)
-=====================================
-
- * Initial alpha release

From 37af0d2a13f2d17e6de9e7d1201118cb633d7c55 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Thu, 19 Jul 2018 21:25:47 +1000
Subject: [PATCH 070/830] make pyproject point at it

---
 pyproject.toml | 29 +++++++++++++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index d1603b5d8beb..1453ddd53003 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,5 +1,30 @@
 [tool.towncrier]
     package = "synapse"
-    filename = "CHANGES.rst"
+    filename = "CHANGES.md"
     directory = "changelog.d"
-    issue_format = "`#{issue} <https://github.com/matrix-org/synapse/issues/{issue}>`_"
+    issue_format = "[\#{issue}](https://github.com/matrix-org/synapse/issues/{issue}>)"
+
+    [[tool.towncrier.type]]
+        directory = "feature"
+        name = "Features"
+        showcontent = true
+
+    [[tool.towncrier.type]]
+        directory = "bugfix"
+        name = "Bugfixes"
+        showcontent = true
+
+    [[tool.towncrier.type]]
+        directory = "doc"
+        name = "Improved Documentation"
+        showcontent = true
+
+    [[tool.towncrier.type]]
+        directory = "removal"
+        name = "Deprecations and Removals"
+        showcontent = true
+
+    [[tool.towncrier.type]]
+        directory = "misc"
+        name = "Internal Changes"
+        showcontent = true

From 7cf76c9a094fd5a7ed13394a7d667382c417c1b9 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Thu, 19 Jul 2018 21:26:30 +1000
Subject: [PATCH 071/830] changelog

---
 changelog.d/3552.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3552.misc

diff --git a/changelog.d/3552.misc b/changelog.d/3552.misc
new file mode 100644
index 000000000000..709c3282b4c3
--- /dev/null
+++ b/changelog.d/3552.misc
@@ -0,0 +1 @@
+Release notes are now in the Markdown format.

From 7814e4cc86be2f409c1efd91587f4e956b52f485 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Thu, 19 Jul 2018 21:37:57 +1000
Subject: [PATCH 072/830] fix up weird translation things

---
 CHANGES.md | 1619 ++++++++++++++++++++++++++--------------------------
 1 file changed, 813 insertions(+), 806 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index d7e9e08904f6..9d4d0c679440 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -74,7 +74,7 @@ Bugfixes
 Improved Documentation
 ----------------------
 
--   `doc/postgres.rst`: fix display of the last command block. Thanks to \@ArchangeGabriel! ([\#3340](https://github.com/matrix-org/synapse/issues/3340))
+-   `doc/postgres.rst`: fix display of the last command block. Thanks to @ArchangeGabriel! ([\#3340](https://github.com/matrix-org/synapse/issues/3340))
 
 Deprecations and Removals
 -------------------------
@@ -89,7 +89,7 @@ Misc
 Changes in synapse v0.31.2 (2018-06-14)
 =======================================
 
-SECURITY UPDATE: Prevent unauthorised users from setting state events in a room when there is no `m.room.power_levels` event in force in the room. (PR \#3397)
+SECURITY UPDATE: Prevent unauthorised users from setting state events in a room when there is no `m.room.power_levels` event in force in the room. (PR #3397)
 
 Discussion around the Matrix Spec change proposal for this change can be followed at <https://github.com/matrix-org/matrix-doc/issues/1304>.
 
@@ -102,7 +102,7 @@ We are not aware of it being actively exploited but please upgrade asap.
 
 Bug Fixes:
 
--   Fix event filtering in get\_missing\_events handler (PR \#3371)
+-   Fix event filtering in get\_missing\_events handler (PR #3371)
 
 Changes in synapse v0.31.0 (2018-06-06)
 =======================================
@@ -111,49 +111,50 @@ Most notable change from v0.30.0 is to switch to the python prometheus library t
 
 Bug Fixes:
 
--   Fix metric documentation tables (PR \#3341)
+-   Fix metric documentation tables (PR #3341)
 -   Fix LaterGauge error handling (694968f)
 -   Fix replication metrics (b7e7fd2)
 
-Changes in synapse v0.31.0-rc1 (2018-06-04) ==========================================
+Changes in synapse v0.31.0-rc1 (2018-06-04)
+===========================================
 
 Features:
 
--   Switch to the Python Prometheus library (PR \#3256, \#3274)
--   Let users leave the server notice room after joining (PR \#3287)
+-   Switch to the Python Prometheus library (PR #3256, #3274)
+-   Let users leave the server notice room after joining (PR #3287)
 
 Changes:
 
--   daily user type phone home stats (PR \#3264)
--   Use iter\* methods for \_filter\_events\_for\_server (PR \#3267)
--   Docs on consent bits (PR \#3268)
--   Remove users from user directory on deactivate (PR \#3277)
--   Avoid sending consent notice to guest users (PR \#3288)
--   disable CPUMetrics if no /proc/self/stat (PR \#3299)
--   Consistently use six\'s iteritems and wrap lazy keys/values in list() if they\'re not meant to be lazy (PR \#3307)
--   Add private IPv6 addresses to example config for url preview blacklist (PR \#3317) Thanks to \@thegcat!
--   Reduce stuck read-receipts: ignore depth when updating (PR \#3318)
--   Put python\'s logs into Trial when running unit tests (PR \#3319)
+-   daily user type phone home stats (PR #3264)
+-   Use iter\* methods for \_filter\_events\_for\_server (PR #3267)
+-   Docs on consent bits (PR #3268)
+-   Remove users from user directory on deactivate (PR #3277)
+-   Avoid sending consent notice to guest users (PR #3288)
+-   disable CPUMetrics if no /proc/self/stat (PR #3299)
+-   Consistently use six\'s iteritems and wrap lazy keys/values in list() if they\'re not meant to be lazy (PR #3307)
+-   Add private IPv6 addresses to example config for url preview blacklist (PR #3317) Thanks to @thegcat!
+-   Reduce stuck read-receipts: ignore depth when updating (PR #3318)
+-   Put python\'s logs into Trial when running unit tests (PR #3319)
 
 Changes, python 3 migration:
 
--   Replace some more comparisons with six (PR \#3243) Thanks to \@NotAFile!
--   replace some iteritems with six (PR \#3244) Thanks to \@NotAFile!
--   Add batch\_iter to utils (PR \#3245) Thanks to \@NotAFile!
--   use repr, not str (PR \#3246) Thanks to \@NotAFile!
--   Misc Python3 fixes (PR \#3247) Thanks to \@NotAFile!
--   Py3 storage/\_base.py (PR \#3278) Thanks to \@NotAFile!
--   more six iteritems (PR \#3279) Thanks to \@NotAFile!
--   More Misc. py3 fixes (PR \#3280) Thanks to \@NotAFile!
--   remaining isintance fixes (PR \#3281) Thanks to \@NotAFile!
--   py3-ize state.py (PR \#3283) Thanks to \@NotAFile!
--   extend tox testing for py3 to avoid regressions (PR \#3302) Thanks to \@krombel!
--   use memoryview in py3 (PR \#3303) Thanks to \@NotAFile!
+-   Replace some more comparisons with six (PR #3243) Thanks to @NotAFile!
+-   replace some iteritems with six (PR #3244) Thanks to @NotAFile!
+-   Add batch\_iter to utils (PR #3245) Thanks to @NotAFile!
+-   use repr, not str (PR #3246) Thanks to @NotAFile!
+-   Misc Python3 fixes (PR #3247) Thanks to @NotAFile!
+-   Py3 storage/\_base.py (PR #3278) Thanks to @NotAFile!
+-   more six iteritems (PR #3279) Thanks to @NotAFile!
+-   More Misc. py3 fixes (PR #3280) Thanks to @NotAFile!
+-   remaining isintance fixes (PR #3281) Thanks to @NotAFile!
+-   py3-ize state.py (PR #3283) Thanks to @NotAFile!
+-   extend tox testing for py3 to avoid regressions (PR #3302) Thanks to @krombel!
+-   use memoryview in py3 (PR #3303) Thanks to @NotAFile!
 
 Bugs:
 
--   Fix federation backfill bugs (PR \#3261)
--   federation: fix LaterGauge usage (PR \#3328) Thanks to \@intelfx!
+-   Fix federation backfill bugs (PR #3261)
+-   federation: fix LaterGauge usage (PR #3328) Thanks to @intelfx!
 
 Changes in synapse v0.30.0 (2018-05-24)
 =======================================
@@ -166,49 +167,50 @@ This feature is specific to Synapse, but uses standard Matrix communication mech
 
 Further Server Notices/Consent Tracking Support:
 
--   Allow overriding the server\_notices user\'s avatar (PR \#3273)
--   Use the localpart in the consent uri (PR \#3272)
--   Support for putting %(consent\_uri)s in messages (PR \#3271)
--   Block attempts to send server notices to remote users (PR \#3270)
--   Docs on consent bits (PR \#3268)
+-   Allow overriding the server\_notices user\'s avatar (PR #3273)
+-   Use the localpart in the consent uri (PR #3272)
+-   Support for putting %(consent\_uri)s in messages (PR #3271)
+-   Block attempts to send server notices to remote users (PR #3270)
+-   Docs on consent bits (PR #3268)
 
-Changes in synapse v0.30.0-rc1 (2018-05-23) ==========================================
+Changes in synapse v0.30.0-rc1 (2018-05-23)
+===========================================
 
 Server Notices/Consent Tracking Support:
 
--   ConsentResource to gather policy consent from users (PR \#3213)
--   Move RoomCreationHandler out of synapse.handlers.Handlers (PR \#3225)
--   Infrastructure for a server notices room (PR \#3232)
--   Send users a server notice about consent (PR \#3236)
--   Reject attempts to send event before privacy consent is given (PR \#3257)
--   Add a \'has\_consented\' template var to consent forms (PR \#3262)
--   Fix dependency on jinja2 (PR \#3263)
+-   ConsentResource to gather policy consent from users (PR #3213)
+-   Move RoomCreationHandler out of synapse.handlers.Handlers (PR #3225)
+-   Infrastructure for a server notices room (PR #3232)
+-   Send users a server notice about consent (PR #3236)
+-   Reject attempts to send event before privacy consent is given (PR #3257)
+-   Add a \'has\_consented\' template var to consent forms (PR #3262)
+-   Fix dependency on jinja2 (PR #3263)
 
 Features:
 
--   Cohort analytics (PR \#3163, \#3241, \#3251)
--   Add lxml to docker image for web previews (PR \#3239) Thanks to \@ptman!
--   Add in flight request metrics (PR \#3252)
+-   Cohort analytics (PR #3163, #3241, #3251)
+-   Add lxml to docker image for web previews (PR #3239) Thanks to @ptman!
+-   Add in flight request metrics (PR #3252)
 
 Changes:
 
--   Remove unused update\_external\_syncs (PR \#3233)
--   Use stream rather depth ordering for push actions (PR \#3212)
--   Make purge\_history operate on tokens (PR \#3221)
--   Don\'t support limitless pagination (PR \#3265)
+-   Remove unused update\_external\_syncs (PR #3233)
+-   Use stream rather depth ordering for push actions (PR #3212)
+-   Make purge\_history operate on tokens (PR #3221)
+-   Don\'t support limitless pagination (PR #3265)
 
 Bug Fixes:
 
--   Fix logcontext resource usage tracking (PR \#3258)
--   Fix error in handling receipts (PR \#3235)
--   Stop the transaction cache caching failures (PR \#3255)
+-   Fix logcontext resource usage tracking (PR #3258)
+-   Fix error in handling receipts (PR #3235)
+-   Stop the transaction cache caching failures (PR #3255)
 
 Changes in synapse v0.29.1 (2018-05-17)
 =======================================
 
 Changes:
 
--   Update docker documentation (PR \#3222)
+-   Update docker documentation (PR #3222)
 
 Changes in synapse v0.29.0 (2018-05-16)
 =======================================
@@ -218,76 +220,76 @@ Not changes since v0.29.0-rc1
 Changes in synapse v0.29.0-rc1 (2018-05-14)
 ===========================================
 
-Notable changes, a docker file for running Synapse (Thanks to \@kaiyou!) and a closed spec bug in the Client Server API. Additionally further prep for Python 3 migration.
+Notable changes, a docker file for running Synapse (Thanks to @kaiyou!) and a closed spec bug in the Client Server API. Additionally further prep for Python 3 migration.
 
 Potentially breaking change:
 
--   Make Client-Server API return 401 for invalid token (PR \#3161).
+-   Make Client-Server API return 401 for invalid token (PR #3161).
 
     This changes the Client-server spec to return a 401 error code instead of 403 when the access token is unrecognised. This is the behaviour required by the specification, but some clients may be relying on the old, incorrect behaviour.
 
-    Thanks to \@NotAFile for fixing this.
+    Thanks to @NotAFile for fixing this.
 
 Features:
 
--   Add a Dockerfile for synapse (PR \#2846) Thanks to \@kaiyou!
+-   Add a Dockerfile for synapse (PR #2846) Thanks to @kaiyou!
 
 Changes - General:
 
--   nuke-room-from-db.sh: added postgresql option and help (PR \#2337) Thanks to \@rubo77!
--   Part user from rooms on account deactivate (PR \#3201)
--   Make \'unexpected logging context\' into warnings (PR \#3007)
--   Set Server header in SynapseRequest (PR \#3208)
--   remove duplicates from groups tables (PR \#3129)
--   Improve exception handling for background processes (PR \#3138)
--   Add missing consumeErrors to improve exception handling (PR \#3139)
--   reraise exceptions more carefully (PR \#3142)
--   Remove redundant call to preserve\_fn (PR \#3143)
--   Trap exceptions thrown within run\_in\_background (PR \#3144)
+-   nuke-room-from-db.sh: added postgresql option and help (PR #2337) Thanks to @rubo77!
+-   Part user from rooms on account deactivate (PR #3201)
+-   Make \'unexpected logging context\' into warnings (PR #3007)
+-   Set Server header in SynapseRequest (PR #3208)
+-   remove duplicates from groups tables (PR #3129)
+-   Improve exception handling for background processes (PR #3138)
+-   Add missing consumeErrors to improve exception handling (PR #3139)
+-   reraise exceptions more carefully (PR #3142)
+-   Remove redundant call to preserve\_fn (PR #3143)
+-   Trap exceptions thrown within run\_in\_background (PR #3144)
 
 Changes - Refactors:
 
--   Refactor /context to reuse pagination storage functions (PR \#3193)
--   Refactor recent events func to use pagination func (PR \#3195)
--   Refactor pagination DB API to return concrete type (PR \#3196)
--   Refactor get\_recent\_events\_for\_room return type (PR \#3198)
--   Refactor sync APIs to reuse pagination API (PR \#3199)
--   Remove unused code path from member change DB func (PR \#3200)
--   Refactor request handling wrappers (PR \#3203)
--   transaction\_id, destination defined twice (PR \#3209) Thanks to \@damir-manapov!
--   Refactor event storage to prepare for changes in state calculations (PR \#3141)
--   Set Server header in SynapseRequest (PR \#3208)
--   Use deferred.addTimeout instead of time\_bound\_deferred (PR \#3127, \#3178)
--   Use run\_in\_background in preference to preserve\_fn (PR \#3140)
+-   Refactor /context to reuse pagination storage functions (PR #3193)
+-   Refactor recent events func to use pagination func (PR #3195)
+-   Refactor pagination DB API to return concrete type (PR #3196)
+-   Refactor get\_recent\_events\_for\_room return type (PR #3198)
+-   Refactor sync APIs to reuse pagination API (PR #3199)
+-   Remove unused code path from member change DB func (PR #3200)
+-   Refactor request handling wrappers (PR #3203)
+-   transaction\_id, destination defined twice (PR #3209) Thanks to @damir-manapov!
+-   Refactor event storage to prepare for changes in state calculations (PR #3141)
+-   Set Server header in SynapseRequest (PR #3208)
+-   Use deferred.addTimeout instead of time\_bound\_deferred (PR #3127, #3178)
+-   Use run\_in\_background in preference to preserve\_fn (PR #3140)
 
 Changes - Python 3 migration:
 
--   Construct HMAC as bytes on py3 (PR \#3156) Thanks to \@NotAFile!
--   run config tests on py3 (PR \#3159) Thanks to \@NotAFile!
--   Open certificate files as bytes (PR \#3084) Thanks to \@NotAFile!
--   Open config file in non-bytes mode (PR \#3085) Thanks to \@NotAFile!
--   Make event properties raise AttributeError instead (PR \#3102) Thanks to \@NotAFile!
--   Use six.moves.urlparse (PR \#3108) Thanks to \@NotAFile!
--   Add py3 tests to tox with folders that work (PR \#3145) Thanks to \@NotAFile!
--   Don\'t yield in list comprehensions (PR \#3150) Thanks to \@NotAFile!
--   Move more xrange to six (PR \#3151) Thanks to \@NotAFile!
--   make imports local (PR \#3152) Thanks to \@NotAFile!
--   move httplib import to six (PR \#3153) Thanks to \@NotAFile!
--   Replace stringIO imports with six (PR \#3154, \#3168) Thanks to \@NotAFile!
--   more bytes strings (PR \#3155) Thanks to \@NotAFile!
+-   Construct HMAC as bytes on py3 (PR #3156) Thanks to @NotAFile!
+-   run config tests on py3 (PR #3159) Thanks to @NotAFile!
+-   Open certificate files as bytes (PR #3084) Thanks to @NotAFile!
+-   Open config file in non-bytes mode (PR #3085) Thanks to @NotAFile!
+-   Make event properties raise AttributeError instead (PR #3102) Thanks to @NotAFile!
+-   Use six.moves.urlparse (PR #3108) Thanks to @NotAFile!
+-   Add py3 tests to tox with folders that work (PR #3145) Thanks to @NotAFile!
+-   Don\'t yield in list comprehensions (PR #3150) Thanks to @NotAFile!
+-   Move more xrange to six (PR #3151) Thanks to @NotAFile!
+-   make imports local (PR #3152) Thanks to @NotAFile!
+-   move httplib import to six (PR #3153) Thanks to @NotAFile!
+-   Replace stringIO imports with six (PR #3154, #3168) Thanks to @NotAFile!
+-   more bytes strings (PR #3155) Thanks to @NotAFile!
 
 Bug Fixes:
 
--   synapse fails to start under Twisted \>= 18.4 (PR \#3157)
--   Fix a class of logcontext leaks (PR \#3170)
--   Fix a couple of logcontext leaks in unit tests (PR \#3172)
--   Fix logcontext leak in media repo (PR \#3174)
--   Escape label values in prometheus metrics (PR \#3175, \#3186)
--   Fix \'Unhandled Error\' logs with Twisted 18.4 (PR \#3182) Thanks to \@Half-Shot!
--   Fix logcontext leaks in rate limiter (PR \#3183)
--   notifications: Convert next\_token to string according to the spec (PR \#3190) Thanks to \@mujx!
--   nuke-room-from-db.sh: fix deletion from search table (PR \#3194) Thanks to \@rubo77!
--   add guard for None on purge\_history api (PR \#3160) Thanks to \@krombel!
+-   synapse fails to start under Twisted \>= 18.4 (PR #3157)
+-   Fix a class of logcontext leaks (PR #3170)
+-   Fix a couple of logcontext leaks in unit tests (PR #3172)
+-   Fix logcontext leak in media repo (PR #3174)
+-   Escape label values in prometheus metrics (PR #3175, #3186)
+-   Fix \'Unhandled Error\' logs with Twisted 18.4 (PR #3182) Thanks to @Half-Shot!
+-   Fix logcontext leaks in rate limiter (PR #3183)
+-   notifications: Convert next\_token to string according to the spec (PR #3190) Thanks to @mujx!
+-   nuke-room-from-db.sh: fix deletion from search table (PR #3194) Thanks to @rubo77!
+-   add guard for None on purge\_history api (PR #3160) Thanks to @krombel!
 
 Changes in synapse v0.28.1 (2018-05-01)
 =======================================
@@ -307,8 +309,8 @@ Changes in synapse v0.28.0 (2018-04-26)
 
 Bug Fixes:
 
--   Fix quarantine media admin API and search reindex (PR \#3130)
--   Fix media admin APIs (PR \#3134)
+-   Fix quarantine media admin API and search reindex (PR #3130)
+-   Fix media admin APIs (PR #3134)
 
 Changes in synapse v0.28.0-rc1 (2018-04-24)
 ===========================================
@@ -319,53 +321,57 @@ Minor performance improvement to federation sending and bug fixes.
 
 Features:
 
--   Add metrics for event processing lag (PR \#3090)
--   Add metrics for ResponseCache (PR \#3092)
+-   Add metrics for event processing lag (PR #3090)
+-   Add metrics for ResponseCache (PR #3092)
 
 Changes:
 
--   Synapse on PyPy (PR \#2760) Thanks to \@Valodim!
--   move handling of auto\_join\_rooms to RegisterHandler (PR \#2996) Thanks to \@krombel!
--   Improve handling of SRV records for federation connections (PR \#3016) Thanks to \@silkeh!
--   Document the behaviour of ResponseCache (PR \#3059)
--   Preparation for py3 (PR \#3061, \#3073, \#3074, \#3075, \#3103, \#3104, \#3106, \#3107, \#3109, \#3110) Thanks to \@NotAFile!
--   update prometheus dashboard to use new metric names (PR \#3069) Thanks to \@krombel!
--   use python3-compatible prints (PR \#3074) Thanks to \@NotAFile!
--   Send federation events concurrently (PR \#3078)
--   Limit concurrent event sends for a room (PR \#3079)
--   Improve R30 stat definition (PR \#3086)
--   Send events to ASes concurrently (PR \#3088)
--   Refactor ResponseCache usage (PR \#3093)
--   Clarify that SRV may not point to a CNAME (PR \#3100) Thanks to \@silkeh!
--   Use str(e) instead of e.message (PR \#3103) Thanks to \@NotAFile!
--   Use six.itervalues in some places (PR \#3106) Thanks to \@NotAFile!
--   Refactor store.have\_events (PR \#3117)
+-   Synapse on PyPy (PR #2760) Thanks to @Valodim!
+-   move handling of auto\_join\_rooms to RegisterHandler (PR #2996) Thanks to @krombel!
+-   Improve handling of SRV records for federation connections (PR #3016) Thanks to @silkeh!
+-   Document the behaviour of ResponseCache (PR #3059)
+-   Preparation for py3 (PR #3061, #3073, #3074, #3075, #3103, #3104, #3106, #3107, #3109, #3110) Thanks to @NotAFile!
+-   update prometheus dashboard to use new metric names (PR #3069) Thanks to @krombel!
+-   use python3-compatible prints (PR #3074) Thanks to @NotAFile!
+-   Send federation events concurrently (PR #3078)
+-   Limit concurrent event sends for a room (PR #3079)
+-   Improve R30 stat definition (PR #3086)
+-   Send events to ASes concurrently (PR #3088)
+-   Refactor ResponseCache usage (PR #3093)
+-   Clarify that SRV may not point to a CNAME (PR #3100) Thanks to @silkeh!
+-   Use str(e) instead of e.message (PR #3103) Thanks to @NotAFile!
+-   Use six.itervalues in some places (PR #3106) Thanks to @NotAFile!
+-   Refactor store.have\_events (PR #3117)
 
 Bug Fixes:
 
--   Return 401 for invalid access\_token on logout (PR \#2938) Thanks to \@dklug!
--   Return a 404 rather than a 500 on rejoining empty rooms (PR \#3080)
--   fix federation\_domain\_whitelist (PR \#3099)
--   Avoid creating events with huge numbers of prev\_events (PR \#3113)
--   Reject events which have lots of prev\_events (PR \#3118)
+-   Return 401 for invalid access\_token on logout (PR #2938) Thanks to @dklug!
+-   Return a 404 rather than a 500 on rejoining empty rooms (PR #3080)
+-   fix federation\_domain\_whitelist (PR #3099)
+-   Avoid creating events with huge numbers of prev\_events (PR #3113)
+-   Reject events which have lots of prev\_events (PR #3118)
 
-Changes in synapse v0.27.4 (2018-04-13) ======================================
+Changes in synapse v0.27.4 (2018-04-13)
+=======================================
 
 Changes:
 
 -   Update canonicaljson dependency (\#3095)
 
-Changes in synapse v0.27.3 (2018-04-11) ======================================
+Changes in synapse v0.27.3 (2018-04-11)
+======================================
 
 Bug fixes:
 
 -   URL quote path segments over federation (\#3082)
 
-Changes in synapse v0.27.3-rc2 (2018-04-09) ==========================================
+Changes in synapse v0.27.3-rc2 (2018-04-09)
+===========================================
 
 v0.27.3-rc1 used a stale version of the develop branch so the changelog overstates the functionality. v0.27.3-rc2 is up to date, rc1 should be ignored.
 
-Changes in synapse v0.27.3-rc1 (2018-04-09) =======================================
+Changes in synapse v0.27.3-rc1 (2018-04-09)
+===========================================
 
 Notable changes include API support for joinability of groups. Also new metrics and phone home stats. Phone home stats include better visibility of system usage so we can tweak synpase to work better for all users rather than our own experience with matrix.org. Also, recording \'r30\' stat which is the measure we use to track overal growth of the Matrix ecosystem. It is defined as:-
 
@@ -376,43 +382,43 @@ Counts the number of native 30 day retained users, defined as:- \* Users who hav
 
 Features:
 
--   Add joinability for groups (PR \#3045)
--   Implement group join API (PR \#3046)
--   Add counter metrics for calculating state delta (PR \#3033)
--   R30 stats (PR \#3041)
--   Measure time it takes to calculate state group ID (PR \#3043)
--   Add basic performance statistics to phone home (PR \#3044)
--   Add response size metrics (PR \#3071)
--   phone home cache size configurations (PR \#3063)
+-   Add joinability for groups (PR #3045)
+-   Implement group join API (PR #3046)
+-   Add counter metrics for calculating state delta (PR #3033)
+-   R30 stats (PR #3041)
+-   Measure time it takes to calculate state group ID (PR #3043)
+-   Add basic performance statistics to phone home (PR #3044)
+-   Add response size metrics (PR #3071)
+-   phone home cache size configurations (PR #3063)
 
 Changes:
 
--   Add a blurb explaining the main synapse worker (PR \#2886) Thanks to \@turt2live!
--   Replace old style error catching with \'as\' keyword (PR \#3000) Thanks to \@NotAFile!
--   Use .iter\* to avoid copies in StateHandler (PR \#3006)
--   Linearize calls to \_generate\_user\_id (PR \#3029)
--   Remove last usage of ujson (PR \#3030)
--   Use simplejson throughout (PR \#3048)
--   Use static JSONEncoders (PR \#3049)
--   Remove uses of events.content (PR \#3060)
--   Improve database cache performance (PR \#3068)
+-   Add a blurb explaining the main synapse worker (PR #2886) Thanks to @turt2live!
+-   Replace old style error catching with \'as\' keyword (PR #3000) Thanks to @NotAFile!
+-   Use .iter\* to avoid copies in StateHandler (PR #3006)
+-   Linearize calls to \_generate\_user\_id (PR #3029)
+-   Remove last usage of ujson (PR #3030)
+-   Use simplejson throughout (PR #3048)
+-   Use static JSONEncoders (PR #3049)
+-   Remove uses of events.content (PR #3060)
+-   Improve database cache performance (PR #3068)
 
 Bug fixes:
 
--   Add room\_id to the response of rooms/{roomId}/join (PR \#2986) Thanks to \@jplatte!
--   Fix replication after switch to simplejson (PR \#3015)
--   404 correctly on missing paths via NoResource (PR \#3022)
--   Fix error when claiming e2e keys from offline servers (PR \#3034)
--   fix tests/storage/test\_user\_directory.py (PR \#3042)
--   use PUT instead of POST for federating groups/m.join\_policy (PR \#3070) Thanks to \@krombel!
--   postgres port script: fix state\_groups\_pkey error (PR \#3072)
+-   Add room\_id to the response of rooms/{roomId}/join (PR #2986) Thanks to @jplatte!
+-   Fix replication after switch to simplejson (PR #3015)
+-   404 correctly on missing paths via NoResource (PR #3022)
+-   Fix error when claiming e2e keys from offline servers (PR #3034)
+-   fix tests/storage/test\_user\_directory.py (PR #3042)
+-   use PUT instead of POST for federating groups/m.join\_policy (PR #3070) Thanks to @krombel!
+-   postgres port script: fix state\_groups\_pkey error (PR #3072)
 
 Changes in synapse v0.27.2 (2018-03-26)
 =======================================
 
 Bug fixes:
 
--   Fix bug which broke TCP replication between workers (PR \#3015)
+-   Fix bug which broke TCP replication between workers (PR #3015)
 
 Changes in synapse v0.27.1 (2018-03-26)
 =======================================
@@ -431,14 +437,14 @@ Pulls in v0.26.1
 
 Bug fixes:
 
--   Fix bug introduced in v0.27.0-rc1 that causes much increased memory usage in state cache (PR \#3005)
+-   Fix bug introduced in v0.27.0-rc1 that causes much increased memory usage in state cache (PR #3005)
 
 Changes in synapse v0.26.1 (2018-03-15)
 =======================================
 
 Bug fixes:
 
--   Fix bug where an invalid event caused server to stop functioning correctly, due to parsing and serializing bugs in ujson library (PR \#3008)
+-   Fix bug where an invalid event caused server to stop functioning correctly, due to parsing and serializing bugs in ujson library (PR #3008)
 
 Changes in synapse v0.27.0-rc1 (2018-03-14)
 ===========================================
@@ -449,41 +455,41 @@ This release also begins the process of renaming a number of the metrics reporte
 
 Features:
 
--   Add ability for ASes to override message send time (PR \#2754)
--   Add support for custom storage providers for media repository (PR \#2867, \#2777, \#2783, \#2789, \#2791, \#2804, \#2812, \#2814, \#2857, \#2868, \#2767)
--   Add purge API features, see [docs/admin\_api/purge\_history\_api.rst](docs/admin_api/purge_history_api.rst) for full details (PR \#2858, \#2867, \#2882, \#2946, \#2962, \#2943)
--   Add support for whitelisting 3PIDs that users can register. (PR \#2813)
--   Add `/room/{id}/event/{id}` API (PR \#2766)
--   Add an admin API to get all the media in a room (PR \#2818) Thanks to \@turt2live!
--   Add `federation_domain_whitelist` option (PR \#2820, \#2821)
+-   Add ability for ASes to override message send time (PR #2754)
+-   Add support for custom storage providers for media repository (PR #2867, #2777, #2783, #2789, #2791, #2804, #2812, #2814, #2857, #2868, #2767)
+-   Add purge API features, see [docs/admin\_api/purge\_history\_api.rst](docs/admin_api/purge_history_api.rst) for full details (PR #2858, #2867, #2882, #2946, #2962, #2943)
+-   Add support for whitelisting 3PIDs that users can register. (PR #2813)
+-   Add `/room/{id}/event/{id}` API (PR #2766)
+-   Add an admin API to get all the media in a room (PR #2818) Thanks to @turt2live!
+-   Add `federation_domain_whitelist` option (PR #2820, #2821)
 
 Changes:
 
--   Continue to factor out processing from main process and into worker processes. See updated [docs/workers.rst](docs/workers.rst) (PR \#2892 - \#2904, \#2913, \#2920 - \#2926, \#2947, \#2847, \#2854, \#2872, \#2873, \#2874, \#2928, \#2929, \#2934, \#2856, \#2976 - \#2984, \#2987 - \#2989, \#2991 - \#2993, \#2995, \#2784)
--   Ensure state cache is used when persisting events (PR \#2864, \#2871, \#2802, \#2835, \#2836, \#2841, \#2842, \#2849)
--   Change the default config to bind on both IPv4 and IPv6 on all platforms (PR \#2435) Thanks to \@silkeh!
--   No longer require a specific version of saml2 (PR \#2695) Thanks to \@okurz!
--   Remove `verbosity`/`log_file` from generated config (PR \#2755)
--   Add and improve metrics and logging (PR \#2770, \#2778, \#2785, \#2786, \#2787, \#2793, \#2794, \#2795, \#2809, \#2810, \#2833, \#2834, \#2844, \#2965, \#2927, \#2975, \#2790, \#2796, \#2838)
--   When using synctl with workers, don\'t start the main synapse automatically (PR \#2774)
--   Minor performance improvements (PR \#2773, \#2792)
--   Use a connection pool for non-federation outbound connections (PR \#2817)
--   Make it possible to run unit tests against postgres (PR \#2829)
--   Update pynacl dependency to 1.2.1 or higher (PR \#2888) Thanks to \@bachp!
--   Remove ability for AS users to call /events and /sync (PR \#2948)
--   Use bcrypt.checkpw (PR \#2949) Thanks to \@krombel!
+-   Continue to factor out processing from main process and into worker processes. See updated [docs/workers.rst](docs/workers.rst) (PR #2892 - \#2904, #2913, #2920 - \#2926, #2947, #2847, #2854, #2872, #2873, #2874, #2928, #2929, #2934, #2856, #2976 - \#2984, #2987 - \#2989, #2991 - \#2993, #2995, #2784)
+-   Ensure state cache is used when persisting events (PR #2864, #2871, #2802, #2835, #2836, #2841, #2842, #2849)
+-   Change the default config to bind on both IPv4 and IPv6 on all platforms (PR #2435) Thanks to @silkeh!
+-   No longer require a specific version of saml2 (PR #2695) Thanks to @okurz!
+-   Remove `verbosity`/`log_file` from generated config (PR #2755)
+-   Add and improve metrics and logging (PR #2770, #2778, #2785, #2786, #2787, #2793, #2794, #2795, #2809, #2810, #2833, #2834, #2844, #2965, #2927, #2975, #2790, #2796, #2838)
+-   When using synctl with workers, don\'t start the main synapse automatically (PR #2774)
+-   Minor performance improvements (PR #2773, #2792)
+-   Use a connection pool for non-federation outbound connections (PR #2817)
+-   Make it possible to run unit tests against postgres (PR #2829)
+-   Update pynacl dependency to 1.2.1 or higher (PR #2888) Thanks to @bachp!
+-   Remove ability for AS users to call /events and /sync (PR #2948)
+-   Use bcrypt.checkpw (PR #2949) Thanks to @krombel!
 
 Bug fixes:
 
--   Fix broken `ldap_config` config option (PR \#2683) Thanks to \@seckrv!
--   Fix error message when user is not allowed to unban (PR \#2761) Thanks to \@turt2live!
--   Fix publicised groups GET API (singular) over federation (PR \#2772)
--   Fix user directory when using `user_directory_search_all_users` config option (PR \#2803, \#2831)
--   Fix error on `/publicRooms` when no rooms exist (PR \#2827)
--   Fix bug in quarantine\_media (PR \#2837)
--   Fix url\_previews when no Content-Type is returned from URL (PR \#2845)
--   Fix rare race in sync API when joining room (PR \#2944)
--   Fix slow event search, switch back from GIST to GIN indexes (PR \#2769, \#2848)
+-   Fix broken `ldap_config` config option (PR #2683) Thanks to @seckrv!
+-   Fix error message when user is not allowed to unban (PR #2761) Thanks to @turt2live!
+-   Fix publicised groups GET API (singular) over federation (PR #2772)
+-   Fix user directory when using `user_directory_search_all_users` config option (PR #2803, #2831)
+-   Fix error on `/publicRooms` when no rooms exist (PR #2827)
+-   Fix bug in quarantine\_media (PR #2837)
+-   Fix url\_previews when no Content-Type is returned from URL (PR #2845)
+-   Fix rare race in sync API when joining room (PR #2944)
+-   Fix slow event search, switch back from GIST to GIN indexes (PR #2769, #2848)
 
 Changes in synapse v0.26.0 (2018-01-05)
 =======================================
@@ -495,93 +501,93 @@ Changes in synapse v0.26.0-rc1 (2017-12-13)
 
 Features:
 
--   Add ability for ASes to publicise groups for their users (PR \#2686)
--   Add all local users to the user\_directory and optionally search them (PR \#2723)
--   Add support for custom login types for validating users (PR \#2729)
+-   Add ability for ASes to publicise groups for their users (PR #2686)
+-   Add all local users to the user\_directory and optionally search them (PR #2723)
+-   Add support for custom login types for validating users (PR #2729)
 
 Changes:
 
--   Update example Prometheus config to new format (PR \#2648) Thanks to \@krombel!
--   Rename redact\_content option to include\_content in Push API (PR \#2650)
--   Declare support for r0.3.0 (PR \#2677)
--   Improve upserts (PR \#2684, \#2688, \#2689, \#2713)
--   Improve documentation of workers (PR \#2700)
--   Improve tracebacks on exceptions (PR \#2705)
--   Allow guest access to group APIs for reading (PR \#2715)
--   Support for posting content in federation\_client script (PR \#2716)
--   Delete devices and pushers on logouts etc (PR \#2722)
+-   Update example Prometheus config to new format (PR #2648) Thanks to @krombel!
+-   Rename redact\_content option to include\_content in Push API (PR #2650)
+-   Declare support for r0.3.0 (PR #2677)
+-   Improve upserts (PR #2684, #2688, #2689, #2713)
+-   Improve documentation of workers (PR #2700)
+-   Improve tracebacks on exceptions (PR #2705)
+-   Allow guest access to group APIs for reading (PR #2715)
+-   Support for posting content in federation\_client script (PR #2716)
+-   Delete devices and pushers on logouts etc (PR #2722)
 
 Bug fixes:
 
--   Fix database port script (PR \#2673)
--   Fix internal server error on login with ldap\_auth\_provider (PR \#2678) Thanks to \@jkolo!
--   Fix error on sqlite 3.7 (PR \#2697)
--   Fix OPTIONS on preview\_url (PR \#2707)
--   Fix error handling on dns lookup (PR \#2711)
--   Fix wrong avatars when inviting multiple users when creating room (PR \#2717)
--   Fix 500 when joining matrix-dev (PR \#2719)
+-   Fix database port script (PR #2673)
+-   Fix internal server error on login with ldap\_auth\_provider (PR #2678) Thanks to @jkolo!
+-   Fix error on sqlite 3.7 (PR #2697)
+-   Fix OPTIONS on preview\_url (PR #2707)
+-   Fix error handling on dns lookup (PR #2711)
+-   Fix wrong avatars when inviting multiple users when creating room (PR #2717)
+-   Fix 500 when joining matrix-dev (PR #2719)
 
 Changes in synapse v0.25.1 (2017-11-17)
 =======================================
 
 Bug fixes:
 
--   Fix login with LDAP and other password provider modules (PR \#2678). Thanks to \@jkolo!
+-   Fix login with LDAP and other password provider modules (PR #2678). Thanks to @jkolo!
 
 Changes in synapse v0.25.0 (2017-11-15)
 =======================================
 
 Bug fixes:
 
--   Fix port script (PR \#2673)
+-   Fix port script (PR #2673)
 
 Changes in synapse v0.25.0-rc1 (2017-11-14)
 ===========================================
 
 Features:
 
--   Add is\_public to groups table to allow for private groups (PR \#2582)
--   Add a route for determining who you are (PR \#2668) Thanks to \@turt2live!
--   Add more features to the password providers (PR \#2608, \#2610, \#2620, \#2622, \#2623, \#2624, \#2626, \#2628, \#2629)
--   Add a hook for custom rest endpoints (PR \#2627)
--   Add API to update group room visibility (PR \#2651)
+-   Add is\_public to groups table to allow for private groups (PR #2582)
+-   Add a route for determining who you are (PR #2668) Thanks to @turt2live!
+-   Add more features to the password providers (PR #2608, #2610, #2620, #2622, #2623, #2624, #2626, #2628, #2629)
+-   Add a hook for custom rest endpoints (PR #2627)
+-   Add API to update group room visibility (PR #2651)
 
 Changes:
 
--   Ignore \<noscript\> tags when generating URL preview descriptions (PR \#2576) Thanks to \@maximevaillancourt!
--   Register some /unstable endpoints in /r0 as well (PR \#2579) Thanks to \@krombel!
--   Support /keys/upload on /r0 as well as /unstable (PR \#2585)
--   Front-end proxy: pass through auth header (PR \#2586)
--   Allow ASes to deactivate their own users (PR \#2589)
--   Remove refresh tokens (PR \#2613)
--   Automatically set default displayname on register (PR \#2617)
--   Log login requests (PR \#2618)
--   Always return is\_public in the /groups/:group\_id/rooms API (PR \#2630)
--   Avoid no-op media deletes (PR \#2637) Thanks to \@spantaleev!
--   Fix various embarrassing typos around user\_directory and add some doc. (PR \#2643)
--   Return whether a user is an admin within a group (PR \#2647)
--   Namespace visibility options for groups (PR \#2657)
--   Downcase UserIDs on registration (PR \#2662)
--   Cache failures when fetching URL previews (PR \#2669)
+-   Ignore \<noscript\> tags when generating URL preview descriptions (PR #2576) Thanks to @maximevaillancourt!
+-   Register some /unstable endpoints in /r0 as well (PR #2579) Thanks to @krombel!
+-   Support /keys/upload on /r0 as well as /unstable (PR #2585)
+-   Front-end proxy: pass through auth header (PR #2586)
+-   Allow ASes to deactivate their own users (PR #2589)
+-   Remove refresh tokens (PR #2613)
+-   Automatically set default displayname on register (PR #2617)
+-   Log login requests (PR #2618)
+-   Always return is\_public in the /groups/:group\_id/rooms API (PR #2630)
+-   Avoid no-op media deletes (PR #2637) Thanks to @spantaleev!
+-   Fix various embarrassing typos around user\_directory and add some doc. (PR #2643)
+-   Return whether a user is an admin within a group (PR #2647)
+-   Namespace visibility options for groups (PR #2657)
+-   Downcase UserIDs on registration (PR #2662)
+-   Cache failures when fetching URL previews (PR #2669)
 
 Bug fixes:
 
--   Fix port script (PR \#2577)
--   Fix error when running synapse with no logfile (PR \#2581)
--   Fix UI auth when deleting devices (PR \#2591)
--   Fix typo when checking if user is invited to group (PR \#2599)
--   Fix the port script to drop NUL values in all tables (PR \#2611)
--   Fix appservices being backlogged and not receiving new events due to a bug in notify\_interested\_services (PR \#2631) Thanks to \@xyzz!
--   Fix updating rooms avatar/display name when modified by admin (PR \#2636) Thanks to \@farialima!
--   Fix bug in state group storage (PR \#2649)
--   Fix 500 on invalid utf-8 in request (PR \#2663)
+-   Fix port script (PR #2577)
+-   Fix error when running synapse with no logfile (PR #2581)
+-   Fix UI auth when deleting devices (PR #2591)
+-   Fix typo when checking if user is invited to group (PR #2599)
+-   Fix the port script to drop NUL values in all tables (PR #2611)
+-   Fix appservices being backlogged and not receiving new events due to a bug in notify\_interested\_services (PR #2631) Thanks to @xyzz!
+-   Fix updating rooms avatar/display name when modified by admin (PR #2636) Thanks to @farialima!
+-   Fix bug in state group storage (PR #2649)
+-   Fix 500 on invalid utf-8 in request (PR #2663)
 
 Changes in synapse v0.24.1 (2017-10-24)
 =======================================
 
 Bug fixes:
 
--   Fix updating group profiles over federation (PR \#2567)
+-   Fix updating group profiles over federation (PR #2567)
 
 Changes in synapse v0.24.0 (2017-10-23)
 =======================================
@@ -593,31 +599,31 @@ Changes in synapse v0.24.0-rc1 (2017-10-19)
 
 Features:
 
--   Add Group Server (PR \#2352, \#2363, \#2374, \#2377, \#2378, \#2382, \#2410, \#2426, \#2430, \#2454, \#2471, \#2472, \#2544)
--   Add support for channel notifications (PR \#2501)
--   Add basic implementation of backup media store (PR \#2538)
--   Add config option to auto-join new users to rooms (PR \#2545)
+-   Add Group Server (PR #2352, #2363, #2374, #2377, #2378, #2382, #2410, #2426, #2430, #2454, #2471, #2472, #2544)
+-   Add support for channel notifications (PR #2501)
+-   Add basic implementation of backup media store (PR #2538)
+-   Add config option to auto-join new users to rooms (PR #2545)
 
 Changes:
 
--   Make the spam checker a module (PR \#2474)
--   Delete expired url cache data (PR \#2478)
--   Ignore incoming events for rooms that we have left (PR \#2490)
--   Allow spam checker to reject invites too (PR \#2492)
--   Add room creation checks to spam checker (PR \#2495)
--   Spam checking: add the invitee to user\_may\_invite (PR \#2502)
--   Process events from federation for different rooms in parallel (PR \#2520)
--   Allow error strings from spam checker (PR \#2531)
--   Improve error handling for missing files in config (PR \#2551)
+-   Make the spam checker a module (PR #2474)
+-   Delete expired url cache data (PR #2478)
+-   Ignore incoming events for rooms that we have left (PR #2490)
+-   Allow spam checker to reject invites too (PR #2492)
+-   Add room creation checks to spam checker (PR #2495)
+-   Spam checking: add the invitee to user\_may\_invite (PR #2502)
+-   Process events from federation for different rooms in parallel (PR #2520)
+-   Allow error strings from spam checker (PR #2531)
+-   Improve error handling for missing files in config (PR #2551)
 
 Bug fixes:
 
--   Fix handling SERVFAILs when doing AAAA lookups for federation (PR \#2477)
--   Fix incompatibility with newer versions of ujson (PR \#2483) Thanks to \@jeremycline!
--   Fix notification keywords that start/end with non-word chars (PR \#2500)
--   Fix stack overflow and logcontexts from linearizer (PR \#2532)
--   Fix 500 error when fields missing from power\_levels event (PR \#2552)
--   Fix 500 error when we get an error handling a PDU (PR \#2553)
+-   Fix handling SERVFAILs when doing AAAA lookups for federation (PR #2477)
+-   Fix incompatibility with newer versions of ujson (PR #2483) Thanks to @jeremycline!
+-   Fix notification keywords that start/end with non-word chars (PR #2500)
+-   Fix stack overflow and logcontexts from linearizer (PR #2532)
+-   Fix 500 error when fields missing from power\_levels event (PR #2552)
+-   Fix 500 error when we get an error handling a PDU (PR #2553)
 
 Changes in synapse v0.23.1 (2017-10-02)
 =======================================
@@ -636,42 +642,42 @@ Changes in synapse v0.23.0-rc2 (2017-09-26)
 
 Bug fixes:
 
--   Fix regression in performance of syncs (PR \#2470)
+-   Fix regression in performance of syncs (PR #2470)
 
 Changes in synapse v0.23.0-rc1 (2017-09-25)
 ===========================================
 
 Features:
 
--   Add a frontend proxy worker (PR \#2344)
--   Add support for event\_id\_only push format (PR \#2450)
--   Add a PoC for filtering spammy events (PR \#2456)
--   Add a config option to block all room invites (PR \#2457)
+-   Add a frontend proxy worker (PR #2344)
+-   Add support for event\_id\_only push format (PR #2450)
+-   Add a PoC for filtering spammy events (PR #2456)
+-   Add a config option to block all room invites (PR #2457)
 
 Changes:
 
--   Use bcrypt module instead of py-bcrypt (PR \#2288) Thanks to \@kyrias!
--   Improve performance of generating push notifications (PR \#2343, \#2357, \#2365, \#2366, \#2371)
--   Improve DB performance for device list handling in sync (PR \#2362)
--   Include a sample prometheus config (PR \#2416)
--   Document known to work postgres version (PR \#2433) Thanks to \@ptman!
+-   Use bcrypt module instead of py-bcrypt (PR #2288) Thanks to @kyrias!
+-   Improve performance of generating push notifications (PR #2343, #2357, #2365, #2366, #2371)
+-   Improve DB performance for device list handling in sync (PR #2362)
+-   Include a sample prometheus config (PR #2416)
+-   Document known to work postgres version (PR #2433) Thanks to @ptman!
 
 Bug fixes:
 
--   Fix caching error in the push evaluator (PR \#2332)
--   Fix bug where pusherpool didn\'t start and broke some rooms (PR \#2342)
--   Fix port script for user directory tables (PR \#2375)
--   Fix device lists notifications when user rejoins a room (PR \#2443, \#2449)
--   Fix sync to always send down current state events in timeline (PR \#2451)
--   Fix bug where guest users were incorrectly kicked (PR \#2453)
--   Fix bug talking to IPv6 only servers using SRV records (PR \#2462)
+-   Fix caching error in the push evaluator (PR #2332)
+-   Fix bug where pusherpool didn\'t start and broke some rooms (PR #2342)
+-   Fix port script for user directory tables (PR #2375)
+-   Fix device lists notifications when user rejoins a room (PR #2443, #2449)
+-   Fix sync to always send down current state events in timeline (PR #2451)
+-   Fix bug where guest users were incorrectly kicked (PR #2453)
+-   Fix bug talking to IPv6 only servers using SRV records (PR #2462)
 
 Changes in synapse v0.22.1 (2017-07-06)
 =======================================
 
 Bug fixes:
 
--   Fix bug where pusher pool didn\'t start and caused issues when interacting with some rooms (PR \#2342)
+-   Fix bug where pusher pool didn\'t start and caused issues when interacting with some rooms (PR #2342)
 
 Changes in synapse v0.22.0 (2017-07-06)
 =======================================
@@ -683,49 +689,49 @@ Changes in synapse v0.22.0-rc2 (2017-07-04)
 
 Changes:
 
--   Improve performance of storing user IPs (PR \#2307, \#2308)
--   Slightly improve performance of verifying access tokens (PR \#2320)
--   Slightly improve performance of event persistence (PR \#2321)
--   Increase default cache factor size from 0.1 to 0.5 (PR \#2330)
+-   Improve performance of storing user IPs (PR #2307, #2308)
+-   Slightly improve performance of verifying access tokens (PR #2320)
+-   Slightly improve performance of event persistence (PR #2321)
+-   Increase default cache factor size from 0.1 to 0.5 (PR #2330)
 
 Bug fixes:
 
--   Fix bug with storing registration sessions that caused frequent CPU churn (PR \#2319)
+-   Fix bug with storing registration sessions that caused frequent CPU churn (PR #2319)
 
 Changes in synapse v0.22.0-rc1 (2017-06-26)
 ===========================================
 
 Features:
 
--   Add a user directory API (PR \#2252, and many more)
--   Add shutdown room API to remove room from local server (PR \#2291)
--   Add API to quarantine media (PR \#2292)
--   Add new config option to not send event contents to push servers (PR \#2301) Thanks to \@cjdelisle!
+-   Add a user directory API (PR #2252, and many more)
+-   Add shutdown room API to remove room from local server (PR #2291)
+-   Add API to quarantine media (PR #2292)
+-   Add new config option to not send event contents to push servers (PR #2301) Thanks to @cjdelisle!
 
 Changes:
 
--   Various performance fixes (PR \#2177, \#2233, \#2230, \#2238, \#2248, \#2256, \#2274)
--   Deduplicate sync filters (PR \#2219) Thanks to \@krombel!
--   Correct a typo in UPGRADE.rst (PR \#2231) Thanks to \@aaronraimist!
--   Add count of one time keys to sync stream (PR \#2237)
--   Only store event\_auth for state events (PR \#2247)
--   Store URL cache preview downloads separately (PR \#2299)
+-   Various performance fixes (PR #2177, #2233, #2230, #2238, #2248, #2256, #2274)
+-   Deduplicate sync filters (PR #2219) Thanks to @krombel!
+-   Correct a typo in UPGRADE.rst (PR #2231) Thanks to @aaronraimist!
+-   Add count of one time keys to sync stream (PR #2237)
+-   Only store event\_auth for state events (PR #2247)
+-   Store URL cache preview downloads separately (PR #2299)
 
 Bug fixes:
 
--   Fix users not getting notifications when AS listened to that user\_id (PR \#2216) Thanks to \@slipeer!
--   Fix users without push set up not getting notifications after joining rooms (PR \#2236)
--   Fix preview url API to trim long descriptions (PR \#2243)
--   Fix bug where we used cached but unpersisted state group as prev group, resulting in broken state of restart (PR \#2263)
--   Fix removing of pushers when using workers (PR \#2267)
--   Fix CORS headers to allow Authorization header (PR \#2285) Thanks to \@krombel!
+-   Fix users not getting notifications when AS listened to that user\_id (PR #2216) Thanks to @slipeer!
+-   Fix users without push set up not getting notifications after joining rooms (PR #2236)
+-   Fix preview url API to trim long descriptions (PR #2243)
+-   Fix bug where we used cached but unpersisted state group as prev group, resulting in broken state of restart (PR #2263)
+-   Fix removing of pushers when using workers (PR #2267)
+-   Fix CORS headers to allow Authorization header (PR #2285) Thanks to @krombel!
 
 Changes in synapse v0.21.1 (2017-06-15)
 =======================================
 
 Bug fixes:
 
--   Fix bug in anonymous usage statistic reporting (PR \#2281)
+-   Fix bug in anonymous usage statistic reporting (PR #2281)
 
 Changes in synapse v0.21.0 (2017-05-18)
 =======================================
@@ -737,116 +743,116 @@ Changes in synapse v0.21.0-rc3 (2017-05-17)
 
 Features:
 
--   Add per user rate-limiting overrides (PR \#2208)
--   Add config option to limit maximum number of events requested by `/sync` and `/messages` (PR \#2221) Thanks to \@psaavedra!
+-   Add per user rate-limiting overrides (PR #2208)
+-   Add config option to limit maximum number of events requested by `/sync` and `/messages` (PR #2221) Thanks to @psaavedra!
 
 Changes:
 
--   Various small performance fixes (PR \#2201, \#2202, \#2224, \#2226, \#2227, \#2228, \#2229)
--   Update username availability checker API (PR \#2209, \#2213)
--   When purging, don\'t de-delta state groups we\'re about to delete (PR \#2214)
--   Documentation to check synapse version (PR \#2215) Thanks to \@hamber-dick!
--   Add an index to event\_search to speed up purge history API (PR \#2218)
+-   Various small performance fixes (PR #2201, #2202, #2224, #2226, #2227, #2228, #2229)
+-   Update username availability checker API (PR #2209, #2213)
+-   When purging, don\'t de-delta state groups we\'re about to delete (PR #2214)
+-   Documentation to check synapse version (PR #2215) Thanks to @hamber-dick!
+-   Add an index to event\_search to speed up purge history API (PR #2218)
 
 Bug fixes:
 
--   Fix API to allow clients to upload one-time-keys with new sigs (PR \#2206)
+-   Fix API to allow clients to upload one-time-keys with new sigs (PR #2206)
 
 Changes in synapse v0.21.0-rc2 (2017-05-08)
 ===========================================
 
 Changes:
 
--   Always mark remotes as up if we receive a signed request from them (PR \#2190)
+-   Always mark remotes as up if we receive a signed request from them (PR #2190)
 
 Bug fixes:
 
--   Fix bug where users got pushed for rooms they had muted (PR \#2200)
+-   Fix bug where users got pushed for rooms they had muted (PR #2200)
 
 Changes in synapse v0.21.0-rc1 (2017-05-08)
 ===========================================
 
 Features:
 
--   Add username availability checker API (PR \#2183)
--   Add read marker API (PR \#2120)
+-   Add username availability checker API (PR #2183)
+-   Add read marker API (PR #2120)
 
 Changes:
 
--   Enable guest access for the 3pl/3pid APIs (PR \#1986)
--   Add setting to support TURN for guests (PR \#2011)
--   Various performance improvements (PR \#2075, \#2076, \#2080, \#2083, \#2108, \#2158, \#2176, \#2185)
--   Make synctl a bit more user friendly (PR \#2078, \#2127) Thanks \@APwhitehat!
--   Replace HTTP replication with TCP replication (PR \#2082, \#2097, \#2098, \#2099, \#2103, \#2014, \#2016, \#2115, \#2116, \#2117)
--   Support authenticated SMTP (PR \#2102) Thanks \@DanielDent!
--   Add a counter metric for successfully-sent transactions (PR \#2121)
--   Propagate errors sensibly from proxied IS requests (PR \#2147)
--   Add more granular event send metrics (PR \#2178)
+-   Enable guest access for the 3pl/3pid APIs (PR #1986)
+-   Add setting to support TURN for guests (PR #2011)
+-   Various performance improvements (PR #2075, #2076, #2080, #2083, #2108, #2158, #2176, #2185)
+-   Make synctl a bit more user friendly (PR #2078, #2127) Thanks @APwhitehat!
+-   Replace HTTP replication with TCP replication (PR #2082, #2097, #2098, #2099, #2103, #2014, #2016, #2115, #2116, #2117)
+-   Support authenticated SMTP (PR #2102) Thanks @DanielDent!
+-   Add a counter metric for successfully-sent transactions (PR #2121)
+-   Propagate errors sensibly from proxied IS requests (PR #2147)
+-   Add more granular event send metrics (PR #2178)
 
 Bug fixes:
 
--   Fix nuke-room script to work with current schema (PR \#1927) Thanks \@zuckschwerdt!
--   Fix db port script to not assume postgres tables are in the public schema (PR \#2024) Thanks \@jerrykan!
--   Fix getting latest device IP for user with no devices (PR \#2118)
--   Fix rejection of invites to unreachable servers (PR \#2145)
--   Fix code for reporting old verify keys in synapse (PR \#2156)
--   Fix invite state to always include all events (PR \#2163)
--   Fix bug where synapse would always fetch state for any missing event (PR \#2170)
--   Fix a leak with timed out HTTP connections (PR \#2180)
--   Fix bug where we didn\'t time out HTTP requests to ASes (PR \#2192)
+-   Fix nuke-room script to work with current schema (PR #1927) Thanks @zuckschwerdt!
+-   Fix db port script to not assume postgres tables are in the public schema (PR #2024) Thanks @jerrykan!
+-   Fix getting latest device IP for user with no devices (PR #2118)
+-   Fix rejection of invites to unreachable servers (PR #2145)
+-   Fix code for reporting old verify keys in synapse (PR #2156)
+-   Fix invite state to always include all events (PR #2163)
+-   Fix bug where synapse would always fetch state for any missing event (PR #2170)
+-   Fix a leak with timed out HTTP connections (PR #2180)
+-   Fix bug where we didn\'t time out HTTP requests to ASes (PR #2192)
 
 Docs:
 
--   Clarify doc for SQLite to PostgreSQL port (PR \#1961) Thanks \@benhylau!
--   Fix typo in synctl help (PR \#2107) Thanks \@HarHarLinks!
--   `web_client_location` documentation fix (PR \#2131) Thanks \@matthewjwolff!
--   Update README.rst with FreeBSD changes (PR \#2132) Thanks \@feld!
--   Clarify setting up metrics (PR \#2149) Thanks \@encks!
+-   Clarify doc for SQLite to PostgreSQL port (PR #1961) Thanks @benhylau!
+-   Fix typo in synctl help (PR #2107) Thanks @HarHarLinks!
+-   `web_client_location` documentation fix (PR #2131) Thanks @matthewjwolff!
+-   Update README.rst with FreeBSD changes (PR #2132) Thanks @feld!
+-   Clarify setting up metrics (PR #2149) Thanks @encks!
 
 Changes in synapse v0.20.0 (2017-04-11)
 =======================================
 
 Bug fixes:
 
--   Fix joining rooms over federation where not all servers in the room saw the new server had joined (PR \#2094)
+-   Fix joining rooms over federation where not all servers in the room saw the new server had joined (PR #2094)
 
 Changes in synapse v0.20.0-rc1 (2017-03-30)
 ===========================================
 
 Features:
 
--   Add delete\_devices API (PR \#1993)
--   Add phone number registration/login support (PR \#1994, \#2055)
+-   Add delete\_devices API (PR #1993)
+-   Add phone number registration/login support (PR #1994, #2055)
 
 Changes:
 
--   Use JSONSchema for validation of filters. Thanks \@pik! (PR \#1783)
--   Reread log config on SIGHUP (PR \#1982)
--   Speed up public room list (PR \#1989)
--   Add helpful texts to logger config options (PR \#1990)
--   Minor `/sync` performance improvements. (PR \#2002, \#2013, \#2022)
--   Add some debug to help diagnose weird federation issue (PR \#2035)
--   Correctly limit retries for all federation requests (PR \#2050, \#2061)
--   Don\'t lock table when persisting new one time keys (PR \#2053)
--   Reduce some CPU work on DB threads (PR \#2054)
--   Cache hosts in room (PR \#2060)
--   Batch sending of device list pokes (PR \#2063)
--   Speed up persist event path in certain edge cases (PR \#2070)
+-   Use JSONSchema for validation of filters. Thanks @pik! (PR #1783)
+-   Reread log config on SIGHUP (PR #1982)
+-   Speed up public room list (PR #1989)
+-   Add helpful texts to logger config options (PR #1990)
+-   Minor `/sync` performance improvements. (PR #2002, #2013, #2022)
+-   Add some debug to help diagnose weird federation issue (PR #2035)
+-   Correctly limit retries for all federation requests (PR #2050, #2061)
+-   Don\'t lock table when persisting new one time keys (PR #2053)
+-   Reduce some CPU work on DB threads (PR #2054)
+-   Cache hosts in room (PR #2060)
+-   Batch sending of device list pokes (PR #2063)
+-   Speed up persist event path in certain edge cases (PR #2070)
 
 Bug fixes:
 
--   Fix bug where current\_state\_events renamed to current\_state\_ids (PR \#1849)
--   Fix routing loop when fetching remote media (PR \#1992)
--   Fix current\_state\_events table to not lie (PR \#1996)
--   Fix CAS login to handle PartialDownloadError (PR \#1997)
--   Fix assertion to stop transaction queue getting wedged (PR \#2010)
--   Fix presence to fallback to last\_active\_ts if it beats the last sync time. Thanks \@Half-Shot! (PR \#2014)
--   Fix bug when federation received a PDU while a room join is in progress (PR \#2016)
--   Fix resetting state on rejected events (PR \#2025)
--   Fix installation issues in readme. Thanks \@ricco386 (PR \#2037)
--   Fix caching of remote servers\' signature keys (PR \#2042)
--   Fix some leaking log context (PR \#2048, \#2049, \#2057, \#2058)
--   Fix rejection of invites not reaching sync (PR \#2056)
+-   Fix bug where current\_state\_events renamed to current\_state\_ids (PR #1849)
+-   Fix routing loop when fetching remote media (PR #1992)
+-   Fix current\_state\_events table to not lie (PR #1996)
+-   Fix CAS login to handle PartialDownloadError (PR #1997)
+-   Fix assertion to stop transaction queue getting wedged (PR #2010)
+-   Fix presence to fallback to last\_active\_ts if it beats the last sync time. Thanks @Half-Shot! (PR #2014)
+-   Fix bug when federation received a PDU while a room join is in progress (PR #2016)
+-   Fix resetting state on rejected events (PR #2025)
+-   Fix installation issues in readme. Thanks @ricco386 (PR #2037)
+-   Fix caching of remote servers\' signature keys (PR #2042)
+-   Fix some leaking log context (PR #2048, #2049, #2057, #2058)
+-   Fix rejection of invites not reaching sync (PR #2056)
 
 Changes in synapse v0.19.3 (2017-03-20)
 =======================================
@@ -865,36 +871,36 @@ Changes in synapse v0.19.3-rc1 (2017-03-08)
 
 Features:
 
--   Add some administration functionalities. Thanks to morteza-araby! (PR \#1784)
+-   Add some administration functionalities. Thanks to morteza-araby! (PR #1784)
 
 Changes:
 
--   Reduce database table sizes (PR \#1873, \#1916, \#1923, \#1963)
--   Update contrib/ to not use syutil. Thanks to andrewshadura! (PR \#1907)
--   Don\'t fetch current state when sending an event in common case (PR \#1955)
+-   Reduce database table sizes (PR #1873, #1916, #1923, #1963)
+-   Update contrib/ to not use syutil. Thanks to andrewshadura! (PR #1907)
+-   Don\'t fetch current state when sending an event in common case (PR #1955)
 
 Bug fixes:
 
--   Fix synapse\_port\_db failure. Thanks to Pneumaticat! (PR \#1904)
--   Fix caching to not cache error responses (PR \#1913)
--   Fix APIs to make kick & ban reasons work (PR \#1917)
--   Fix bugs in the /keys/changes api (PR \#1921)
--   Fix bug where users couldn\'t forget rooms they were banned from (PR \#1922)
--   Fix issue with long language values in pushers API (PR \#1925)
--   Fix a race in transaction queue (PR \#1930)
--   Fix dynamic thumbnailing to preserve aspect ratio. Thanks to jkolo! (PR \#1945)
--   Fix device list update to not constantly resync (PR \#1964)
--   Fix potential for huge memory usage when getting device that have changed (PR \#1969)
+-   Fix synapse\_port\_db failure. Thanks to Pneumaticat! (PR #1904)
+-   Fix caching to not cache error responses (PR #1913)
+-   Fix APIs to make kick & ban reasons work (PR #1917)
+-   Fix bugs in the /keys/changes api (PR #1921)
+-   Fix bug where users couldn\'t forget rooms they were banned from (PR #1922)
+-   Fix issue with long language values in pushers API (PR #1925)
+-   Fix a race in transaction queue (PR #1930)
+-   Fix dynamic thumbnailing to preserve aspect ratio. Thanks to jkolo! (PR #1945)
+-   Fix device list update to not constantly resync (PR #1964)
+-   Fix potential for huge memory usage when getting device that have changed (PR #1969)
 
 Changes in synapse v0.19.2 (2017-02-20)
 =======================================
 
--   Fix bug with event visibility check in /context/ API. Thanks to Tokodomo for pointing it out! (PR \#1929)
+-   Fix bug with event visibility check in /context/ API. Thanks to Tokodomo for pointing it out! (PR #1929)
 
 Changes in synapse v0.19.1 (2017-02-09)
 =======================================
 
--   Fix bug where state was incorrectly reset in a room when synapse received an event over federation that did not pass auth checks (PR \#1892)
+-   Fix bug where state was incorrectly reset in a room when synapse received an event over federation that did not pass auth checks (PR #1892)
 
 Changes in synapse v0.19.0 (2017-02-04)
 =======================================
@@ -904,59 +910,59 @@ No changes since RC 4.
 Changes in synapse v0.19.0-rc4 (2017-02-02)
 ===========================================
 
--   Bump cache sizes for common membership queries (PR \#1879)
+-   Bump cache sizes for common membership queries (PR #1879)
 
 Changes in synapse v0.19.0-rc3 (2017-02-02)
 ===========================================
 
--   Fix email push in pusher worker (PR \#1875)
--   Make presence.get\_new\_events a bit faster (PR \#1876)
--   Make /keys/changes a bit more performant (PR \#1877)
+-   Fix email push in pusher worker (PR #1875)
+-   Make presence.get\_new\_events a bit faster (PR #1876)
+-   Make /keys/changes a bit more performant (PR #1877)
 
 Changes in synapse v0.19.0-rc2 (2017-02-02)
 ===========================================
 
--   Include newly joined users in /keys/changes API (PR \#1872)
+-   Include newly joined users in /keys/changes API (PR #1872)
 
 Changes in synapse v0.19.0-rc1 (2017-02-02)
 ===========================================
 
 Features:
 
--   Add support for specifying multiple bind addresses (PR \#1709, \#1712, \#1795, \#1835). Thanks to \@kyrias!
--   Add /account/3pid/delete endpoint (PR \#1714)
--   Add config option to configure the Riot URL used in notification emails (PR \#1811). Thanks to \@aperezdc!
--   Add username and password config options for turn server (PR \#1832). Thanks to \@xsteadfastx!
--   Implement device lists updates over federation (PR \#1857, \#1861, \#1864)
--   Implement /keys/changes (PR \#1869, \#1872)
+-   Add support for specifying multiple bind addresses (PR #1709, #1712, #1795, #1835). Thanks to @kyrias!
+-   Add /account/3pid/delete endpoint (PR #1714)
+-   Add config option to configure the Riot URL used in notification emails (PR #1811). Thanks to @aperezdc!
+-   Add username and password config options for turn server (PR #1832). Thanks to @xsteadfastx!
+-   Implement device lists updates over federation (PR #1857, #1861, #1864)
+-   Implement /keys/changes (PR #1869, #1872)
 
 Changes:
 
--   Improve IPv6 support (PR \#1696). Thanks to \@kyrias and \@glyph!
--   Log which files we saved attachments to in the media\_repository (PR \#1791)
--   Linearize updates to membership via PUT /state/ to better handle multiple joins (PR \#1787)
--   Limit number of entries to prefill from cache on startup (PR \#1792)
--   Remove full\_twisted\_stacktraces option (PR \#1802)
--   Measure size of some caches by sum of the size of cached values (PR \#1815)
--   Measure metrics of string\_cache (PR \#1821)
--   Reduce logging verbosity (PR \#1822, \#1823, \#1824)
--   Don\'t clobber a displayname or avatar\_url if provided by an m.room.member event (PR \#1852)
--   Better handle 401/404 response for federation /send/ (PR \#1866, \#1871)
+-   Improve IPv6 support (PR #1696). Thanks to @kyrias and @glyph!
+-   Log which files we saved attachments to in the media\_repository (PR #1791)
+-   Linearize updates to membership via PUT /state/ to better handle multiple joins (PR #1787)
+-   Limit number of entries to prefill from cache on startup (PR #1792)
+-   Remove full\_twisted\_stacktraces option (PR #1802)
+-   Measure size of some caches by sum of the size of cached values (PR #1815)
+-   Measure metrics of string\_cache (PR #1821)
+-   Reduce logging verbosity (PR #1822, #1823, #1824)
+-   Don\'t clobber a displayname or avatar\_url if provided by an m.room.member event (PR #1852)
+-   Better handle 401/404 response for federation /send/ (PR #1866, #1871)
 
 Fixes:
 
--   Fix ability to change password to a non-ascii one (PR \#1711)
--   Fix push getting stuck due to looking at the wrong view of state (PR \#1820)
--   Fix email address comparison to be case insensitive (PR \#1827)
--   Fix occasional inconsistencies of room membership (PR \#1836, \#1840)
+-   Fix ability to change password to a non-ascii one (PR #1711)
+-   Fix push getting stuck due to looking at the wrong view of state (PR #1820)
+-   Fix email address comparison to be case insensitive (PR #1827)
+-   Fix occasional inconsistencies of room membership (PR #1836, #1840)
 
 Performance:
 
--   Don\'t block messages sending on bumping presence (PR \#1789)
--   Change device\_inbox stream index to include user (PR \#1793)
--   Optimise state resolution (PR \#1818)
--   Use DB cache of joined users for presence (PR \#1862)
--   Add an index to make membership queries faster (PR \#1867)
+-   Don\'t block messages sending on bumping presence (PR #1789)
+-   Change device\_inbox stream index to include user (PR #1793)
+-   Optimise state resolution (PR #1818)
+-   Use DB cache of joined users for presence (PR #1862)
+-   Add an index to make membership queries faster (PR #1867)
 
 Changes in synapse v0.18.7 (2017-01-09)
 =======================================
@@ -984,70 +990,70 @@ Changes in synapse v0.18.6 (2017-01-06)
 
 Bug fixes:
 
--   Fix bug when checking if a guest user is allowed to join a room (PR \#1772) Thanks to Patrik Oldsberg for diagnosing and the fix!
+-   Fix bug when checking if a guest user is allowed to join a room (PR #1772) Thanks to Patrik Oldsberg for diagnosing and the fix!
 
 Changes in synapse v0.18.6-rc3 (2017-01-05)
 ===========================================
 
 Bug fixes:
 
--   Fix bug where we failed to send ban events to the banned server (PR \#1758)
--   Fix bug where we sent event that didn\'t originate on this server to other servers (PR \#1764)
--   Fix bug where processing an event from a remote server took a long time because we were making long HTTP requests (PR \#1765, PR \#1744)
+-   Fix bug where we failed to send ban events to the banned server (PR #1758)
+-   Fix bug where we sent event that didn\'t originate on this server to other servers (PR #1764)
+-   Fix bug where processing an event from a remote server took a long time because we were making long HTTP requests (PR #1765, PR #1744)
 
 Changes:
 
--   Improve logging for debugging deadlocks (PR \#1766, PR \#1767)
+-   Improve logging for debugging deadlocks (PR #1766, PR #1767)
 
 Changes in synapse v0.18.6-rc2 (2016-12-30)
 ===========================================
 
 Bug fixes:
 
--   Fix memory leak in twisted by initialising logging correctly (PR \#1731)
--   Fix bug where fetching missing events took an unacceptable amount of time in large rooms (PR \#1734)
+-   Fix memory leak in twisted by initialising logging correctly (PR #1731)
+-   Fix bug where fetching missing events took an unacceptable amount of time in large rooms (PR #1734)
 
 Changes in synapse v0.18.6-rc1 (2016-12-29)
 ===========================================
 
 Bug fixes:
 
--   Make sure that outbound connections are closed (PR \#1725)
+-   Make sure that outbound connections are closed (PR #1725)
 
 Changes in synapse v0.18.5 (2016-12-16)
 =======================================
 
 Bug fixes:
 
--   Fix federation /backfill returning events it shouldn\'t (PR \#1700)
--   Fix crash in url preview (PR \#1701)
+-   Fix federation /backfill returning events it shouldn\'t (PR #1700)
+-   Fix crash in url preview (PR #1701)
 
 Changes in synapse v0.18.5-rc3 (2016-12-13)
 ===========================================
 
 Features:
 
--   Add support for E2E for guests (PR \#1653)
--   Add new API appservice specific public room list (PR \#1676)
--   Add new room membership APIs (PR \#1680)
+-   Add support for E2E for guests (PR #1653)
+-   Add new API appservice specific public room list (PR #1676)
+-   Add new room membership APIs (PR #1680)
 
 Changes:
 
--   Enable guest access for private rooms by default (PR \#653)
--   Limit the number of events that can be created on a given room concurrently (PR \#1620)
--   Log the args that we have on UI auth completion (PR \#1649)
--   Stop generating refresh\_tokens (PR \#1654)
--   Stop putting a time caveat on access tokens (PR \#1656)
--   Remove unspecced GET endpoints for e2e keys (PR \#1694)
+-   Enable guest access for private rooms by default (PR #653)
+-   Limit the number of events that can be created on a given room concurrently (PR #1620)
+-   Log the args that we have on UI auth completion (PR #1649)
+-   Stop generating refresh\_tokens (PR #1654)
+-   Stop putting a time caveat on access tokens (PR #1656)
+-   Remove unspecced GET endpoints for e2e keys (PR #1694)
 
 Bug fixes:
 
--   Fix handling of 500 and 429\'s over federation (PR \#1650)
--   Fix Content-Type header parsing (PR \#1660)
--   Fix error when previewing sites that include unicode, thanks to kyrias (PR \#1664)
--   Fix some cases where we drop read receipts (PR \#1678)
--   Fix bug where calls to `/sync` didn\'t correctly timeout (PR \#1683)
--   Fix bug where E2E key query would fail if a single remote host failed (PR \#1686)
+-   Fix handling of 500 and 429\'s over federation (PR #1650)
+-   Fix Content-Type header parsing (PR #1660)
+-   Fix error when previewing sites that include unicode, thanks to kyrias (PR #1664)
+-   Fix some cases where we drop read receipts (PR #1678)
+-   Fix bug where calls to `/sync` didn\'t correctly timeout (PR #1683)
+-   Fix bug where E2E key query would fail if a single remote host failed (PR #1686)
 
 Changes in synapse v0.18.5-rc2 (2016-11-24)
 ===========================================
@@ -1061,37 +1067,37 @@ Changes in synapse v0.18.5-rc1 (2016-11-24)
 
 Features:
 
--   Implement \"event\_fields\" in filters (PR \#1638)
+-   Implement \"event\_fields\" in filters (PR #1638)
 
 Changes:
 
--   Use external ldap auth pacakge (PR \#1628)
--   Split out federation transaction sending to a worker (PR \#1635)
--   Fail with a coherent error message if /sync?filter= is invalid (PR \#1636)
--   More efficient notif count queries (PR \#1644)
+-   Use external ldap auth pacakge (PR #1628)
+-   Split out federation transaction sending to a worker (PR #1635)
+-   Fail with a coherent error message if /sync?filter= is invalid (PR #1636)
+-   More efficient notif count queries (PR #1644)
 
 Changes in synapse v0.18.4 (2016-11-22)
 =======================================
 
 Bug fixes:
 
--   Add workaround for buggy clients that the fail to register (PR \#1632)
+-   Add workaround for buggy clients that the fail to register (PR #1632)
 
 Changes in synapse v0.18.4-rc1 (2016-11-14)
 ===========================================
 
 Changes:
 
--   Various database efficiency improvements (PR \#1188, \#1192)
--   Update default config to blacklist more internal IPs, thanks to Euan Kemp (PR \#1198)
--   Allow specifying duration in minutes in config, thanks to Daniel Dent (PR \#1625)
+-   Various database efficiency improvements (PR #1188, #1192)
+-   Update default config to blacklist more internal IPs, thanks to Euan Kemp (PR #1198)
+-   Allow specifying duration in minutes in config, thanks to Daniel Dent (PR #1625)
 
 Bug fixes:
 
--   Fix media repo to set CORs headers on responses (PR \#1190)
--   Fix registration to not error on non-ascii passwords (PR \#1191)
--   Fix create event code to limit the number of prev\_events (PR \#1615)
--   Fix bug in transaction ID deduplication (PR \#1624)
+-   Fix media repo to set CORs headers on responses (PR #1190)
+-   Fix registration to not error on non-ascii passwords (PR #1191)
+-   Fix create event code to limit the number of prev\_events (PR #1615)
+-   Fix bug in transaction ID deduplication (PR #1624)
 
 Changes in synapse v0.18.3 (2016-11-08)
 =======================================
@@ -1112,32 +1118,32 @@ Changes in synapse v0.18.2-rc5 (2016-10-28)
 
 Bug fixes:
 
--   Fix prometheus process metrics in worker processes (PR \#1184)
+-   Fix prometheus process metrics in worker processes (PR #1184)
 
 Changes in synapse v0.18.2-rc4 (2016-10-27)
 ===========================================
 
 Bug fixes:
 
--   Fix `user_threepids` schema delta, which in some instances prevented startup after upgrade (PR \#1183)
+-   Fix `user_threepids` schema delta, which in some instances prevented startup after upgrade (PR #1183)
 
 Changes in synapse v0.18.2-rc3 (2016-10-27)
 ===========================================
 
 Changes:
 
--   Allow clients to supply access tokens as headers (PR \#1098)
--   Clarify error codes for GET /filter/, thanks to Alexander Maznev (PR \#1164)
--   Make password reset email field case insensitive (PR \#1170)
--   Reduce redundant database work in email pusher (PR \#1174)
--   Allow configurable rate limiting per AS (PR \#1175)
--   Check whether to ratelimit sooner to avoid work (PR \#1176)
--   Standardise prometheus metrics (PR \#1177)
+-   Allow clients to supply access tokens as headers (PR #1098)
+-   Clarify error codes for GET /filter/, thanks to Alexander Maznev (PR #1164)
+-   Make password reset email field case insensitive (PR #1170)
+-   Reduce redundant database work in email pusher (PR #1174)
+-   Allow configurable rate limiting per AS (PR #1175)
+-   Check whether to ratelimit sooner to avoid work (PR #1176)
+-   Standardise prometheus metrics (PR #1177)
 
 Bug fixes:
 
--   Fix incredibly slow back pagination query (PR \#1178)
--   Fix infinite typing bug (PR \#1179)
+-   Fix incredibly slow back pagination query (PR #1178)
+-   Fix infinite typing bug (PR #1179)
 
 Changes in synapse v0.18.2-rc2 (2016-10-25)
 ===========================================
@@ -1149,22 +1155,23 @@ Changes in synapse v0.18.2-rc1 (2016-10-17)
 
 Changes:
 
--   Remove redundant event\_auth index (PR \#1113)
--   Reduce DB hits for replication (PR \#1141)
--   Implement pluggable password auth (PR \#1155)
--   Remove rate limiting from app service senders and fix get\_or\_create\_user requester, thanks to Patrik Oldsberg (PR \#1157)
--   window.postmessage for Interactive Auth fallback (PR \#1159)
--   Use sys.executable instead of hardcoded python, thanks to Pedro Larroy (PR \#1162)
--   Add config option for adding additional TLS fingerprints (PR \#1167)
--   User-interactive auth on delete device (PR \#1168)
+-   Remove redundant event\_auth index (PR #1113)
+-   Reduce DB hits for replication (PR #1141)
+-   Implement pluggable password auth (PR #1155)
+-   Remove rate limiting from app service senders and fix get\_or\_create\_user requester, thanks to Patrik Oldsberg (PR #1157)
+-   window.postmessage for Interactive Auth fallback (PR #1159)
+-   Use sys.executable instead of hardcoded python, thanks to Pedro Larroy (PR #1162)
+-   Add config option for adding additional TLS fingerprints (PR #1167)
+-   User-interactive auth on delete device (PR #1168)
 
 Bug fixes:
 
--   Fix not being allowed to set your own state\_key, thanks to Patrik Oldsberg (PR \#1150)
--   Fix interactive auth to return 401 from for incorrect password (PR \#1160, \#1166)
--   Fix email push notifs being dropped (PR \#1169)
+-   Fix not being allowed to set your own state\_key, thanks to Patrik Oldsberg (PR #1150)
+-   Fix interactive auth to return 401 from for incorrect password (PR #1160, #1166)
+-   Fix email push notifs being dropped (PR #1169)
 
-Changes in synapse v0.18.1 (2016-10-05) ======================================
+Changes in synapse v0.18.1 (2016-10-05)
+=======================================
 
 No changes since v0.18.1-rc1
 
@@ -1173,19 +1180,19 @@ Changes in synapse v0.18.1-rc1 (2016-09-30)
 
 Features:
 
--   Add total\_room\_count\_estimate to `/publicRooms` (PR \#1133)
+-   Add total\_room\_count\_estimate to `/publicRooms` (PR #1133)
 
 Changes:
 
--   Time out typing over federation (PR \#1140)
--   Restructure LDAP authentication (PR \#1153)
+-   Time out typing over federation (PR #1140)
+-   Restructure LDAP authentication (PR #1153)
 
 Bug fixes:
 
--   Fix 3pid invites when server is already in the room (PR \#1136)
--   Fix upgrading with SQLite taking lots of CPU for a few days after upgrade (PR \#1144)
--   Fix upgrading from very old database versions (PR \#1145)
--   Fix port script to work with recently added tables (PR \#1146)
+-   Fix 3pid invites when server is already in the room (PR #1136)
+-   Fix upgrading with SQLite taking lots of CPU for a few days after upgrade (PR #1144)
+-   Fix upgrading from very old database versions (PR #1145)
+-   Fix port script to work with recently added tables (PR #1146)
 
 Changes in synapse v0.18.0 (2016-09-19)
 =======================================
@@ -1194,39 +1201,39 @@ The release includes major changes to the state storage database schemas, which
 
 Changes:
 
--   Make public room search case insensitive (PR \#1127)
+-   Make public room search case insensitive (PR #1127)
 
 Bug fixes:
 
--   Fix and clean up publicRooms pagination (PR \#1129)
+-   Fix and clean up publicRooms pagination (PR #1129)
 
 Changes in synapse v0.18.0-rc1 (2016-09-16)
 ===========================================
 
 Features:
 
--   Add `only=highlight` on `/notifications` (PR \#1081)
--   Add server param to /publicRooms (PR \#1082)
--   Allow clients to ask for the whole of a single state event (PR \#1094)
--   Add is\_direct param to /createRoom (PR \#1108)
--   Add pagination support to publicRooms (PR \#1121)
--   Add very basic filter API to /publicRooms (PR \#1126)
--   Add basic direct to device messaging support for E2E (PR \#1074, \#1084, \#1104, \#1111)
+-   Add `only=highlight` on `/notifications` (PR #1081)
+-   Add server param to /publicRooms (PR #1082)
+-   Allow clients to ask for the whole of a single state event (PR #1094)
+-   Add is\_direct param to /createRoom (PR #1108)
+-   Add pagination support to publicRooms (PR #1121)
+-   Add very basic filter API to /publicRooms (PR #1126)
+-   Add basic direct to device messaging support for E2E (PR #1074, #1084, #1104, #1111)
 
 Changes:
 
--   Move to storing state\_groups\_state as deltas, greatly reducing DB size (PR \#1065)
--   Reduce amount of state pulled out of the DB during common requests (PR \#1069)
--   Allow PDF to be rendered from media repo (PR \#1071)
--   Reindex state\_groups\_state after pruning (PR \#1085)
--   Clobber EDUs in send queue (PR \#1095)
--   Conform better to the CAS protocol specification (PR \#1100)
--   Limit how often we ask for keys from dead servers (PR \#1114)
+-   Move to storing state\_groups\_state as deltas, greatly reducing DB size (PR #1065)
+-   Reduce amount of state pulled out of the DB during common requests (PR #1069)
+-   Allow PDF to be rendered from media repo (PR #1071)
+-   Reindex state\_groups\_state after pruning (PR #1085)
+-   Clobber EDUs in send queue (PR #1095)
+-   Conform better to the CAS protocol specification (PR #1100)
+-   Limit how often we ask for keys from dead servers (PR #1114)
 
 Bug fixes:
 
--   Fix /notifications API when used with `from` param (PR \#1080)
--   Fix backfill when cannot find an event. (PR \#1107)
+-   Fix /notifications API when used with `from` param (PR #1080)
+-   Fix backfill when cannot find an event. (PR #1107)
 
 Changes in synapse v0.17.3 (2016-09-09)
 =======================================
@@ -1245,20 +1252,20 @@ Changes in synapse v0.17.2-rc1 (2016-09-05)
 
 Features:
 
--   Start adding store-and-forward direct-to-device messaging (PR \#1046, \#1050, \#1062, \#1066)
+-   Start adding store-and-forward direct-to-device messaging (PR #1046, #1050, #1062, #1066)
 
 Changes:
 
--   Avoid pulling the full state of a room out so often (PR \#1047, \#1049, \#1063, \#1068)
--   Don\'t notify for online to online presence transitions. (PR \#1054)
--   Occasionally persist unpersisted presence updates (PR \#1055)
--   Allow application services to have an optional \'url\' (PR \#1056)
--   Clean up old sent transactions from DB (PR \#1059)
+-   Avoid pulling the full state of a room out so often (PR #1047, #1049, #1063, #1068)
+-   Don\'t notify for online to online presence transitions. (PR #1054)
+-   Occasionally persist unpersisted presence updates (PR #1055)
+-   Allow application services to have an optional \'url\' (PR #1056)
+-   Clean up old sent transactions from DB (PR #1059)
 
 Bug fixes:
 
--   Fix None check in backfill (PR \#1043)
--   Fix membership changes to be idempotent (PR \#1067)
+-   Fix None check in backfill (PR #1043)
+-   Fix membership changes to be idempotent (PR #1067)
 -   Fix bug in get\_pdu where it would sometimes return events with incorrect signature
 
 Changes in synapse v0.17.1 (2016-08-24)
@@ -1266,86 +1273,86 @@ Changes in synapse v0.17.1 (2016-08-24)
 
 Changes:
 
--   Delete old received\_transactions rows (PR \#1038)
--   Pass through user-supplied content in /join/\$room\_id (PR \#1039)
+-   Delete old received\_transactions rows (PR #1038)
+-   Pass through user-supplied content in /join/\$room\_id (PR #1039)
 
 Bug fixes:
 
--   Fix bug with backfill (PR \#1040)
+-   Fix bug with backfill (PR #1040)
 
 Changes in synapse v0.17.1-rc1 (2016-08-22)
 ===========================================
 
 Features:
 
--   Add notification API (PR \#1028)
+-   Add notification API (PR #1028)
 
 Changes:
 
--   Don\'t print stack traces when failing to get remote keys (PR \#996)
--   Various federation /event/ perf improvements (PR \#998)
--   Only process one local membership event per room at a time (PR \#1005)
--   Move default display name push rule (PR \#1011, \#1023)
--   Fix up preview URL API. Add tests. (PR \#1015)
--   Set `Content-Security-Policy` on media repo (PR \#1021)
--   Make notify\_interested\_services faster (PR \#1022)
--   Add usage stats to prometheus monitoring (PR \#1037)
+-   Don\'t print stack traces when failing to get remote keys (PR #996)
+-   Various federation /event/ perf improvements (PR #998)
+-   Only process one local membership event per room at a time (PR #1005)
+-   Move default display name push rule (PR #1011, #1023)
+-   Fix up preview URL API. Add tests. (PR #1015)
+-   Set `Content-Security-Policy` on media repo (PR #1021)
+-   Make notify\_interested\_services faster (PR #1022)
+-   Add usage stats to prometheus monitoring (PR #1037)
 
 Bug fixes:
 
--   Fix token login (PR \#993)
--   Fix CAS login (PR \#994, \#995)
--   Fix /sync to not clobber status\_msg (PR \#997)
--   Fix redacted state events to include prev\_content (PR \#1003)
--   Fix some bugs in the auth/ldap handler (PR \#1007)
--   Fix backfill request to limit URI length, so that remotes don\'t reject the requests due to path length limits (PR \#1012)
--   Fix AS push code to not send duplicate events (PR \#1025)
+-   Fix token login (PR #993)
+-   Fix CAS login (PR #994, #995)
+-   Fix /sync to not clobber status\_msg (PR #997)
+-   Fix redacted state events to include prev\_content (PR #1003)
+-   Fix some bugs in the auth/ldap handler (PR #1007)
+-   Fix backfill request to limit URI length, so that remotes don\'t reject the requests due to path length limits (PR #1012)
+-   Fix AS push code to not send duplicate events (PR #1025)
 
 Changes in synapse v0.17.0 (2016-08-08)
 =======================================
 
 This release contains significant security bug fixes regarding authenticating events received over federation. PLEASE UPGRADE.
 
-This release changes the LDAP configuration format in a backwards incompatible way, see PR \#843 for details.
+This release changes the LDAP configuration format in a backwards incompatible way, see PR #843 for details.
 
 Changes:
 
--   Add federation /version API (PR \#990)
--   Make psutil dependency optional (PR \#992)
+-   Add federation /version API (PR #990)
+-   Make psutil dependency optional (PR #992)
 
 Bug fixes:
 
--   Fix URL preview API to exclude HTML comments in description (PR \#988)
--   Fix error handling of remote joins (PR \#991)
+-   Fix URL preview API to exclude HTML comments in description (PR #988)
+-   Fix error handling of remote joins (PR #991)
 
 Changes in synapse v0.17.0-rc4 (2016-08-05)
 ===========================================
 
 Changes:
 
--   Change the way we summarize URLs when previewing (PR \#973)
--   Add new `/state_ids/` federation API (PR \#979)
--   Speed up processing of `/state/` response (PR \#986)
+-   Change the way we summarize URLs when previewing (PR #973)
+-   Add new `/state_ids/` federation API (PR #979)
+-   Speed up processing of `/state/` response (PR #986)
 
 Bug fixes:
 
--   Fix event persistence when event has already been partially persisted (PR \#975, \#983, \#985)
--   Fix port script to also copy across backfilled events (PR \#982)
+-   Fix event persistence when event has already been partially persisted (PR #975, #983, #985)
+-   Fix port script to also copy across backfilled events (PR #982)
 
 Changes in synapse v0.17.0-rc3 (2016-08-02)
 ===========================================
 
 Changes:
 
--   Forbid non-ASes from registering users whose names begin with \'\_\' (PR \#958)
--   Add some basic admin API docs (PR \#963)
+-   Forbid non-ASes from registering users whose names begin with \'\_\' (PR #958)
+-   Add some basic admin API docs (PR #963)
 
 Bug fixes:
 
--   Send the correct host header when fetching keys (PR \#941)
--   Fix joining a room that has missing auth events (PR \#964)
--   Fix various push bugs (PR \#966, \#970)
--   Fix adding emails on registration (PR \#968)
+-   Send the correct host header when fetching keys (PR #941)
+-   Fix joining a room that has missing auth events (PR #964)
+-   Fix various push bugs (PR #966, #970)
+-   Fix adding emails on registration (PR #968)
 
 Changes in synapse v0.17.0-rc2 (2016-08-02)
 ===========================================
@@ -1355,51 +1362,51 @@ Changes in synapse v0.17.0-rc2 (2016-08-02)
 Changes in synapse v0.17.0-rc1 (2016-07-28)
 ===========================================
 
-This release changes the LDAP configuration format in a backwards incompatible way, see PR \#843 for details.
+This release changes the LDAP configuration format in a backwards incompatible way, see PR #843 for details.
 
 Features:
 
--   Add purge\_media\_cache admin API (PR \#902)
--   Add deactivate account admin API (PR \#903)
--   Add optional pepper to password hashing (PR \#907, \#910 by KentShikama)
--   Add an admin option to shared secret registration (breaks backwards compat) (PR \#909)
--   Add purge local room history API (PR \#911, \#923, \#924)
--   Add requestToken endpoints (PR \#915)
--   Add an /account/deactivate endpoint (PR \#921)
--   Add filter param to /messages. Add \'contains\_url\' to filter. (PR \#922)
--   Add device\_id support to /login (PR \#929)
--   Add device\_id support to /v2/register flow. (PR \#937, \#942)
--   Add GET /devices endpoint (PR \#939, \#944)
--   Add GET /device/{deviceId} (PR \#943)
--   Add update and delete APIs for devices (PR \#949)
+-   Add purge\_media\_cache admin API (PR #902)
+-   Add deactivate account admin API (PR #903)
+-   Add optional pepper to password hashing (PR #907, #910 by KentShikama)
+-   Add an admin option to shared secret registration (breaks backwards compat) (PR #909)
+-   Add purge local room history API (PR #911, #923, #924)
+-   Add requestToken endpoints (PR #915)
+-   Add an /account/deactivate endpoint (PR #921)
+-   Add filter param to /messages. Add \'contains\_url\' to filter. (PR #922)
+-   Add device\_id support to /login (PR #929)
+-   Add device\_id support to /v2/register flow. (PR #937, #942)
+-   Add GET /devices endpoint (PR #939, #944)
+-   Add GET /device/{deviceId} (PR #943)
+-   Add update and delete APIs for devices (PR #949)
 
 Changes:
 
--   Rewrite LDAP Authentication against ldap3 (PR \#843 by mweinelt)
--   Linearize some federation endpoints based on (origin, room\_id) (PR \#879)
--   Remove the legacy v0 content upload API. (PR \#888)
--   Use similar naming we use in email notifs for push (PR \#894)
--   Optionally include password hash in createUser endpoint (PR \#905 by KentShikama)
--   Use a query that postgresql optimises better for get\_events\_around (PR \#906)
--   Fall back to \'username\' if \'user\' is not given for appservice registration. (PR \#927 by Half-Shot)
--   Add metrics for psutil derived memory usage (PR \#936)
--   Record device\_id in client\_ips (PR \#938)
--   Send the correct host header when fetching keys (PR \#941)
--   Log the hostname the reCAPTCHA was completed on (PR \#946)
--   Make the device id on e2e key upload optional (PR \#956)
--   Add r0.2.0 to the \"supported versions\" list (PR \#960)
--   Don\'t include name of room for invites in push (PR \#961)
+-   Rewrite LDAP Authentication against ldap3 (PR #843 by mweinelt)
+-   Linearize some federation endpoints based on (origin, room\_id) (PR #879)
+-   Remove the legacy v0 content upload API. (PR #888)
+-   Use similar naming we use in email notifs for push (PR #894)
+-   Optionally include password hash in createUser endpoint (PR #905 by KentShikama)
+-   Use a query that postgresql optimises better for get\_events\_around (PR #906)
+-   Fall back to \'username\' if \'user\' is not given for appservice registration. (PR #927 by Half-Shot)
+-   Add metrics for psutil derived memory usage (PR #936)
+-   Record device\_id in client\_ips (PR #938)
+-   Send the correct host header when fetching keys (PR #941)
+-   Log the hostname the reCAPTCHA was completed on (PR #946)
+-   Make the device id on e2e key upload optional (PR #956)
+-   Add r0.2.0 to the \"supported versions\" list (PR #960)
+-   Don\'t include name of room for invites in push (PR #961)
 
 Bug fixes:
 
--   Fix substitution failure in mail template (PR \#887)
--   Put most recent 20 messages in email notif (PR \#892)
--   Ensure that the guest user is in the database when upgrading accounts (PR \#914)
--   Fix various edge cases in auth handling (PR \#919)
--   Fix 500 ISE when sending alias event without a state\_key (PR \#925)
--   Fix bug where we stored rejections in the state\_group, persist all rejections (PR \#948)
--   Fix lack of check of if the user is banned when handling 3pid invites (PR \#952)
--   Fix a couple of bugs in the transaction and keyring code (PR \#954, \#955)
+-   Fix substitution failure in mail template (PR #887)
+-   Put most recent 20 messages in email notif (PR #892)
+-   Ensure that the guest user is in the database when upgrading accounts (PR #914)
+-   Fix various edge cases in auth handling (PR #919)
+-   Fix 500 ISE when sending alias event without a state\_key (PR #925)
+-   Fix bug where we stored rejections in the state\_group, persist all rejections (PR #948)
+-   Fix lack of check of if the user is banned when handling 3pid invites (PR #952)
+-   Fix a couple of bugs in the transaction and keyring code (PR #954, #955)
 
 Changes in synapse v0.16.1-r1 (2016-07-08)
 ==========================================
@@ -1413,13 +1420,13 @@ Changes in synapse v0.16.1 (2016-06-20)
 
 Bug fixes:
 
--   Fix assorted bugs in `/preview_url` (PR \#872)
--   Fix TypeError when setting unicode passwords (PR \#873)
+-   Fix assorted bugs in `/preview_url` (PR #872)
+-   Fix TypeError when setting unicode passwords (PR #873)
 
 Performance improvements:
 
--   Turn `use_frozen_events` off by default (PR \#877)
--   Disable responding with canonical json for federation (PR \#878)
+-   Turn `use_frozen_events` off by default (PR #877)
+-   Disable responding with canonical json for federation (PR #878)
 
 Changes in synapse v0.16.1-rc1 (2016-06-15)
 ===========================================
@@ -1428,20 +1435,20 @@ Features: None
 
 Changes:
 
--   Log requester for `/publicRoom` endpoints when possible (PR \#856)
--   502 on `/thumbnail` when can\'t connect to remote server (PR \#862)
--   Linearize fetching of gaps on incoming events (PR \#871)
+-   Log requester for `/publicRoom` endpoints when possible (PR #856)
+-   502 on `/thumbnail` when can\'t connect to remote server (PR #862)
+-   Linearize fetching of gaps on incoming events (PR #871)
 
 Bugs fixes:
 
--   Fix bug where rooms where marked as published by default (PR \#857)
--   Fix bug where joining room with an event with invalid sender (PR \#868)
--   Fix bug where backfilled events were sent down sync streams (PR \#869)
--   Fix bug where outgoing connections could wedge indefinitely, causing push notifications to be unreliable (PR \#870)
+-   Fix bug where rooms where marked as published by default (PR #857)
+-   Fix bug where joining room with an event with invalid sender (PR #868)
+-   Fix bug where backfilled events were sent down sync streams (PR #869)
+-   Fix bug where outgoing connections could wedge indefinitely, causing push notifications to be unreliable (PR #870)
 
 Performance improvements:
 
--   Improve `/publicRooms` performance(PR \#859)
+-   Improve `/publicRooms` performance(PR #859)
 
 Changes in synapse v0.16.0 (2016-06-09)
 =======================================
@@ -1450,31 +1457,31 @@ NB: As of v0.14 all AS config files must have an ID field.
 
 Bug fixes:
 
--   Don\'t make rooms published by default (PR \#857)
+-   Don\'t make rooms published by default (PR #857)
 
 Changes in synapse v0.16.0-rc2 (2016-06-08)
 ===========================================
 
 Features:
 
--   Add configuration option for tuning GC via `gc.set_threshold` (PR \#849)
+-   Add configuration option for tuning GC via `gc.set_threshold` (PR #849)
 
 Changes:
 
--   Record metrics about GC (PR \#771, \#847, \#852)
--   Add metric counter for number of persisted events (PR \#841)
+-   Record metrics about GC (PR #771, #847, #852)
+-   Add metric counter for number of persisted events (PR #841)
 
 Bug fixes:
 
--   Fix \'From\' header in email notifications (PR \#843)
--   Fix presence where timeouts were not being fired for the first 8h after restarts (PR \#842)
+-   Fix \'From\' header in email notifications (PR #843)
+-   Fix presence where timeouts were not being fired for the first 8h after restarts (PR #842)
 -   Fix bug where synapse sent malformed transactions to AS\'s when retrying transactions (Commits 310197b, 8437906)
 
 Performance improvements:
 
--   Remove event fetching from DB threads (PR \#835)
--   Change the way we cache events (PR \#836)
--   Add events to cache when we persist them (PR \#840)
+-   Remove event fetching from DB threads (PR #835)
+-   Change the way we cache events (PR #836)
+-   Add events to cache when we persist them (PR #840)
 
 Changes in synapse v0.16.0-rc1 (2016-06-03)
 ===========================================
@@ -1483,74 +1490,74 @@ Version 0.15 was not released. See v0.15.0-rc1 below for additional changes.
 
 Features:
 
--   Add email notifications for missed messages (PR \#759, \#786, \#799, \#810, \#815, \#821)
--   Add a `url_preview_ip_range_whitelist` config param (PR \#760)
--   Add /report endpoint (PR \#762)
--   Add basic ignore user API (PR \#763)
--   Add an openidish mechanism for proving that you own a given user\_id (PR \#765)
--   Allow clients to specify a server\_name to avoid \'No known servers\' (PR \#794)
--   Add secondary\_directory\_servers option to fetch room list from other servers (PR \#808, \#813)
+-   Add email notifications for missed messages (PR #759, #786, #799, #810, #815, #821)
+-   Add a `url_preview_ip_range_whitelist` config param (PR #760)
+-   Add /report endpoint (PR #762)
+-   Add basic ignore user API (PR #763)
+-   Add an openidish mechanism for proving that you own a given user\_id (PR #765)
+-   Allow clients to specify a server\_name to avoid \'No known servers\' (PR #794)
+-   Add secondary\_directory\_servers option to fetch room list from other servers (PR #808, #813)
 
 Changes:
 
--   Report per request metrics for all of the things using request\_handler (PR \#756)
--   Correctly handle `NULL` password hashes from the database (PR \#775)
--   Allow receipts for events we haven\'t seen in the db (PR \#784)
--   Make synctl read a cache factor from config file (PR \#785)
--   Increment badge count per missed convo, not per msg (PR \#793)
--   Special case m.room.third\_party\_invite event auth to match invites (PR \#814)
+-   Report per request metrics for all of the things using request\_handler (PR #756)
+-   Correctly handle `NULL` password hashes from the database (PR #775)
+-   Allow receipts for events we haven\'t seen in the db (PR #784)
+-   Make synctl read a cache factor from config file (PR #785)
+-   Increment badge count per missed convo, not per msg (PR #793)
+-   Special case m.room.third\_party\_invite event auth to match invites (PR #814)
 
 Bug fixes:
 
--   Fix typo in event\_auth servlet path (PR \#757)
--   Fix password reset (PR \#758)
+-   Fix typo in event\_auth servlet path (PR #757)
+-   Fix password reset (PR #758)
 
 Performance improvements:
 
--   Reduce database inserts when sending transactions (PR \#767)
--   Queue events by room for persistence (PR \#768)
--   Add cache to `get_user_by_id` (PR \#772)
--   Add and use `get_domain_from_id` (PR \#773)
--   Use tree cache for `get_linearized_receipts_for_room` (PR \#779)
--   Remove unused indices (PR \#782)
--   Add caches to `bulk_get_push_rules*` (PR \#804)
--   Cache `get_event_reference_hashes` (PR \#806)
--   Add `get_users_with_read_receipts_in_room` cache (PR \#809)
--   Use state to calculate `get_users_in_room` (PR \#811)
--   Load push rules in storage layer so that they get cached (PR \#825)
--   Make `get_joined_hosts_for_room` use get\_users\_in\_room (PR \#828)
--   Poke notifier on next reactor tick (PR \#829)
--   Change CacheMetrics to be quicker (PR \#830)
+-   Reduce database inserts when sending transactions (PR #767)
+-   Queue events by room for persistence (PR #768)
+-   Add cache to `get_user_by_id` (PR #772)
+-   Add and use `get_domain_from_id` (PR #773)
+-   Use tree cache for `get_linearized_receipts_for_room` (PR #779)
+-   Remove unused indices (PR #782)
+-   Add caches to `bulk_get_push_rules*` (PR #804)
+-   Cache `get_event_reference_hashes` (PR #806)
+-   Add `get_users_with_read_receipts_in_room` cache (PR #809)
+-   Use state to calculate `get_users_in_room` (PR #811)
+-   Load push rules in storage layer so that they get cached (PR #825)
+-   Make `get_joined_hosts_for_room` use get\_users\_in\_room (PR #828)
+-   Poke notifier on next reactor tick (PR #829)
+-   Change CacheMetrics to be quicker (PR #830)
 
 Changes in synapse v0.15.0-rc1 (2016-04-26)
 ===========================================
 
 Features:
 
--   Add login support for Javascript Web Tokens, thanks to Niklas Riekenbrauck (PR \#671,\#687)
--   Add URL previewing support (PR \#688)
--   Add login support for LDAP, thanks to Christoph Witzany (PR \#701)
--   Add GET endpoint for pushers (PR \#716)
+-   Add login support for Javascript Web Tokens, thanks to Niklas Riekenbrauck (PR #671,\#687)
+-   Add URL previewing support (PR #688)
+-   Add login support for LDAP, thanks to Christoph Witzany (PR #701)
+-   Add GET endpoint for pushers (PR #716)
 
 Changes:
 
--   Never notify for member events (PR \#667)
--   Deduplicate identical `/sync` requests (PR \#668)
--   Require user to have left room to forget room (PR \#673)
--   Use DNS cache if within TTL (PR \#677)
--   Let users see their own leave events (PR \#699)
--   Deduplicate membership changes (PR \#700)
--   Increase performance of pusher code (PR \#705)
--   Respond with error status 504 if failed to talk to remote server (PR \#731)
--   Increase search performance on postgres (PR \#745)
+-   Never notify for member events (PR #667)
+-   Deduplicate identical `/sync` requests (PR #668)
+-   Require user to have left room to forget room (PR #673)
+-   Use DNS cache if within TTL (PR #677)
+-   Let users see their own leave events (PR #699)
+-   Deduplicate membership changes (PR #700)
+-   Increase performance of pusher code (PR #705)
+-   Respond with error status 504 if failed to talk to remote server (PR #731)
+-   Increase search performance on postgres (PR #745)
 
 Bug fixes:
 
--   Fix bug where disabling all notifications still resulted in push (PR \#678)
--   Fix bug where users couldn\'t reject remote invites if remote refused (PR \#691)
--   Fix bug where synapse attempted to backfill from itself (PR \#693)
--   Fix bug where profile information was not correctly added when joining remote rooms (PR \#703)
--   Fix bug where register API required incorrect key name for AS registration (PR \#727)
+-   Fix bug where disabling all notifications still resulted in push (PR #678)
+-   Fix bug where users couldn\'t reject remote invites if remote refused (PR #691)
+-   Fix bug where synapse attempted to backfill from itself (PR #693)
+-   Fix bug where profile information was not correctly added when joining remote rooms (PR #703)
+-   Fix bug where register API required incorrect key name for AS registration (PR #727)
 
 Changes in synapse v0.14.0 (2016-03-30)
 =======================================
@@ -1562,58 +1569,58 @@ Changes in synapse v0.14.0-rc2 (2016-03-23)
 
 Features:
 
--   Add published room list API (PR \#657)
+-   Add published room list API (PR #657)
 
 Changes:
 
--   Change various caches to consume less memory (PR \#656, \#658, \#660, \#662, \#663, \#665)
--   Allow rooms to be published without requiring an alias (PR \#664)
+-   Change various caches to consume less memory (PR #656, #658, #660, #662, #663, #665)
+-   Allow rooms to be published without requiring an alias (PR #664)
 -   Intern common strings in caches to reduce memory footprint (\#666)
 
 Bug fixes:
 
--   Fix reject invites over federation (PR \#646)
--   Fix bug where registration was not idempotent (PR \#649)
--   Update aliases event after deleting aliases (PR \#652)
--   Fix unread notification count, which was sometimes wrong (PR \#661)
+-   Fix reject invites over federation (PR #646)
+-   Fix bug where registration was not idempotent (PR #649)
+-   Update aliases event after deleting aliases (PR #652)
+-   Fix unread notification count, which was sometimes wrong (PR #661)
 
 Changes in synapse v0.14.0-rc1 (2016-03-14)
 ===========================================
 
 Features:
 
--   Add event\_id to response to state event PUT (PR \#581)
--   Allow guest users access to messages in rooms they have joined (PR \#587)
--   Add config for what state is included in a room invite (PR \#598)
--   Send the inviter\'s member event in room invite state (PR \#607)
--   Add error codes for malformed/bad JSON in /login (PR \#608)
--   Add support for changing the actions for default rules (PR \#609)
--   Add environment variable SYNAPSE\_CACHE\_FACTOR, default it to 0.1 (PR \#612)
--   Add ability for alias creators to delete aliases (PR \#614)
--   Add profile information to invites (PR \#624)
+-   Add event\_id to response to state event PUT (PR #581)
+-   Allow guest users access to messages in rooms they have joined (PR #587)
+-   Add config for what state is included in a room invite (PR #598)
+-   Send the inviter\'s member event in room invite state (PR #607)
+-   Add error codes for malformed/bad JSON in /login (PR #608)
+-   Add support for changing the actions for default rules (PR #609)
+-   Add environment variable SYNAPSE\_CACHE\_FACTOR, default it to 0.1 (PR #612)
+-   Add ability for alias creators to delete aliases (PR #614)
+-   Add profile information to invites (PR #624)
 
 Changes:
 
--   Enforce user\_id exclusivity for AS registrations (PR \#572)
--   Make adding push rules idempotent (PR \#587)
--   Improve presence performance (PR \#582, \#586)
--   Change presence semantics for `last_active_ago` (PR \#582, \#586)
--   Don\'t allow `m.room.create` to be changed (PR \#596)
--   Add 800x600 to default list of valid thumbnail sizes (PR \#616)
--   Always include kicks and bans in full /sync (PR \#625)
--   Send history visibility on boundary changes (PR \#626)
--   Register endpoint now returns a refresh\_token (PR \#637)
+-   Enforce user\_id exclusivity for AS registrations (PR #572)
+-   Make adding push rules idempotent (PR #587)
+-   Improve presence performance (PR #582, #586)
+-   Change presence semantics for `last_active_ago` (PR #582, #586)
+-   Don\'t allow `m.room.create` to be changed (PR #596)
+-   Add 800x600 to default list of valid thumbnail sizes (PR #616)
+-   Always include kicks and bans in full /sync (PR #625)
+-   Send history visibility on boundary changes (PR #626)
+-   Register endpoint now returns a refresh\_token (PR #637)
 
 Bug fixes:
 
--   Fix bug where we returned incorrect state in /sync (PR \#573)
--   Always return a JSON object from push rule API (PR \#606)
--   Fix bug where registering without a user id sometimes failed (PR \#610)
--   Report size of ExpiringCache in cache size metrics (PR \#611)
--   Fix rejection of invites to empty rooms (PR \#615)
--   Fix usage of `bcrypt` to not use `checkpw` (PR \#619)
--   Pin `pysaml2` dependency (PR \#634)
--   Fix bug in `/sync` where timeline order was incorrect for backfilled events (PR \#635)
+-   Fix bug where we returned incorrect state in /sync (PR #573)
+-   Always return a JSON object from push rule API (PR #606)
+-   Fix bug where registering without a user id sometimes failed (PR #610)
+-   Report size of ExpiringCache in cache size metrics (PR #611)
+-   Fix rejection of invites to empty rooms (PR #615)
+-   Fix usage of `bcrypt` to not use `checkpw` (PR #619)
+-   Pin `pysaml2` dependency (PR #634)
+-   Fix bug in `/sync` where timeline order was incorrect for backfilled events (PR #635)
 
 Changes in synapse v0.13.3 (2016-02-11)
 =======================================
@@ -1623,7 +1630,7 @@ Changes in synapse v0.13.3 (2016-02-11)
 Changes in synapse v0.13.2 (2016-02-11)
 =======================================
 
--   Fix bug where `/events` would fail to skip some events if there had been more events than the limit specified since the last request (PR \#570)
+-   Fix bug where `/events` would fail to skip some events if there had been more events than the limit specified since the last request (PR #570)
 
 Changes in synapse v0.13.1 (2016-02-10)
 =======================================
@@ -1637,176 +1644,176 @@ This version includes an upgrade of the schema, specifically adding an index to
 
 Changes:
 
--   Improve general performance (PR \#540, \#543. \#544, \#54, \#549, \#567)
--   Change guest user ids to be incrementing integers (PR \#550)
--   Improve performance of public room list API (PR \#552)
--   Change profile API to omit keys rather than return null (PR \#557)
--   Add `/media/r0` endpoint prefix, which is equivalent to `/media/v1/` (PR \#595)
+-   Improve general performance (PR #540, #543. \#544, #54, #549, #567)
+-   Change guest user ids to be incrementing integers (PR #550)
+-   Improve performance of public room list API (PR #552)
+-   Change profile API to omit keys rather than return null (PR #557)
+-   Add `/media/r0` endpoint prefix, which is equivalent to `/media/v1/` (PR #595)
 
 Bug fixes:
 
--   Fix bug with upgrading guest accounts where it would fail if you opened the registration email on a different device (PR \#547)
--   Fix bug where unread count could be wrong (PR \#568)
+-   Fix bug with upgrading guest accounts where it would fail if you opened the registration email on a different device (PR #547)
+-   Fix bug where unread count could be wrong (PR #568)
 
 Changes in synapse v0.12.1-rc1 (2016-01-29)
 ===========================================
 
 Features:
 
--   Add unread notification counts in `/sync` (PR \#456)
--   Add support for inviting 3pids in `/createRoom` (PR \#460)
--   Add ability for guest accounts to upgrade (PR \#462)
--   Add `/versions` API (PR \#468)
--   Add `event` to `/context` API (PR \#492)
--   Add specific error code for invalid user names in `/register` (PR \#499)
--   Add support for push badge counts (PR \#507)
--   Add support for non-guest users to peek in rooms using `/events` (PR \#510)
+-   Add unread notification counts in `/sync` (PR #456)
+-   Add support for inviting 3pids in `/createRoom` (PR #460)
+-   Add ability for guest accounts to upgrade (PR #462)
+-   Add `/versions` API (PR #468)
+-   Add `event` to `/context` API (PR #492)
+-   Add specific error code for invalid user names in `/register` (PR #499)
+-   Add support for push badge counts (PR #507)
+-   Add support for non-guest users to peek in rooms using `/events` (PR #510)
 
 Changes:
 
--   Change `/sync` so that guest users only get rooms they\'ve joined (PR \#469)
--   Change to require unbanning before other membership changes (PR \#501)
--   Change default push rules to notify for all messages (PR \#486)
--   Change default push rules to not notify on membership changes (PR \#514)
--   Change default push rules in one to one rooms to only notify for events that are messages (PR \#529)
--   Change `/sync` to reject requests with a `from` query param (PR \#512)
--   Change server manhole to use SSH rather than telnet (PR \#473)
--   Change server to require AS users to be registered before use (PR \#487)
--   Change server not to start when ASes are invalidly configured (PR \#494)
--   Change server to require ID and `as_token` to be unique for AS\'s (PR \#496)
--   Change maximum pagination limit to 1000 (PR \#497)
+-   Change `/sync` so that guest users only get rooms they\'ve joined (PR #469)
+-   Change to require unbanning before other membership changes (PR #501)
+-   Change default push rules to notify for all messages (PR #486)
+-   Change default push rules to not notify on membership changes (PR #514)
+-   Change default push rules in one to one rooms to only notify for events that are messages (PR #529)
+-   Change `/sync` to reject requests with a `from` query param (PR #512)
+-   Change server manhole to use SSH rather than telnet (PR #473)
+-   Change server to require AS users to be registered before use (PR #487)
+-   Change server not to start when ASes are invalidly configured (PR #494)
+-   Change server to require ID and `as_token` to be unique for AS\'s (PR #496)
+-   Change maximum pagination limit to 1000 (PR #497)
 
 Bug fixes:
 
--   Fix bug where `/sync` didn\'t return when something under the leave key changed (PR \#461)
--   Fix bug where we returned smaller rather than larger than requested thumbnails when `method=crop` (PR \#464)
--   Fix thumbnails API to only return cropped thumbnails when asking for a cropped thumbnail (PR \#475)
--   Fix bug where we occasionally still logged access tokens (PR \#477)
--   Fix bug where `/events` would always return immediately for guest users (PR \#480)
--   Fix bug where `/sync` unexpectedly returned old left rooms (PR \#481)
--   Fix enabling and disabling push rules (PR \#498)
--   Fix bug where `/register` returned 500 when given unicode username (PR \#513)
+-   Fix bug where `/sync` didn\'t return when something under the leave key changed (PR #461)
+-   Fix bug where we returned smaller rather than larger than requested thumbnails when `method=crop` (PR #464)
+-   Fix thumbnails API to only return cropped thumbnails when asking for a cropped thumbnail (PR #475)
+-   Fix bug where we occasionally still logged access tokens (PR #477)
+-   Fix bug where `/events` would always return immediately for guest users (PR #480)
+-   Fix bug where `/sync` unexpectedly returned old left rooms (PR #481)
+-   Fix enabling and disabling push rules (PR #498)
+-   Fix bug where `/register` returned 500 when given unicode username (PR #513)
 
 Changes in synapse v0.12.0 (2016-01-04)
 =======================================
 
--   Expose `/login` under `r0` (PR \#459)
+-   Expose `/login` under `r0` (PR #459)
 
 Changes in synapse v0.12.0-rc3 (2015-12-23)
 ===========================================
 
--   Allow guest accounts access to `/sync` (PR \#455)
--   Allow filters to include/exclude rooms at the room level rather than just from the components of the sync for each room. (PR \#454)
--   Include urls for room avatars in the response to `/publicRooms` (PR \#453)
--   Don\'t set a identicon as the avatar for a user when they register (PR \#450)
--   Add a `display_name` to third-party invites (PR \#449)
--   Send more information to the identity server for third-party invites so that it can send richer messages to the invitee (PR \#446)
--   Cache the responses to `/initialSync` for 5 minutes. If a client retries a request to `/initialSync` before the a response was computed to the first request then the same response is used for both requests (PR \#457)
--   Fix a bug where synapse would always request the signing keys of remote servers even when the key was cached locally (PR \#452)
--   Fix 500 when pagination search results (PR \#447)
--   Fix a bug where synapse was leaking raw email address in third-party invites (PR \#448)
+-   Allow guest accounts access to `/sync` (PR #455)
+-   Allow filters to include/exclude rooms at the room level rather than just from the components of the sync for each room. (PR #454)
+-   Include urls for room avatars in the response to `/publicRooms` (PR #453)
+-   Don\'t set a identicon as the avatar for a user when they register (PR #450)
+-   Add a `display_name` to third-party invites (PR #449)
+-   Send more information to the identity server for third-party invites so that it can send richer messages to the invitee (PR #446)
+-   Cache the responses to `/initialSync` for 5 minutes. If a client retries a request to `/initialSync` before the a response was computed to the first request then the same response is used for both requests (PR #457)
+-   Fix a bug where synapse would always request the signing keys of remote servers even when the key was cached locally (PR #452)
+-   Fix 500 when pagination search results (PR #447)
+-   Fix a bug where synapse was leaking raw email address in third-party invites (PR #448)
 
 Changes in synapse v0.12.0-rc2 (2015-12-14)
 ===========================================
 
--   Add caches for whether rooms have been forgotten by a user (PR \#434)
--   Remove instructions to use `--process-dependency-link` since all of the dependencies of synapse are on PyPI (PR \#436)
--   Parallelise the processing of `/sync` requests (PR \#437)
--   Fix race updating presence in `/events` (PR \#444)
--   Fix bug back-populating search results (PR \#441)
--   Fix bug calculating state in `/sync` requests (PR \#442)
+-   Add caches for whether rooms have been forgotten by a user (PR #434)
+-   Remove instructions to use `--process-dependency-link` since all of the dependencies of synapse are on PyPI (PR #436)
+-   Parallelise the processing of `/sync` requests (PR #437)
+-   Fix race updating presence in `/events` (PR #444)
+-   Fix bug back-populating search results (PR #441)
+-   Fix bug calculating state in `/sync` requests (PR #442)
 
 Changes in synapse v0.12.0-rc1 (2015-12-10)
 ===========================================
 
--   Host the client APIs released as r0 by <https://matrix.org/docs/spec/r0.0.0/client_server.html> on paths prefixed by `/_matrix/client/r0`. (PR \#430, PR \#415, PR \#400)
+-   Host the client APIs released as r0 by <https://matrix.org/docs/spec/r0.0.0/client_server.html> on paths prefixed by `/_matrix/client/r0`. (PR #430, PR #415, PR #400)
 -   Updates the client APIs to match r0 of the matrix specification.
-    -   All APIs return events in the new event format, old APIs also include the fields needed to parse the event using the old format for compatibility. (PR \#402)
-    -   Search results are now given as a JSON array rather than a JSON object (PR \#405)
-    -   Miscellaneous changes to search (PR \#403, PR \#406, PR \#412)
-    -   Filter JSON objects may now be passed as query parameters to `/sync` (PR \#431)
-    -   Fix implementation of `/admin/whois` (PR \#418)
-    -   Only include the rooms that user has left in `/sync` if the client requests them in the filter (PR \#423)
-    -   Don\'t push for `m.room.message` by default (PR \#411)
-    -   Add API for setting per account user data (PR \#392)
-    -   Allow users to forget rooms (PR \#385)
+    -   All APIs return events in the new event format, old APIs also include the fields needed to parse the event using the old format for compatibility. (PR #402)
+    -   Search results are now given as a JSON array rather than a JSON object (PR #405)
+    -   Miscellaneous changes to search (PR #403, PR #406, PR #412)
+    -   Filter JSON objects may now be passed as query parameters to `/sync` (PR #431)
+    -   Fix implementation of `/admin/whois` (PR #418)
+    -   Only include the rooms that user has left in `/sync` if the client requests them in the filter (PR #423)
+    -   Don\'t push for `m.room.message` by default (PR #411)
+    -   Add API for setting per account user data (PR #392)
+    -   Allow users to forget rooms (PR #385)
 -   Performance improvements and monitoring:
-    -   Add per-request counters for CPU time spent on the main python thread. (PR \#421, PR \#420)
-    -   Add per-request counters for time spent in the database (PR \#429)
-    -   Make state updates in the C+S API idempotent (PR \#416)
-    -   Only fire `user_joined_room` if the user has actually joined. (PR \#410)
-    -   Reuse a single http client, rather than creating new ones (PR \#413)
--   Fixed a bug upgrading from older versions of synapse on postgresql (PR \#417)
+    -   Add per-request counters for CPU time spent on the main python thread. (PR #421, PR #420)
+    -   Add per-request counters for time spent in the database (PR #429)
+    -   Make state updates in the C+S API idempotent (PR #416)
+    -   Only fire `user_joined_room` if the user has actually joined. (PR #410)
+    -   Reuse a single http client, rather than creating new ones (PR #413)
+-   Fixed a bug upgrading from older versions of synapse on postgresql (PR #417)
 
 Changes in synapse v0.11.1 (2015-11-20)
 =======================================
 
--   Add extra options to search API (PR \#394)
--   Fix bug where we did not correctly cap federation retry timers. This meant it could take several hours for servers to start talking to ressurected servers, even when they were receiving traffic from them (PR \#393)
--   Don\'t advertise login token flow unless CAS is enabled. This caused issues where some clients would always use the fallback API if they did not recognize all login flows (PR \#391)
--   Change /v2 sync API to rename `private_user_data` to `account_data` (PR \#386)
--   Change /v2 sync API to remove the `event_map` and rename keys in `rooms` object (PR \#389)
+-   Add extra options to search API (PR #394)
+-   Fix bug where we did not correctly cap federation retry timers. This meant it could take several hours for servers to start talking to ressurected servers, even when they were receiving traffic from them (PR #393)
+-   Don\'t advertise login token flow unless CAS is enabled. This caused issues where some clients would always use the fallback API if they did not recognize all login flows (PR #391)
+-   Change /v2 sync API to rename `private_user_data` to `account_data` (PR #386)
+-   Change /v2 sync API to remove the `event_map` and rename keys in `rooms` object (PR #389)
 
 Changes in synapse v0.11.0-r2 (2015-11-19)
 ==========================================
 
--   Fix bug in database port script (PR \#387)
+-   Fix bug in database port script (PR #387)
 
 Changes in synapse v0.11.0-r1 (2015-11-18)
 ==========================================
 
--   Retry and fail federation requests more aggressively for requests that block client side requests (PR \#384)
+-   Retry and fail federation requests more aggressively for requests that block client side requests (PR #384)
 
 Changes in synapse v0.11.0 (2015-11-17)
 =======================================
 
--   Change CAS login API (PR \#349)
+-   Change CAS login API (PR #349)
 
 Changes in synapse v0.11.0-rc2 (2015-11-13)
 ===========================================
 
--   Various changes to /sync API response format (PR \#373)
--   Fix regression when setting display name in newly joined room over federation (PR \#368)
--   Fix problem where /search was slow when using SQLite (PR \#366)
+-   Various changes to /sync API response format (PR #373)
+-   Fix regression when setting display name in newly joined room over federation (PR #368)
+-   Fix problem where /search was slow when using SQLite (PR #366)
 
 Changes in synapse v0.11.0-rc1 (2015-11-11)
 ===========================================
 
--   Add Search API (PR \#307, \#324, \#327, \#336, \#350, \#359)
--   Add \'archived\' state to v2 /sync API (PR \#316)
--   Add ability to reject invites (PR \#317)
--   Add config option to disable password login (PR \#322)
--   Add the login fallback API (PR \#330)
--   Add room context API (PR \#334)
--   Add room tagging support (PR \#335)
--   Update v2 /sync API to match spec (PR \#305, \#316, \#321, \#332, \#337, \#341)
--   Change retry schedule for application services (PR \#320)
--   Change retry schedule for remote servers (PR \#340)
--   Fix bug where we hosted static content in the incorrect place (PR \#329)
--   Fix bug where we didn\'t increment retry interval for remote servers (PR \#343)
+-   Add Search API (PR #307, #324, #327, #336, #350, #359)
+-   Add \'archived\' state to v2 /sync API (PR #316)
+-   Add ability to reject invites (PR #317)
+-   Add config option to disable password login (PR #322)
+-   Add the login fallback API (PR #330)
+-   Add room context API (PR #334)
+-   Add room tagging support (PR #335)
+-   Update v2 /sync API to match spec (PR #305, #316, #321, #332, #337, #341)
+-   Change retry schedule for application services (PR #320)
+-   Change retry schedule for remote servers (PR #340)
+-   Fix bug where we hosted static content in the incorrect place (PR #329)
+-   Fix bug where we didn\'t increment retry interval for remote servers (PR #343)
 
 Changes in synapse v0.10.1-rc1 (2015-10-15)
 ===========================================
 
--   Add support for CAS, thanks to Steven Hammerton (PR \#295, \#296)
--   Add support for using macaroons for `access_token` (PR \#256, \#229)
--   Add support for `m.room.canonical_alias` (PR \#287)
--   Add support for viewing the history of rooms that they have left. (PR \#276, \#294)
--   Add support for refresh tokens (PR \#240)
--   Add flag on creation which disables federation of the room (PR \#279)
--   Add some room state to invites. (PR \#275)
--   Atomically persist events when joining a room over federation (PR \#283)
--   Change default history visibility for private rooms (PR \#271)
--   Allow users to redact their own sent events (PR \#262)
--   Use tox for tests (PR \#247)
--   Split up syutil into separate libraries (PR \#243)
+-   Add support for CAS, thanks to Steven Hammerton (PR #295, #296)
+-   Add support for using macaroons for `access_token` (PR #256, #229)
+-   Add support for `m.room.canonical_alias` (PR #287)
+-   Add support for viewing the history of rooms that they have left. (PR #276, #294)
+-   Add support for refresh tokens (PR #240)
+-   Add flag on creation which disables federation of the room (PR #279)
+-   Add some room state to invites. (PR #275)
+-   Atomically persist events when joining a room over federation (PR #283)
+-   Change default history visibility for private rooms (PR #271)
+-   Allow users to redact their own sent events (PR #262)
+-   Use tox for tests (PR #247)
+-   Split up syutil into separate libraries (PR #243)
 
 Changes in synapse v0.10.0-r2 (2015-09-16)
 ==========================================
 
 -   Fix bug where we always fetched remote server signing keys instead of using ones in our cache.
 -   Fix adding threepids to an existing account.
--   Fix bug with invinting over federation where remote server was already in the room. (PR \#281, SYN-392)
+-   Fix bug with invinting over federation where remote server was already in the room. (PR #281, SYN-392)
 
 Changes in synapse v0.10.0-r1 (2015-09-08)
 ==========================================
@@ -1832,20 +1839,20 @@ Changes in synapse v0.10.0-rc5 (2015-08-27)
 Changes in synapse v0.10.0-rc4 (2015-08-27)
 ===========================================
 
--   Allow UTF-8 filenames for upload. (PR \#259)
+-   Allow UTF-8 filenames for upload. (PR #259)
 
 Changes in synapse v0.10.0-rc3 (2015-08-25)
 ===========================================
 
--   Add `--keys-directory` config option to specify where files such as certs and signing keys should be stored in, when using `--generate-config` or `--generate-keys`. (PR \#250)
--   Allow `--config-path` to specify a directory, causing synapse to use all \*.yaml files in the directory as config files. (PR \#249)
--   Add `web_client_location` config option to specify static files to be hosted by synapse under `/_matrix/client`. (PR \#245)
+-   Add `--keys-directory` config option to specify where files such as certs and signing keys should be stored in, when using `--generate-config` or `--generate-keys`. (PR #250)
+-   Allow `--config-path` to specify a directory, causing synapse to use all \*.yaml files in the directory as config files. (PR #249)
+-   Add `web_client_location` config option to specify static files to be hosted by synapse under `/_matrix/client`. (PR #245)
 -   Add helper utility to synapse to read and parse the config files and extract the value of a given key. For example:
 
         $ python -m synapse.config read server_name -c homeserver.yaml
         localhost
 
-    (PR \#246)
+    (PR #246)
 
 Changes in synapse v0.10.0-rc2 (2015-08-24)
 ===========================================
@@ -1860,37 +1867,37 @@ Also see v0.9.4-rc1 changelog, which has been amalgamated into this release.
 
 General:
 
--   Upgrade to Twisted 15 (PR \#173)
--   Add support for serving and fetching encryption keys over federation. (PR \#208)
--   Add support for logging in with email address (PR \#234)
--   Add support for new `m.room.canonical_alias` event. (PR \#233)
+-   Upgrade to Twisted 15 (PR #173)
+-   Add support for serving and fetching encryption keys over federation. (PR #208)
+-   Add support for logging in with email address (PR #234)
+-   Add support for new `m.room.canonical_alias` event. (PR #233)
 -   Change synapse to treat user IDs case insensitively during registration and login. (If two users already exist with case insensitive matching user ids, synapse will continue to require them to specify their user ids exactly.)
--   Error if a user tries to register with an email already in use. (PR \#211)
--   Add extra and improve existing caches (PR \#212, \#219, \#226, \#228)
--   Batch various storage request (PR \#226, \#228)
--   Fix bug where we didn\'t correctly log the entity that triggered the request if the request came in via an application service (PR \#230)
--   Fix bug where we needlessly regenerated the full list of rooms an AS is interested in. (PR \#232)
--   Add support for AS\'s to use v2\_alpha registration API (PR \#210)
+-   Error if a user tries to register with an email already in use. (PR #211)
+-   Add extra and improve existing caches (PR #212, #219, #226, #228)
+-   Batch various storage request (PR #226, #228)
+-   Fix bug where we didn\'t correctly log the entity that triggered the request if the request came in via an application service (PR #230)
+-   Fix bug where we needlessly regenerated the full list of rooms an AS is interested in. (PR #232)
+-   Add support for AS\'s to use v2\_alpha registration API (PR #210)
 
 Configuration:
 
--   Add `--generate-keys` that will generate any missing cert and key files in the configuration files. This is equivalent to running `--generate-config` on an existing configuration file. (PR \#220)
--   `--generate-config` now no longer requires a `--server-name` parameter when used on existing configuration files. (PR \#220)
--   Add `--print-pidfile` flag that controls the printing of the pid to stdout of the demonised process. (PR \#213)
+-   Add `--generate-keys` that will generate any missing cert and key files in the configuration files. This is equivalent to running `--generate-config` on an existing configuration file. (PR #220)
+-   `--generate-config` now no longer requires a `--server-name` parameter when used on existing configuration files. (PR #220)
+-   Add `--print-pidfile` flag that controls the printing of the pid to stdout of the demonised process. (PR #213)
 
 Media Repository:
 
--   Fix bug where we picked a lower resolution image than requested. (PR \#205)
--   Add support for specifying if a the media repository should dynamically thumbnail images or not. (PR \#206)
+-   Fix bug where we picked a lower resolution image than requested. (PR #205)
+-   Add support for specifying if a the media repository should dynamically thumbnail images or not. (PR #206)
 
 Metrics:
 
--   Add statistics from the reactor to the metrics API. (PR \#224, \#225)
+-   Add statistics from the reactor to the metrics API. (PR #224, #225)
 
 Demo Homeservers:
 
--   Fix starting the demo homeservers without rate-limiting enabled. (PR \#182)
--   Fix enabling registration on demo homeservers (PR \#223)
+-   Fix starting the demo homeservers without rate-limiting enabled. (PR #182)
+-   Fix enabling registration on demo homeservers (PR #223)
 
 Changes in synapse v0.9.4-rc1 (2015-07-21)
 ==========================================
@@ -1898,13 +1905,13 @@ Changes in synapse v0.9.4-rc1 (2015-07-21)
 General:
 
 -   Add basic implementation of receipts. (SPEC-99)
--   Add support for configuration presets in room creation API. (PR \#203)
+-   Add support for configuration presets in room creation API. (PR #203)
 -   Add auth event that limits the visibility of history for new users. (SPEC-134)
--   Add SAML2 login/registration support. (PR \#201. Thanks Muthu Subramanian!)
--   Add client side key management APIs for end to end encryption. (PR \#198)
+-   Add SAML2 login/registration support. (PR #201. Thanks Muthu Subramanian!)
+-   Add client side key management APIs for end to end encryption. (PR #198)
 -   Change power level semantics so that you cannot kick, ban or change power levels of users that have equal or greater power level than you. (SYN-192)
--   Improve performance by bulk inserting events where possible. (PR \#193)
--   Improve performance by bulk verifying signatures where possible. (PR \#194)
+-   Improve performance by bulk inserting events where possible. (PR #193)
+-   Improve performance by bulk verifying signatures where possible. (PR #194)
 
 Configuration:
 
@@ -2401,7 +2408,7 @@ Presence has been reenabled in this release.
 
 Homeserver:
 
-:   -   
+:   -
 
         Update client to server API, including:
 

From 87086ac69df5a795d0979801692b01c514937775 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Thu, 19 Jul 2018 21:42:40 +1000
Subject: [PATCH 073/830] fixes

---
 MANIFEST.in    | 1 +
 pyproject.toml | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/MANIFEST.in b/MANIFEST.in
index 97f57f443fb6..7076b608d4c7 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -2,6 +2,7 @@ include synctl
 include LICENSE
 include VERSION
 include *.rst
+include *.md
 include demo/README
 include demo/demo.tls.dh
 include demo/*.py
diff --git a/pyproject.toml b/pyproject.toml
index 1453ddd53003..f4d6f0c6bbfa 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -2,7 +2,7 @@
     package = "synapse"
     filename = "CHANGES.md"
     directory = "changelog.d"
-    issue_format = "[\#{issue}](https://github.com/matrix-org/synapse/issues/{issue}>)"
+    issue_format = "[\\#{issue}](https://github.com/matrix-org/synapse/issues/{issue}>)"
 
     [[tool.towncrier.type]]
         directory = "feature"

From 924eb34d9428a4163a03249abbb6f40d4baa29c6 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Thu, 19 Jul 2018 18:32:02 +0100
Subject: [PATCH 074/830] add a filtered_types param to limit filtering to
 specific types

---
 synapse/handlers/sync.py |  65 ++++++++++++----------
 synapse/storage/state.py | 113 +++++++++++++++++++++------------------
 2 files changed, 96 insertions(+), 82 deletions(-)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 0c21ac2c7781..cb711b87588c 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -417,38 +417,44 @@ def _load_filtered_recents(self, room_id, sync_config, now_token,
         ))
 
     @defer.inlineCallbacks
-    def get_state_after_event(self, event, types=None):
+    def get_state_after_event(self, event, types=None, filtered_types=None):
         """
         Get the room state after the given event
 
         Args:
             event(synapse.events.EventBase): event of interest
-            types(list[(str|None, str|None)]|None): List of (type, state_key) tuples
+            types(list[(str, str|None)]|None): List of (type, state_key) tuples
                 which are used to filter the state fetched. If `state_key` is None,
-                all events are returned of the given type.  Presence of type of `None`
-                indicates that types not in the list should not be filtered out.
+                all events are returned of the given type.
                 May be None, which matches any key.
+            filtered_types(list[str]|None): Only apply filtering via `types` to this
+                list of event types.  Other types of events are returned unfiltered.
+                If None, `types` filtering is applied to all events.
+
         Returns:
             A Deferred map from ((type, state_key)->Event)
         """
-        state_ids = yield self.store.get_state_ids_for_event(event.event_id, types)
+        state_ids = yield self.store.get_state_ids_for_event(
+            event.event_id, types, filtered_types=filtered_types
+        )
         if event.is_state():
             state_ids = state_ids.copy()
             state_ids[(event.type, event.state_key)] = event.event_id
         defer.returnValue(state_ids)
 
     @defer.inlineCallbacks
-    def get_state_at(self, room_id, stream_position, types=None):
+    def get_state_at(self, room_id, stream_position, types=None, filtered_types=None):
         """ Get the room state at a particular stream position
 
         Args:
             room_id(str): room for which to get state
             stream_position(StreamToken): point at which to get state
-            types(list[(str|None, str|None)]|None): List of (type, state_key) tuples
+            types(list[(str, str|None)]|None): List of (type, state_key) tuples
                 which are used to filter the state fetched. If `state_key` is None,
-                all events are returned of the given type.  Presence of type of `None`
-                indicates that types not in the list should not be filtered out.
-                May be None, which matches any key.
+                all events are returned of the given type.
+            filtered_types(list[str]|None): Only apply filtering via `types` to this
+                list of event types.  Other types of events are returned unfiltered.
+                If None, `types` filtering is applied to all events.
 
         Returns:
             A Deferred map from ((type, state_key)->Event)
@@ -463,7 +469,9 @@ def get_state_at(self, room_id, stream_position, types=None):
 
         if last_events:
             last_event = last_events[-1]
-            state = yield self.get_state_after_event(last_event, types)
+            state = yield self.get_state_after_event(
+                last_event, types, filtered_types=filtered_types
+            )
 
         else:
             # no events in this room - so presumably no state
@@ -499,6 +507,7 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
             types = None
             member_state_ids = {}
             lazy_load_members = sync_config.filter_collection.lazy_load_members()
+            filtered_types = None
 
             if lazy_load_members:
                 # We only request state for the members needed to display the
@@ -516,29 +525,25 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                 # to be done based on event_id, and we don't have the member
                 # event ids until we've pulled them out of the DB.
 
-                if not types:
-                    # an optimisation to stop needlessly trying to calculate
-                    # member_state_ids
-                    #
-                    # XXX: i can't remember what this trying to do. why would
-                    # types ever be []? --matthew
-                    lazy_load_members = False
-
-                types.append((None, None))  # don't just filter to room members
+                # only apply the filtering to room members
+                filtered_types = [EventTypes.Member]
 
             if full_state:
                 if batch:
                     current_state_ids = yield self.store.get_state_ids_for_event(
-                        batch.events[-1].event_id, types=types
+                        batch.events[-1].event_id, types=types,
+                        filtered_types=filtered_types
                     )
 
                     state_ids = yield self.store.get_state_ids_for_event(
-                        batch.events[0].event_id, types=types
+                        batch.events[0].event_id, types=types,
+                        filtered_types=filtered_types
                     )
 
                 else:
                     current_state_ids = yield self.get_state_at(
-                        room_id, stream_position=now_token, types=types
+                        room_id, stream_position=now_token, types=types,
+                        filtered_types=filtered_types
                     )
 
                     state_ids = current_state_ids
@@ -563,15 +568,18 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                 )
             elif batch.limited:
                 state_at_previous_sync = yield self.get_state_at(
-                    room_id, stream_position=since_token, types=types
+                    room_id, stream_position=since_token, types=types,
+                    filtered_types=filtered_types
                 )
 
                 current_state_ids = yield self.store.get_state_ids_for_event(
-                    batch.events[-1].event_id, types=types
+                    batch.events[-1].event_id, types=types,
+                    filtered_types=filtered_types
                 )
 
                 state_at_timeline_start = yield self.store.get_state_ids_for_event(
-                    batch.events[0].event_id, types=types
+                    batch.events[0].event_id, types=types,
+                    filtered_types=filtered_types
                 )
 
                 if lazy_load_members:
@@ -603,11 +611,10 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                     # event_ids) at this point. We know we can do it based on mxid as this
                     # is an non-gappy incremental sync.
 
-                    # strip off the (None, None) and filter to just room members
-                    types = types[:-1]
                     if types:
                         state_ids = yield self.store.get_state_ids_for_event(
-                            batch.events[0].event_id, types=types
+                            batch.events[0].event_id, types=types,
+                            filtered_types=filtered_types
                         )
 
         state = {}
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index c5ff44fef7c3..ee531a2ce069 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -185,7 +185,7 @@ def get_state_groups(self, room_id, event_ids):
         })
 
     @defer.inlineCallbacks
-    def _get_state_groups_from_groups(self, groups, types):
+    def _get_state_groups_from_groups(self, groups, types, filtered_types=None):
         """Returns the state groups for a given set of groups, filtering on
         types of state events.
 
@@ -193,9 +193,10 @@ def _get_state_groups_from_groups(self, groups, types):
             groups(list[int]): list of state group IDs to query
             types(list[str|None, str|None])|None: List of 2-tuples of the form
                 (`type`, `state_key`), where a `state_key` of `None` matches all
-                state_keys for the `type`. Presence of type of `None` indicates
-                that types not in the list should not be filtered out. If None,
-                all types are returned.
+                state_keys for the `type`. If None, all types are returned.
+            filtered_types(list[str]|None): Only apply filtering via `types` to this
+                list of event types.  Other types of events are returned unfiltered.
+                If None, `types` filtering is applied to all events.
 
         Returns:
             dictionary state_group -> (dict of (type, state_key) -> event id)
@@ -206,26 +207,21 @@ def _get_state_groups_from_groups(self, groups, types):
         for chunk in chunks:
             res = yield self.runInteraction(
                 "_get_state_groups_from_groups",
-                self._get_state_groups_from_groups_txn, chunk, types,
+                self._get_state_groups_from_groups_txn, chunk, types, filtered_types
             )
             results.update(res)
 
         defer.returnValue(results)
 
-    def _get_state_groups_from_groups_txn(self, txn, groups, types=None):
+    def _get_state_groups_from_groups_txn(
+        self, txn, groups, types=None, filtered_types=None
+    ):
         results = {group: {} for group in groups}
 
-        include_other_types = False
+        include_other_types = False if filtered_types is None else True
 
         if types is not None:
-            type_set = set(types)
-            if (None, None) in type_set:
-                # special case (None, None) to mean that other types should be
-                # returned - i.e. we were just filtering down the state keys
-                # for particular types.
-                include_other_types = True
-                type_set.remove((None, None))
-            types = list(type_set)  # deduplicate types list
+            types = list(set(types))  # deduplicate types list
 
         if isinstance(self.database_engine, PostgresEngine):
             # Temporarily disable sequential scans in this transaction. This is
@@ -276,7 +272,7 @@ def _get_state_groups_from_groups_txn(self, txn, groups, types=None):
                 if include_other_types:
                     # XXX: check whether this slows postgres down like a list of
                     # ORs does too?
-                    unique_types = set([t for (t, _) in types])
+                    unique_types = set(filtered_types)
                     clause_to_args.append(
                         (
                             "AND type <> ? " * len(unique_types),
@@ -313,7 +309,7 @@ def _get_state_groups_from_groups_txn(self, txn, groups, types=None):
                         where_args.extend([typ[0], typ[1]])
 
                 if include_other_types:
-                    unique_types = set([t for (t, _) in types])
+                    unique_types = set(filtered_types)
                     where_clauses.append(
                         "(" + " AND ".join(["type <> ?"] * len(unique_types)) + ")"
                     )
@@ -373,18 +369,20 @@ def _get_state_groups_from_groups_txn(self, txn, groups, types=None):
         return results
 
     @defer.inlineCallbacks
-    def get_state_for_events(self, event_ids, types):
+    def get_state_for_events(self, event_ids, types, filtered_types):
         """Given a list of event_ids and type tuples, return a list of state
         dicts for each event. The state dicts will only have the type/state_keys
         that are in the `types` list.
 
         Args:
             event_ids (list[string])
-            types (list[(str|None, str|None)]|None): List of (type, state_key) tuples
+            types (list[(str, str|None)]|None): List of (type, state_key) tuples
                 which are used to filter the state fetched. If `state_key` is None,
-                all events are returned of the given type.  Presence of type of `None`
-                indicates that types not in the list should not be filtered out.
+                all events are returned of the given type.
                 May be None, which matches any key.
+            filtered_types(list[str]|None): Only apply filtering via `types` to this
+                list of event types.  Other types of events are returned unfiltered.
+                If None, `types` filtering is applied to all events.
 
         Returns:
             deferred: A list of dicts corresponding to the event_ids given.
@@ -395,7 +393,7 @@ def get_state_for_events(self, event_ids, types):
         )
 
         groups = set(itervalues(event_to_groups))
-        group_to_state = yield self._get_state_for_groups(groups, types)
+        group_to_state = yield self._get_state_for_groups(groups, types, filtered_types)
 
         state_event_map = yield self.get_events(
             [ev_id for sd in itervalues(group_to_state) for ev_id in itervalues(sd)],
@@ -414,17 +412,19 @@ def get_state_for_events(self, event_ids, types):
         defer.returnValue({event: event_to_state[event] for event in event_ids})
 
     @defer.inlineCallbacks
-    def get_state_ids_for_events(self, event_ids, types=None):
+    def get_state_ids_for_events(self, event_ids, types=None, filtered_types=None):
         """
         Get the state dicts corresponding to a list of events
 
         Args:
             event_ids(list(str)): events whose state should be returned
-            types(list[(str|None, str|None)]|None): List of (type, state_key) tuples
+            types(list[(str, str|None)]|None): List of (type, state_key) tuples
                 which are used to filter the state fetched. If `state_key` is None,
-                all events are returned of the given type.  Presence of type of `None`
-                indicates that types not in the list should not be filtered out.
+                all events are returned of the given type.
                 May be None, which matches any key.
+            filtered_types(list[str]|None): Only apply filtering via `types` to this
+                list of event types.  Other types of events are returned unfiltered.
+                If None, `types` filtering is applied to all events.
 
         Returns:
             A deferred dict from event_id -> (type, state_key) -> state_event
@@ -434,7 +434,7 @@ def get_state_ids_for_events(self, event_ids, types=None):
         )
 
         groups = set(itervalues(event_to_groups))
-        group_to_state = yield self._get_state_for_groups(groups, types)
+        group_to_state = yield self._get_state_for_groups(groups, types, filtered_types)
 
         event_to_state = {
             event_id: group_to_state[group]
@@ -444,41 +444,45 @@ def get_state_ids_for_events(self, event_ids, types=None):
         defer.returnValue({event: event_to_state[event] for event in event_ids})
 
     @defer.inlineCallbacks
-    def get_state_for_event(self, event_id, types=None):
+    def get_state_for_event(self, event_id, types=None, filtered_types=None):
         """
         Get the state dict corresponding to a particular event
 
         Args:
             event_id(str): event whose state should be returned
-            types(list[(str|None, str|None)]|None): List of (type, state_key) tuples
+            types(list[(str, str|None)]|None): List of (type, state_key) tuples
                 which are used to filter the state fetched. If `state_key` is None,
-                all events are returned of the given type.  Presence of type of `None`
-                indicates that types not in the list should not be filtered out.
+                all events are returned of the given type.
                 May be None, which matches any key.
+            filtered_types(list[str]|None): Only apply filtering via `types` to this
+                list of event types.  Other types of events are returned unfiltered.
+                If None, `types` filtering is applied to all events.
 
         Returns:
             A deferred dict from (type, state_key) -> state_event
         """
-        state_map = yield self.get_state_for_events([event_id], types)
+        state_map = yield self.get_state_for_events([event_id], types, filtered_types)
         defer.returnValue(state_map[event_id])
 
     @defer.inlineCallbacks
-    def get_state_ids_for_event(self, event_id, types=None):
+    def get_state_ids_for_event(self, event_id, types=None, filtered_types=None):
         """
         Get the state dict corresponding to a particular event
 
         Args:
             event_id(str): event whose state should be returned
-            types(list[(str|None, str|None)]|None): List of (type, state_key) tuples
+            types(list[(str, str|None)]|None): List of (type, state_key) tuples
                 which are used to filter the state fetched. If `state_key` is None,
-                all events are returned of the given type.  Presence of type of `None`
-                indicates that types not in the list should not be filtered out.
+                all events are returned of the given type.
                 May be None, which matches any key.
+            filtered_types(list[str]|None): Only apply filtering via `types` to this
+                list of event types.  Other types of events are returned unfiltered.
+                If None, `types` filtering is applied to all events.
 
         Returns:
             A deferred dict from (type, state_key) -> state_event
         """
-        state_map = yield self.get_state_ids_for_events([event_id], types)
+        state_map = yield self.get_state_ids_for_events([event_id], types, filtered_types)
         defer.returnValue(state_map[event_id])
 
     @cached(max_entries=50000)
@@ -509,7 +513,7 @@ def _get_state_group_for_events(self, event_ids):
 
         defer.returnValue({row["event_id"]: row["state_group"] for row in rows})
 
-    def _get_some_state_from_cache(self, group, types):
+    def _get_some_state_from_cache(self, group, types, filtered_types=None):
         """Checks if group is in cache. See `_get_state_for_groups`
 
         Returns 3-tuple (`state_dict`, `missing_types`, `got_all`).
@@ -520,29 +524,30 @@ def _get_some_state_from_cache(self, group, types):
 
         Args:
             group(int): The state group to lookup
-            types(list[str|None, str|None]): List of 2-tuples of the form
+            types(list[str, str|None]): List of 2-tuples of the form
                 (`type`, `state_key`), where a `state_key` of `None` matches all
-                state_keys for the `type`. Presence of type of `None` indicates
-                that types not in the list should not be filtered out.
+                state_keys for the `type`.
+            filtered_types(list[str]|None): Only apply filtering via `types` to this
+                list of event types.  Other types of events are returned unfiltered.
+                If None, `types` filtering is applied to all events.
         """
         is_all, known_absent, state_dict_ids = self._state_group_cache.get(group)
 
         type_to_key = {}
+
+        # tracks which of the requested types are missing from our cache
         missing_types = set()
 
-        include_other_types = False
+        include_other_types = True if filtered_types is None else False
 
         for typ, state_key in types:
             key = (typ, state_key)
 
-            if typ is None:
-                include_other_types = True
-                next
-
             if state_key is None:
                 type_to_key[typ] = None
                 # XXX: why do we mark the type as missing from our cache just
                 # because we weren't filtering on a specific value of state_key?
+                # is it because the cache doesn't handle wildcards?
                 missing_types.add(key)
             else:
                 if type_to_key.get(typ, object()) is not None:
@@ -556,7 +561,7 @@ def _get_some_state_from_cache(self, group, types):
         def include(typ, state_key):
             valid_state_keys = type_to_key.get(typ, sentinel)
             if valid_state_keys is sentinel:
-                return include_other_types
+                return include_other_types and typ not in filtered_types
             if valid_state_keys is None:
                 return True
             if state_key in valid_state_keys:
@@ -585,21 +590,23 @@ def _get_all_state_from_cache(self, group):
         return state_dict_ids, is_all
 
     @defer.inlineCallbacks
-    def _get_state_for_groups(self, groups, types=None):
+    def _get_state_for_groups(self, groups, types=None, filtered_types=None):
         """Gets the state at each of a list of state groups, optionally
         filtering by type/state_key
 
         Args:
             groups (iterable[int]): list of state groups for which we want
                 to get the state.
-            types (None|iterable[(None|str, None|str)]):
+            types (None|iterable[(None, None|str)]):
                 indicates the state type/keys required. If None, the whole
                 state is fetched and returned.
 
                 Otherwise, each entry should be a `(type, state_key)` tuple to
                 include in the response. A `state_key` of None is a wildcard
-                meaning that we require all state with that type. A `type` of None
-                indicates that types not in the list should not be filtered out.
+                meaning that we require all state with that type.
+            filtered_types(list[str]|None): Only apply filtering via `types` to this
+                list of event types.  Other types of events are returned unfiltered.
+                If None, `types` filtering is applied to all events.
 
         Returns:
             Deferred[dict[int, dict[(type, state_key), EventBase]]]
@@ -612,7 +619,7 @@ def _get_state_for_groups(self, groups, types=None):
         if types is not None:
             for group in set(groups):
                 state_dict_ids, _, got_all = self._get_some_state_from_cache(
-                    group, types,
+                    group, types, filtered_types
                 )
                 results[group] = state_dict_ids
 
@@ -645,7 +652,7 @@ def _get_state_for_groups(self, groups, types=None):
                 types_to_fetch = types
 
             group_to_state_dict = yield self._get_state_groups_from_groups(
-                missing_groups, types_to_fetch,
+                missing_groups, types_to_fetch, filtered_types
             )
 
             for group, group_state_dict in iteritems(group_to_state_dict):

From bcaec2915ac74937171e27d507b8f9c0e39d3677 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Thu, 19 Jul 2018 19:03:50 +0100
Subject: [PATCH 075/830] incorporate review

---
 synapse/handlers/sync.py | 44 ++++++++++++++++++++++++----------------
 synapse/storage/state.py |  7 ++++---
 2 files changed, 31 insertions(+), 20 deletions(-)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index cb711b87588c..b597f94cf641 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -435,7 +435,7 @@ def get_state_after_event(self, event, types=None, filtered_types=None):
             A Deferred map from ((type, state_key)->Event)
         """
         state_ids = yield self.store.get_state_ids_for_event(
-            event.event_id, types, filtered_types=filtered_types
+            event.event_id, types, filtered_types=filtered_types,
         )
         if event.is_state():
             state_ids = state_ids.copy()
@@ -470,7 +470,7 @@ def get_state_at(self, room_id, stream_position, types=None, filtered_types=None
         if last_events:
             last_event = last_events[-1]
             state = yield self.get_state_after_event(
-                last_event, types, filtered_types=filtered_types
+                last_event, types, filtered_types=filtered_types,
             )
 
         else:
@@ -505,7 +505,6 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
         with Measure(self.clock, "compute_state_delta"):
 
             types = None
-            member_state_ids = {}
             lazy_load_members = sync_config.filter_collection.lazy_load_members()
             filtered_types = None
 
@@ -521,10 +520,6 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                     )
                 ]
 
-                # We can't remove redundant member types at this stage as it has
-                # to be done based on event_id, and we don't have the member
-                # event ids until we've pulled them out of the DB.
-
                 # only apply the filtering to room members
                 filtered_types = [EventTypes.Member]
 
@@ -532,27 +527,32 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                 if batch:
                     current_state_ids = yield self.store.get_state_ids_for_event(
                         batch.events[-1].event_id, types=types,
-                        filtered_types=filtered_types
+                        filtered_types=filtered_types,
                     )
 
                     state_ids = yield self.store.get_state_ids_for_event(
                         batch.events[0].event_id, types=types,
-                        filtered_types=filtered_types
+                        filtered_types=filtered_types,
                     )
 
                 else:
                     current_state_ids = yield self.get_state_at(
                         room_id, stream_position=now_token, types=types,
-                        filtered_types=filtered_types
+                        filtered_types=filtered_types,
                     )
 
                     state_ids = current_state_ids
 
+                # track the membership state events as of the beginning of this
+                # timeline sequence, so they can be filtered out of the state
+                # if we are lazy loading members.
                 if lazy_load_members:
                     member_state_ids = {
                         t: state_ids[t]
                         for t in state_ids if t[0] == EventTypes.Member
                     }
+                else:
+                    member_state_ids = {}
 
                 timeline_state = {
                     (event.type, event.state_key): event.event_id
@@ -569,28 +569,38 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
             elif batch.limited:
                 state_at_previous_sync = yield self.get_state_at(
                     room_id, stream_position=since_token, types=types,
-                    filtered_types=filtered_types
+                    filtered_types=filtered_types,
                 )
 
                 current_state_ids = yield self.store.get_state_ids_for_event(
                     batch.events[-1].event_id, types=types,
-                    filtered_types=filtered_types
+                    filtered_types=filtered_types,
                 )
 
                 state_at_timeline_start = yield self.store.get_state_ids_for_event(
                     batch.events[0].event_id, types=types,
-                    filtered_types=filtered_types
+                    filtered_types=filtered_types,
                 )
 
+                # track the membership state events as of the beginning of this
+                # timeline sequence, so they can be filtered out of the state
+                # if we are lazy loading members.
                 if lazy_load_members:
-                    # TODO: filter out redundant members based on their event_ids
-                    # (not mxids) at this point. In practice, limited syncs are
+                    # TODO: optionally filter out redundant membership events at this
+                    # point, to stop repeatedly sending members in every /sync as if
+                    # the client isn't tracking them.
+                    # When implement, this should filter using event_ids (not mxids).
+                    # In practice, limited syncs are
                     # relatively rare so it's not a total disaster to send redundant
-                    # members down at this point.
+                    # members down at this point. Redundant members are ones which
+                    # repeatedly get sent down /sync because we don't know if the client
+                    # is caching them or not.
                     member_state_ids = {
                         t: state_at_timeline_start[t]
                         for t in state_at_timeline_start if t[0] == EventTypes.Member
                     }
+                else:
+                    member_state_ids = {}
 
                 timeline_state = {
                     (event.type, event.state_key): event.event_id
@@ -614,7 +624,7 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                     if types:
                         state_ids = yield self.store.get_state_ids_for_event(
                             batch.events[0].event_id, types=types,
-                            filtered_types=filtered_types
+                            filtered_types=filtered_types,
                         )
 
         state = {}
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index ee531a2ce069..75c6366e7ac2 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -545,9 +545,10 @@ def _get_some_state_from_cache(self, group, types, filtered_types=None):
 
             if state_key is None:
                 type_to_key[typ] = None
-                # XXX: why do we mark the type as missing from our cache just
-                # because we weren't filtering on a specific value of state_key?
-                # is it because the cache doesn't handle wildcards?
+                # we mark the type as missing from the cache because
+                # when the cache was populated it might have been done with a
+                # restricted set of state_keys, so the wildcard will not work
+                # and the cache may be incomplete.
                 missing_types.add(key)
             else:
                 if type_to_key.get(typ, object()) is not None:

From 2f558300cc648e633342746dc7b42a36fcb6b32e Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Thu, 19 Jul 2018 19:22:27 +0100
Subject: [PATCH 076/830] fix thinkos; unbreak tests

---
 synapse/storage/state.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index 75c6366e7ac2..f09be7172dc0 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -369,7 +369,7 @@ def _get_state_groups_from_groups_txn(
         return results
 
     @defer.inlineCallbacks
-    def get_state_for_events(self, event_ids, types, filtered_types):
+    def get_state_for_events(self, event_ids, types, filtered_types=None):
         """Given a list of event_ids and type tuples, return a list of state
         dicts for each event. The state dicts will only have the type/state_keys
         that are in the `types` list.
@@ -538,7 +538,7 @@ def _get_some_state_from_cache(self, group, types, filtered_types=None):
         # tracks which of the requested types are missing from our cache
         missing_types = set()
 
-        include_other_types = True if filtered_types is None else False
+        include_other_types = False if filtered_types is None else True
 
         for typ, state_key in types:
             key = (typ, state_key)

From 1fa4f7e03e5cdaebeda0f0b4c49120c991f5bf57 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Thu, 19 Jul 2018 20:19:32 +0100
Subject: [PATCH 077/830] first cut of a UT for testing state store (untested)

---
 tests/storage/test_state.py | 151 ++++++++++++++++++++++++++++++++++++
 1 file changed, 151 insertions(+)
 create mode 100644 tests/storage/test_state.py

diff --git a/tests/storage/test_state.py b/tests/storage/test_state.py
new file mode 100644
index 000000000000..acf36e077f4c
--- /dev/null
+++ b/tests/storage/test_state.py
@@ -0,0 +1,151 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.internet import defer
+
+from synapse.api.constants import EventTypes, Membership
+from synapse.types import RoomID, UserID
+
+import tests.unittest
+import tests.utils
+
+
+class StateStoreTestCase(tests.unittest.TestCase):
+    def __init__(self, *args, **kwargs):
+        super(StateStoreTestCase, self).__init__(*args, **kwargs)
+        self.store = None  # type: synapse.storage.DataStore
+
+    @defer.inlineCallbacks
+    def setUp(self):
+        hs = yield tests.utils.setup_test_homeserver()
+
+        self.store = hs.get_datastore()
+        self.event_builder_factory = hs.get_event_builder_factory()
+        self.event_creation_handler = hs.get_event_creation_handler()
+
+        self.u_alice = UserID.from_string("@alice:test")
+        self.u_bob = UserID.from_string("@bob:test")
+
+        # User elsewhere on another host
+        self.u_charlie = UserID.from_string("@charlie:elsewhere")
+
+        self.room = RoomID.from_string("!abc123:test")
+
+        yield self.store.store_room(
+            self.room.to_string(),
+            room_creator_user_id="@creator:text",
+            is_public=True
+        )
+
+    @defer.inlineCallbacks
+    def inject_state_event(self, room, sender, typ, state_key, content):
+        builder = self.event_builder_factory.new({
+            "type": typ,
+            "sender": sender.to_string(),
+            "state_key": state_key,
+            "room_id": room.to_string(),
+            "content": content,
+        })
+
+        event, context = yield self.event_creation_handler.create_new_client_event(
+            builder
+        )
+
+        yield self.store.persist_event(event, context)
+
+        defer.returnValue(event)
+
+    @defer.inlineCallbacks
+    def test_get_state_for_events(self):
+
+        # this defaults to a linear DAG as each new injection defaults to whatever
+        # forward extremities are currently in the DB for this room.
+        (e1, c1) = yield self.inject_state_event(
+            self.room, self.u_alice, EventTypes.Create, '', {},
+        )
+        (e2, c2) = yield self.inject_state_event(
+            self.room, self.u_alice, EventTypes.Name, '', {
+                "name": "test room"
+            },
+        )
+        (e3, c3) = yield self.inject_state_event(
+            self.room, self.u_alice, EventTypes.Member, self.u_alice, {
+                "membership": Membership.JOIN
+            },
+        )
+        (e4, c4) = yield self.inject_state_event(
+            self.room, self.u_bob, EventTypes.Member, self.u_bob, {
+                "membership": Membership.JOIN
+            },
+        )
+        (e5, c5) = yield self.inject_state_event(
+            self.room, self.u_bob, EventTypes.Member, self.u_bob, {
+                "membership": Membership.LEAVE
+            },
+        )
+
+        # check we get the full state as of the final event
+        state = yield self.store.get_state_for_events(
+            e5.event_id, None, filtered_types=None
+        )
+
+        self.assertDictEqual({
+            (e1.type, e1.state_key): e1.event_id,
+            (e2.type, e2.state_key): e2.event_id,
+            (e3.type, e3.state_key): e3.event_id,
+            # e4 is overwritten by e5
+            (e5.type, e5.state_key): e5.event_id,
+        }, state)
+
+        # check we can filter to the m.room.name event (with a '' state key)
+        state = yield self.store.get_state_for_events(
+            e5.event_id, ((EventTypes.Name, '')), filtered_types=None
+        )
+
+        self.assertDictEqual({
+            (e2.type, e2.state_key): e2.event_id,
+        }, state)
+
+        # check we can filter to the m.room.name event (with a wildcard None state key)
+        state = yield self.store.get_state_for_events(
+            e5.event_id, ((EventTypes.Name, None)), filtered_types=None
+        )
+
+        self.assertDictEqual({
+            (e2.type, e2.state_key): e2.event_id,
+        }, state)
+
+        # check we can grab the m.room.member events (with a wildcard None state key)
+        state = yield self.store.get_state_for_events(
+            e5.event_id, ((EventTypes.Member, None)), filtered_types=None
+        )
+
+        self.assertDictEqual({
+            (e3.type, e3.state_key): e3.event_id,
+            (e5.type, e5.state_key): e5.event_id,
+        }, state)
+
+        # check we can use filter_types to grab a specific room member
+        # without filtering out the other event types
+        state = yield self.store.get_state_for_events(
+            e5.event_id, ((EventTypes.Member, self.u_alice)),
+            filtered_types=[EventTypes.Member],
+        )
+
+        self.assertDictEqual({
+            (e1.type, e1.state_key): e3.event_id,
+            (e2.type, e2.state_key): e3.event_id,
+            (e3.type, e3.state_key): e5.event_id,
+        }, state)

From 650daf56285c515d9c0875a9a3894a033337b0c9 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Thu, 19 Jul 2018 20:49:44 +0100
Subject: [PATCH 078/830] make test work

---
 tests/storage/test_state.py | 83 ++++++++++++++++++++-----------------
 1 file changed, 46 insertions(+), 37 deletions(-)

diff --git a/tests/storage/test_state.py b/tests/storage/test_state.py
index acf36e077f4c..8924ba9f7f90 100644
--- a/tests/storage/test_state.py
+++ b/tests/storage/test_state.py
@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import logging
+
 from twisted.internet import defer
 
 from synapse.api.constants import EventTypes, Membership
@@ -21,6 +23,8 @@
 import tests.unittest
 import tests.utils
 
+logger = logging.getLogger(__name__)
+
 
 class StateStoreTestCase(tests.unittest.TestCase):
     def __init__(self, *args, **kwargs):
@@ -38,9 +42,6 @@ def setUp(self):
         self.u_alice = UserID.from_string("@alice:test")
         self.u_bob = UserID.from_string("@bob:test")
 
-        # User elsewhere on another host
-        self.u_charlie = UserID.from_string("@charlie:elsewhere")
-
         self.room = RoomID.from_string("!abc123:test")
 
         yield self.store.store_room(
@@ -67,85 +68,93 @@ def inject_state_event(self, room, sender, typ, state_key, content):
 
         defer.returnValue(event)
 
+    def assertStateMapEqual(self, s1, s2):
+        for t in s1:
+            # just compare event IDs for simplicity
+            self.assertEqual(s1[t].event_id, s2[t].event_id)
+        self.assertEqual(len(s1), len(s2))
+
     @defer.inlineCallbacks
-    def test_get_state_for_events(self):
+    def test_get_state_for_event(self):
 
         # this defaults to a linear DAG as each new injection defaults to whatever
         # forward extremities are currently in the DB for this room.
-        (e1, c1) = yield self.inject_state_event(
+        e1 = yield self.inject_state_event(
             self.room, self.u_alice, EventTypes.Create, '', {},
         )
-        (e2, c2) = yield self.inject_state_event(
+        e2 = yield self.inject_state_event(
             self.room, self.u_alice, EventTypes.Name, '', {
                 "name": "test room"
             },
         )
-        (e3, c3) = yield self.inject_state_event(
-            self.room, self.u_alice, EventTypes.Member, self.u_alice, {
+        e3 = yield self.inject_state_event(
+            self.room, self.u_alice, EventTypes.Member, self.u_alice.to_string(), {
                 "membership": Membership.JOIN
             },
         )
-        (e4, c4) = yield self.inject_state_event(
-            self.room, self.u_bob, EventTypes.Member, self.u_bob, {
+        e4 = yield self.inject_state_event(
+            self.room, self.u_bob, EventTypes.Member, self.u_bob.to_string(), {
                 "membership": Membership.JOIN
             },
         )
-        (e5, c5) = yield self.inject_state_event(
-            self.room, self.u_bob, EventTypes.Member, self.u_bob, {
+        e5 = yield self.inject_state_event(
+            self.room, self.u_bob, EventTypes.Member, self.u_bob.to_string(), {
                 "membership": Membership.LEAVE
             },
         )
 
         # check we get the full state as of the final event
-        state = yield self.store.get_state_for_events(
+        state = yield self.store.get_state_for_event(
             e5.event_id, None, filtered_types=None
         )
 
-        self.assertDictEqual({
-            (e1.type, e1.state_key): e1.event_id,
-            (e2.type, e2.state_key): e2.event_id,
-            (e3.type, e3.state_key): e3.event_id,
+        self.assertIsNotNone(e4)
+
+        self.assertStateMapEqual({
+            (e1.type, e1.state_key): e1,
+            (e2.type, e2.state_key): e2,
+            (e3.type, e3.state_key): e3,
             # e4 is overwritten by e5
-            (e5.type, e5.state_key): e5.event_id,
+            (e5.type, e5.state_key): e5,
         }, state)
 
         # check we can filter to the m.room.name event (with a '' state key)
-        state = yield self.store.get_state_for_events(
-            e5.event_id, ((EventTypes.Name, '')), filtered_types=None
+        state = yield self.store.get_state_for_event(
+            e5.event_id, [(EventTypes.Name, '')], filtered_types=None
         )
 
-        self.assertDictEqual({
-            (e2.type, e2.state_key): e2.event_id,
+        self.assertStateMapEqual({
+            (e2.type, e2.state_key): e2,
         }, state)
 
         # check we can filter to the m.room.name event (with a wildcard None state key)
-        state = yield self.store.get_state_for_events(
-            e5.event_id, ((EventTypes.Name, None)), filtered_types=None
+        state = yield self.store.get_state_for_event(
+            e5.event_id, [(EventTypes.Name, None)], filtered_types=None
         )
 
-        self.assertDictEqual({
-            (e2.type, e2.state_key): e2.event_id,
+        self.assertStateMapEqual({
+            (e2.type, e2.state_key): e2,
         }, state)
 
         # check we can grab the m.room.member events (with a wildcard None state key)
-        state = yield self.store.get_state_for_events(
-            e5.event_id, ((EventTypes.Member, None)), filtered_types=None
+        state = yield self.store.get_state_for_event(
+            e5.event_id, [(EventTypes.Member, None)], filtered_types=None
         )
 
-        self.assertDictEqual({
-            (e3.type, e3.state_key): e3.event_id,
-            (e5.type, e5.state_key): e5.event_id,
+        self.assertStateMapEqual({
+            (e3.type, e3.state_key): e3,
+            (e5.type, e5.state_key): e5,
         }, state)
 
         # check we can use filter_types to grab a specific room member
         # without filtering out the other event types
-        state = yield self.store.get_state_for_events(
-            e5.event_id, ((EventTypes.Member, self.u_alice)),
+        state = yield self.store.get_state_for_event(
+            e5.event_id, [(EventTypes.Member, self.u_alice.to_string())],
             filtered_types=[EventTypes.Member],
         )
 
-        self.assertDictEqual({
-            (e1.type, e1.state_key): e3.event_id,
-            (e2.type, e2.state_key): e3.event_id,
-            (e3.type, e3.state_key): e5.event_id,
+        self.assertStateMapEqual({
+            (e1.type, e1.state_key): e1,
+            (e2.type, e2.state_key): e2,
+            (e3.type, e3.state_key): e3,
         }, state)

From d7275eecf3abcd5f7c645e5da47c7e7310350333 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 20 Jul 2018 12:37:12 +0100
Subject: [PATCH 079/830] Add a sleep to the Limiter to fix stack overflows.

Fixes #3570
---
 synapse/util/async.py      | 23 ++++++++++++++++++++---
 tests/util/test_limiter.py |  8 ++++----
 2 files changed, 24 insertions(+), 7 deletions(-)

diff --git a/synapse/util/async.py b/synapse/util/async.py
index 5d0fb3913058..7d5acecb1c00 100644
--- a/synapse/util/async.py
+++ b/synapse/util/async.py
@@ -248,11 +248,15 @@ class Limiter(object):
             # do some work.
 
     """
-    def __init__(self, max_count):
+    def __init__(self, max_count, clock=None):
         """
         Args:
             max_count(int): The maximum number of concurrent access
         """
+        if not clock:
+            from twisted.internet import reactor
+            clock = Clock(reactor)
+        self._clock = clock
         self.max_count = max_count
 
         # key_to_defer is a map from the key to a 2 element list where
@@ -277,10 +281,23 @@ def queue(self, key):
             with PreserveLoggingContext():
                 yield new_defer
             logger.info("Acquired limiter lock for key %r", key)
+            entry[0] += 1
+
+            # if the code holding the lock completes synchronously, then it
+            # will recursively run the next claimant on the list. That can
+            # relatively rapidly lead to stack exhaustion. This is essentially
+            # the same problem as http://twistedmatrix.com/trac/ticket/9304.
+            #
+            # In order to break the cycle, we add a cheeky sleep(0) here to
+            # ensure that we fall back to the reactor between each iteration.
+            #
+            # (This needs to happen while we hold the lock, and the context manager's exit
+            # code must be synchronous, so this is the only sensible place.)
+            yield self._clock.sleep(0)
+
         else:
             logger.info("Acquired uncontended limiter lock for key %r", key)
-
-        entry[0] += 1
+            entry[0] += 1
 
         @contextmanager
         def _ctx_manager():
diff --git a/tests/util/test_limiter.py b/tests/util/test_limiter.py
index a5a767b1ff67..f7b942f5c1be 100644
--- a/tests/util/test_limiter.py
+++ b/tests/util/test_limiter.py
@@ -48,21 +48,21 @@ def test_limiter(self):
             self.assertFalse(d4.called)
             self.assertFalse(d5.called)
 
-        self.assertTrue(d4.called)
+        cm4 = yield d4
         self.assertFalse(d5.called)
 
         with cm3:
             self.assertFalse(d5.called)
 
-        self.assertTrue(d5.called)
+        cm5 = yield d5
 
         with cm2:
             pass
 
-        with (yield d4):
+        with cm4:
             pass
 
-        with (yield d5):
+        with cm5:
             pass
 
         d6 = limiter.queue(key)

From 8462c26485fb4f19fc52accc05870c0ea4c8eb6a Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 20 Jul 2018 12:43:23 +0100
Subject: [PATCH 080/830] Improvements to the Limiter

* give them names, to improve logging
* use a deque rather than a list for efficiency
---
 synapse/handlers/message.py |  2 +-
 synapse/util/async.py       | 33 ++++++++++++++++++++-------------
 2 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index a39b852cebfc..8c12c6990f5d 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -427,7 +427,7 @@ def __init__(self, hs):
 
         # We arbitrarily limit concurrent event creation for a room to 5.
         # This is to stop us from diverging history *too* much.
-        self.limiter = Limiter(max_count=5)
+        self.limiter = Limiter(max_count=5, name="room_event_creation_limit")
 
         self.action_generator = hs.get_action_generator()
 
diff --git a/synapse/util/async.py b/synapse/util/async.py
index 7d5acecb1c00..22071ddef739 100644
--- a/synapse/util/async.py
+++ b/synapse/util/async.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2018 New Vector Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,7 +13,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import collections
 import logging
 from contextlib import contextmanager
 
@@ -248,11 +249,16 @@ class Limiter(object):
             # do some work.
 
     """
-    def __init__(self, max_count, clock=None):
+    def __init__(self, max_count=1, name=None, clock=None):
         """
         Args:
-            max_count(int): The maximum number of concurrent access
+            max_count(int): The maximum number of concurrent accesses
         """
+        if name is None:
+            self.name = id(self)
+        else:
+            self.name = name
+
         if not clock:
             from twisted.internet import reactor
             clock = Clock(reactor)
@@ -260,14 +266,14 @@ def __init__(self, max_count, clock=None):
         self.max_count = max_count
 
         # key_to_defer is a map from the key to a 2 element list where
-        # the first element is the number of things executing
-        # the second element is a list of deferreds for the things blocked from
+        # the first element is the number of things executing, and
+        # the second element is a deque of deferreds for the things blocked from
         # executing.
         self.key_to_defer = {}
 
     @defer.inlineCallbacks
     def queue(self, key):
-        entry = self.key_to_defer.setdefault(key, [0, []])
+        entry = self.key_to_defer.setdefault(key, [0, collections.deque()])
 
         # If the number of things executing is greater than the maximum
         # then add a deferred to the list of blocked items
@@ -277,10 +283,10 @@ def queue(self, key):
             new_defer = defer.Deferred()
             entry[1].append(new_defer)
 
-            logger.info("Waiting to acquire limiter lock for key %r", key)
-            with PreserveLoggingContext():
-                yield new_defer
-            logger.info("Acquired limiter lock for key %r", key)
+            logger.info("Waiting to acquire limiter lock %r for key %r", self.name, key)
+            yield make_deferred_yieldable(new_defer)
+
+            logger.info("Acquired limiter lock %r for key %r", self.name, key)
             entry[0] += 1
 
             # if the code holding the lock completes synchronously, then it
@@ -296,7 +302,7 @@ def queue(self, key):
             yield self._clock.sleep(0)
 
         else:
-            logger.info("Acquired uncontended limiter lock for key %r", key)
+            logger.info("Acquired uncontended limiter lock %r for key %r", self.name, key)
             entry[0] += 1
 
         @contextmanager
@@ -304,15 +310,16 @@ def _ctx_manager():
             try:
                 yield
             finally:
-                logger.info("Releasing limiter lock for key %r", key)
+                logger.info("Releasing limiter lock %r for key %r", self.name, key)
 
                 # We've finished executing so check if there are any things
                 # blocked waiting to execute and start one of them
                 entry[0] -= 1
 
                 if entry[1]:
-                    next_def = entry[1].pop(0)
+                    next_def = entry[1].popleft()
 
+                    # we need to run the next thing in the sentinel context.
                     with PreserveLoggingContext():
                         next_def.callback(None)
                 elif entry[0] == 0:

From 7c712f95bbc7f405355d5714c92d65551f64fec2 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 20 Jul 2018 13:11:43 +0100
Subject: [PATCH 081/830] Combine Limiter and Linearizer

Linearizer was effectively a Limiter with max_count=1, so rather than
maintaining two sets of code, let's combine them.
---
 synapse/handlers/message.py   |  4 +-
 synapse/util/async.py         | 99 ++++-------------------------------
 tests/util/test_limiter.py    | 70 -------------------------
 tests/util/test_linearizer.py | 47 +++++++++++++++++
 4 files changed, 59 insertions(+), 161 deletions(-)
 delete mode 100644 tests/util/test_limiter.py

diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 8c12c6990f5d..abc07ea87c88 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -33,7 +33,7 @@
 from synapse.events.validator import EventValidator
 from synapse.replication.http.send_event import send_event_to_master
 from synapse.types import RoomAlias, RoomStreamToken, UserID
-from synapse.util.async import Limiter, ReadWriteLock
+from synapse.util.async import Linearizer, ReadWriteLock
 from synapse.util.frozenutils import frozendict_json_encoder
 from synapse.util.logcontext import run_in_background
 from synapse.util.metrics import measure_func
@@ -427,7 +427,7 @@ def __init__(self, hs):
 
         # We arbitrarily limit concurrent event creation for a room to 5.
         # This is to stop us from diverging history *too* much.
-        self.limiter = Limiter(max_count=5, name="room_event_creation_limit")
+        self.limiter = Linearizer(max_count=5, name="room_event_creation_limit")
 
         self.action_generator = hs.get_action_generator()
 
diff --git a/synapse/util/async.py b/synapse/util/async.py
index 22071ddef739..5a50d9700fa3 100644
--- a/synapse/util/async.py
+++ b/synapse/util/async.py
@@ -157,91 +157,8 @@ def _concurrently_execute_inner():
 
 
 class Linearizer(object):
-    """Linearizes access to resources based on a key. Useful to ensure only one
-    thing is happening at a time on a given resource.
-
-    Example:
-
-        with (yield linearizer.queue("test_key")):
-            # do some work.
-
-    """
-    def __init__(self, name=None, clock=None):
-        if name is None:
-            self.name = id(self)
-        else:
-            self.name = name
-        self.key_to_defer = {}
-
-        if not clock:
-            from twisted.internet import reactor
-            clock = Clock(reactor)
-        self._clock = clock
-
-    @defer.inlineCallbacks
-    def queue(self, key):
-        # If there is already a deferred in the queue, we pull it out so that
-        # we can wait on it later.
-        # Then we replace it with a deferred that we resolve *after* the
-        # context manager has exited.
-        # We only return the context manager after the previous deferred has
-        # resolved.
-        # This all has the net effect of creating a chain of deferreds that
-        # wait for the previous deferred before starting their work.
-        current_defer = self.key_to_defer.get(key)
-
-        new_defer = defer.Deferred()
-        self.key_to_defer[key] = new_defer
-
-        if current_defer:
-            logger.info(
-                "Waiting to acquire linearizer lock %r for key %r", self.name, key
-            )
-            try:
-                with PreserveLoggingContext():
-                    yield current_defer
-            except Exception:
-                logger.exception("Unexpected exception in Linearizer")
-
-            logger.info("Acquired linearizer lock %r for key %r", self.name,
-                        key)
-
-            # if the code holding the lock completes synchronously, then it
-            # will recursively run the next claimant on the list. That can
-            # relatively rapidly lead to stack exhaustion. This is essentially
-            # the same problem as http://twistedmatrix.com/trac/ticket/9304.
-            #
-            # In order to break the cycle, we add a cheeky sleep(0) here to
-            # ensure that we fall back to the reactor between each iteration.
-            #
-            # (There's no particular need for it to happen before we return
-            # the context manager, but it needs to happen while we hold the
-            # lock, and the context manager's exit code must be synchronous,
-            # so actually this is the only sensible place.
-            yield self._clock.sleep(0)
-
-        else:
-            logger.info("Acquired uncontended linearizer lock %r for key %r",
-                        self.name, key)
-
-        @contextmanager
-        def _ctx_manager():
-            try:
-                yield
-            finally:
-                logger.info("Releasing linearizer lock %r for key %r", self.name, key)
-                with PreserveLoggingContext():
-                    new_defer.callback(None)
-                current_d = self.key_to_defer.get(key)
-                if current_d is new_defer:
-                    self.key_to_defer.pop(key, None)
-
-        defer.returnValue(_ctx_manager())
-
-
-class Limiter(object):
     """Limits concurrent access to resources based on a key. Useful to ensure
-    only a few thing happen at a time on a given resource.
+    only a few things happen at a time on a given resource.
 
     Example:
 
@@ -249,7 +166,7 @@ class Limiter(object):
             # do some work.
 
     """
-    def __init__(self, max_count=1, name=None, clock=None):
+    def __init__(self, name=None, max_count=1, clock=None):
         """
         Args:
             max_count(int): The maximum number of concurrent accesses
@@ -283,10 +200,12 @@ def queue(self, key):
             new_defer = defer.Deferred()
             entry[1].append(new_defer)
 
-            logger.info("Waiting to acquire limiter lock %r for key %r", self.name, key)
+            logger.info(
+                "Waiting to acquire linearizer lock %r for key %r", self.name, key,
+            )
             yield make_deferred_yieldable(new_defer)
 
-            logger.info("Acquired limiter lock %r for key %r", self.name, key)
+            logger.info("Acquired linearizer lock %r for key %r", self.name, key)
             entry[0] += 1
 
             # if the code holding the lock completes synchronously, then it
@@ -302,7 +221,9 @@ def queue(self, key):
             yield self._clock.sleep(0)
 
         else:
-            logger.info("Acquired uncontended limiter lock %r for key %r", self.name, key)
+            logger.info(
+                "Acquired uncontended linearizer lock %r for key %r", self.name, key,
+            )
             entry[0] += 1
 
         @contextmanager
@@ -310,7 +231,7 @@ def _ctx_manager():
             try:
                 yield
             finally:
-                logger.info("Releasing limiter lock %r for key %r", self.name, key)
+                logger.info("Releasing linearizer lock %r for key %r", self.name, key)
 
                 # We've finished executing so check if there are any things
                 # blocked waiting to execute and start one of them
diff --git a/tests/util/test_limiter.py b/tests/util/test_limiter.py
deleted file mode 100644
index f7b942f5c1be..000000000000
--- a/tests/util/test_limiter.py
+++ /dev/null
@@ -1,70 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2016 OpenMarket Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-from twisted.internet import defer
-
-from synapse.util.async import Limiter
-
-from tests import unittest
-
-
-class LimiterTestCase(unittest.TestCase):
-
-    @defer.inlineCallbacks
-    def test_limiter(self):
-        limiter = Limiter(3)
-
-        key = object()
-
-        d1 = limiter.queue(key)
-        cm1 = yield d1
-
-        d2 = limiter.queue(key)
-        cm2 = yield d2
-
-        d3 = limiter.queue(key)
-        cm3 = yield d3
-
-        d4 = limiter.queue(key)
-        self.assertFalse(d4.called)
-
-        d5 = limiter.queue(key)
-        self.assertFalse(d5.called)
-
-        with cm1:
-            self.assertFalse(d4.called)
-            self.assertFalse(d5.called)
-
-        cm4 = yield d4
-        self.assertFalse(d5.called)
-
-        with cm3:
-            self.assertFalse(d5.called)
-
-        cm5 = yield d5
-
-        with cm2:
-            pass
-
-        with cm4:
-            pass
-
-        with cm5:
-            pass
-
-        d6 = limiter.queue(key)
-        with (yield d6):
-            pass
diff --git a/tests/util/test_linearizer.py b/tests/util/test_linearizer.py
index c95907b32cc4..c9563124f92b 100644
--- a/tests/util/test_linearizer.py
+++ b/tests/util/test_linearizer.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
+# Copyright 2018 New Vector Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -65,3 +66,49 @@ def func(i, sleep=False):
             func(i)
 
         return func(1000)
+
+    @defer.inlineCallbacks
+    def test_multiple_entries(self):
+        limiter = Linearizer(max_count=3)
+
+        key = object()
+
+        d1 = limiter.queue(key)
+        cm1 = yield d1
+
+        d2 = limiter.queue(key)
+        cm2 = yield d2
+
+        d3 = limiter.queue(key)
+        cm3 = yield d3
+
+        d4 = limiter.queue(key)
+        self.assertFalse(d4.called)
+
+        d5 = limiter.queue(key)
+        self.assertFalse(d5.called)
+
+        with cm1:
+            self.assertFalse(d4.called)
+            self.assertFalse(d5.called)
+
+        cm4 = yield d4
+        self.assertFalse(d5.called)
+
+        with cm3:
+            self.assertFalse(d5.called)
+
+        cm5 = yield d5
+
+        with cm2:
+            pass
+
+        with cm4:
+            pass
+
+        with cm5:
+            pass
+
+        d6 = limiter.queue(key)
+        with (yield d6):
+            pass

From 683f4058c18c51bd84d99faa9b64bd81c57ee8cb Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 20 Jul 2018 13:16:39 +0100
Subject: [PATCH 082/830] changelogs

---
 changelog.d/3570.bugfix | 1 +
 changelog.d/3571.misc   | 1 +
 2 files changed, 2 insertions(+)
 create mode 100644 changelog.d/3570.bugfix
 create mode 100644 changelog.d/3571.misc

diff --git a/changelog.d/3570.bugfix b/changelog.d/3570.bugfix
new file mode 100644
index 000000000000..cec5158a9902
--- /dev/null
+++ b/changelog.d/3570.bugfix
@@ -0,0 +1 @@
+Fix potential stack overflow and deadlock under heavy load
\ No newline at end of file
diff --git a/changelog.d/3571.misc b/changelog.d/3571.misc
new file mode 100644
index 000000000000..8908324e6803
--- /dev/null
+++ b/changelog.d/3571.misc
@@ -0,0 +1 @@
+Merge Linearizer and Limiter

From e1a237eaabf0ba37f242897700f9bf00729976b8 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Fri, 20 Jul 2018 22:41:13 +1000
Subject: [PATCH 083/830] Admin API for creating new users (#3415)

---
 changelog.d/3415.misc              |   0
 docs/admin_api/register_api.rst    |  63 ++++++
 scripts/register_new_matrix_user   |  32 ++-
 synapse/rest/client/v1/admin.py    | 122 ++++++++++++
 synapse/secrets.py                 |  42 ++++
 synapse/server.py                  |   5 +
 tests/rest/client/v1/test_admin.py | 305 +++++++++++++++++++++++++++++
 tests/utils.py                     |   3 +
 8 files changed, 569 insertions(+), 3 deletions(-)
 create mode 100644 changelog.d/3415.misc
 create mode 100644 docs/admin_api/register_api.rst
 create mode 100644 synapse/secrets.py
 create mode 100644 tests/rest/client/v1/test_admin.py

diff --git a/changelog.d/3415.misc b/changelog.d/3415.misc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/docs/admin_api/register_api.rst b/docs/admin_api/register_api.rst
new file mode 100644
index 000000000000..209cd140fdf2
--- /dev/null
+++ b/docs/admin_api/register_api.rst
@@ -0,0 +1,63 @@
+Shared-Secret Registration
+==========================
+
+This API allows for the creation of users in an administrative and
+non-interactive way. This is generally used for bootstrapping a Synapse
+instance with administrator accounts.
+
+To authenticate yourself to the server, you will need both the shared secret
+(``registration_shared_secret`` in the homeserver configuration), and a
+one-time nonce. If the registration shared secret is not configured, this API
+is not enabled.
+
+To fetch the nonce, you need to request one from the API::
+
+  > GET /_matrix/client/r0/admin/register
+
+  < {"nonce": "thisisanonce"}
+
+Once you have the nonce, you can make a ``POST`` to the same URL with a JSON
+body containing the nonce, username, password, whether they are an admin
+(optional, False by default), and a HMAC digest of the content.
+
+As an example::
+
+  > POST /_matrix/client/r0/admin/register
+  > {
+     "nonce": "thisisanonce",
+     "username": "pepper_roni",
+     "password": "pizza",
+     "admin": true,
+     "mac": "mac_digest_here"
+    }
+
+  < {
+     "access_token": "token_here",
+     "user_id": "@pepper_roni@test",
+     "home_server": "test",
+     "device_id": "device_id_here"
+    }
+
+The MAC is the hex digest output of the HMAC-SHA1 algorithm, with the key being
+the shared secret and the content being the nonce, user, password, and either
+the string "admin" or "notadmin", each separated by NULs. For an example of
+generation in Python::
+
+  import hmac, hashlib
+
+  def generate_mac(nonce, user, password, admin=False):
+
+      mac = hmac.new(
+        key=shared_secret,
+        digestmod=hashlib.sha1,
+      )
+
+      mac.update(nonce.encode('utf8'))
+      mac.update(b"\x00")
+      mac.update(user.encode('utf8'))
+      mac.update(b"\x00")
+      mac.update(password.encode('utf8'))
+      mac.update(b"\x00")
+      mac.update(b"admin" if admin else b"notadmin")
+
+      return mac.hexdigest()
diff --git a/scripts/register_new_matrix_user b/scripts/register_new_matrix_user
index 12ed20d623ed..8c3d429351c4 100755
--- a/scripts/register_new_matrix_user
+++ b/scripts/register_new_matrix_user
@@ -26,11 +26,37 @@ import yaml
 
 
 def request_registration(user, password, server_location, shared_secret, admin=False):
+    req = urllib2.Request(
+        "%s/_matrix/client/r0/admin/register" % (server_location,),
+        headers={'Content-Type': 'application/json'}
+    )
+
+    try:
+        if sys.version_info[:3] >= (2, 7, 9):
+            # As of version 2.7.9, urllib2 now checks SSL certs
+            import ssl
+            f = urllib2.urlopen(req, context=ssl.SSLContext(ssl.PROTOCOL_SSLv23))
+        else:
+            f = urllib2.urlopen(req)
+        body = f.read()
+        f.close()
+        nonce = json.loads(body)["nonce"]
+    except urllib2.HTTPError as e:
+        print "ERROR! Received %d %s" % (e.code, e.reason,)
+        if 400 <= e.code < 500:
+            if e.info().type == "application/json":
+                resp = json.load(e)
+                if "error" in resp:
+                    print resp["error"]
+        sys.exit(1)
+
     mac = hmac.new(
         key=shared_secret,
         digestmod=hashlib.sha1,
     )
 
+    mac.update(nonce)
+    mac.update("\x00")
     mac.update(user)
     mac.update("\x00")
     mac.update(password)
@@ -40,10 +66,10 @@ def request_registration(user, password, server_location, shared_secret, admin=F
     mac = mac.hexdigest()
 
     data = {
-        "user": user,
+        "nonce": nonce,
+        "username": user,
         "password": password,
         "mac": mac,
-        "type": "org.matrix.login.shared_secret",
         "admin": admin,
     }
 
@@ -52,7 +78,7 @@ def request_registration(user, password, server_location, shared_secret, admin=F
     print "Sending registration request..."
 
     req = urllib2.Request(
-        "%s/_matrix/client/api/v1/register" % (server_location,),
+        "%s/_matrix/client/r0/admin/register" % (server_location,),
         data=json.dumps(data),
         headers={'Content-Type': 'application/json'}
     )
diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py
index 2dc50e582b14..9e9c17597009 100644
--- a/synapse/rest/client/v1/admin.py
+++ b/synapse/rest/client/v1/admin.py
@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import hashlib
+import hmac
 import logging
 
 from six.moves import http_client
@@ -63,6 +65,125 @@ def on_GET(self, request, user_id):
         defer.returnValue((200, ret))
 
 
+class UserRegisterServlet(ClientV1RestServlet):
+    """
+    Attributes:
+         NONCE_TIMEOUT (int): Seconds until a generated nonce won't be accepted
+         nonces (dict[str, int]): The nonces that we will accept. A dict of
+             nonce to the time it was generated, in int seconds.
+    """
+    PATTERNS = client_path_patterns("/admin/register")
+    NONCE_TIMEOUT = 60
+
+    def __init__(self, hs):
+        super(UserRegisterServlet, self).__init__(hs)
+        self.handlers = hs.get_handlers()
+        self.reactor = hs.get_reactor()
+        self.nonces = {}
+        self.hs = hs
+
+    def _clear_old_nonces(self):
+        """
+        Clear out old nonces that are older than NONCE_TIMEOUT.
+        """
+        now = int(self.reactor.seconds())
+
+        for k, v in list(self.nonces.items()):
+            if now - v > self.NONCE_TIMEOUT:
+                del self.nonces[k]
+
+    def on_GET(self, request):
+        """
+        Generate a new nonce.
+        """
+        self._clear_old_nonces()
+
+        nonce = self.hs.get_secrets().token_hex(64)
+        self.nonces[nonce] = int(self.reactor.seconds())
+        return (200, {"nonce": nonce.encode('ascii')})
+
+    @defer.inlineCallbacks
+    def on_POST(self, request):
+        self._clear_old_nonces()
+
+        if not self.hs.config.registration_shared_secret:
+            raise SynapseError(400, "Shared secret registration is not enabled")
+
+        body = parse_json_object_from_request(request)
+
+        if "nonce" not in body:
+            raise SynapseError(
+                400, "nonce must be specified", errcode=Codes.BAD_JSON,
+            )
+
+        nonce = body["nonce"]
+
+        if nonce not in self.nonces:
+            raise SynapseError(
+                400, "unrecognised nonce",
+            )
+
+        # Delete the nonce, so it can't be reused, even if it's invalid
+        del self.nonces[nonce]
+
+        if "username" not in body:
+            raise SynapseError(
+                400, "username must be specified", errcode=Codes.BAD_JSON,
+            )
+        else:
+            if (not isinstance(body['username'], str) or len(body['username']) > 512):
+                raise SynapseError(400, "Invalid username")
+
+            username = body["username"].encode("utf-8")
+            if b"\x00" in username:
+                raise SynapseError(400, "Invalid username")
+
+        if "password" not in body:
+            raise SynapseError(
+                400, "password must be specified", errcode=Codes.BAD_JSON,
+            )
+        else:
+            if (not isinstance(body['password'], str) or len(body['password']) > 512):
+                raise SynapseError(400, "Invalid password")
+
+            password = body["password"].encode("utf-8")
+            if b"\x00" in password:
+                raise SynapseError(400, "Invalid password")
+
+        admin = body.get("admin", None)
+        got_mac = body["mac"]
+
+        want_mac = hmac.new(
+            key=self.hs.config.registration_shared_secret.encode(),
+            digestmod=hashlib.sha1,
+        )
+        want_mac.update(nonce)
+        want_mac.update(b"\x00")
+        want_mac.update(username)
+        want_mac.update(b"\x00")
+        want_mac.update(password)
+        want_mac.update(b"\x00")
+        want_mac.update(b"admin" if admin else b"notadmin")
+        want_mac = want_mac.hexdigest()
+
+        if not hmac.compare_digest(want_mac, got_mac):
+            raise SynapseError(
+                403, "HMAC incorrect",
+            )
+
+        # Reuse the parts of RegisterRestServlet to reduce code duplication
+        from synapse.rest.client.v2_alpha.register import RegisterRestServlet
+        register = RegisterRestServlet(self.hs)
+
+        (user_id, _) = yield register.registration_handler.register(
+            localpart=username.lower(), password=password, admin=bool(admin),
+            generate_token=False,
+        )
+
+        result = yield register._create_registration_details(user_id, body)
+        defer.returnValue((200, result))
+
+
 class WhoisRestServlet(ClientV1RestServlet):
     PATTERNS = client_path_patterns("/admin/whois/(?P<user_id>[^/]*)")
 
@@ -614,3 +735,4 @@ def register_servlets(hs, http_server):
     ShutdownRoomRestServlet(hs).register(http_server)
     QuarantineMediaInRoom(hs).register(http_server)
     ListMediaInRoom(hs).register(http_server)
+    UserRegisterServlet(hs).register(http_server)
diff --git a/synapse/secrets.py b/synapse/secrets.py
new file mode 100644
index 000000000000..f397daaa5edb
--- /dev/null
+++ b/synapse/secrets.py
@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Injectable secrets module for Synapse.
+
+See https://docs.python.org/3/library/secrets.html#module-secrets for the API
+used in Python 3.6, and the API emulated in Python 2.7.
+"""
+
+import six
+
+if six.PY3:
+    import secrets
+
+    def Secrets():
+        return secrets
+
+
+else:
+
+    import os
+    import binascii
+
+    class Secrets(object):
+        def token_bytes(self, nbytes=32):
+            return os.urandom(nbytes)
+
+        def token_hex(self, nbytes=32):
+            return binascii.hexlify(self.token_bytes(nbytes))
diff --git a/synapse/server.py b/synapse/server.py
index 92bea96c5c01..fd4f99225893 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -74,6 +74,7 @@
     MediaRepository,
     MediaRepositoryResource,
 )
+from synapse.secrets import Secrets
 from synapse.server_notices.server_notices_manager import ServerNoticesManager
 from synapse.server_notices.server_notices_sender import ServerNoticesSender
 from synapse.server_notices.worker_server_notices_sender import WorkerServerNoticesSender
@@ -158,6 +159,7 @@ def build_DEPENDENCY(self)
         'groups_server_handler',
         'groups_attestation_signing',
         'groups_attestation_renewer',
+        'secrets',
         'spam_checker',
         'room_member_handler',
         'federation_registry',
@@ -405,6 +407,9 @@ def build_groups_attestation_signing(self):
     def build_groups_attestation_renewer(self):
         return GroupAttestionRenewer(self)
 
+    def build_secrets(self):
+        return Secrets()
+
     def build_spam_checker(self):
         return SpamChecker(self)
 
diff --git a/tests/rest/client/v1/test_admin.py b/tests/rest/client/v1/test_admin.py
new file mode 100644
index 000000000000..8c90145601a6
--- /dev/null
+++ b/tests/rest/client/v1/test_admin.py
@@ -0,0 +1,305 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import hashlib
+import hmac
+import json
+
+from mock import Mock
+
+from synapse.http.server import JsonResource
+from synapse.rest.client.v1.admin import register_servlets
+from synapse.util import Clock
+
+from tests import unittest
+from tests.server import (
+    ThreadedMemoryReactorClock,
+    make_request,
+    render,
+    setup_test_homeserver,
+)
+
+
+class UserRegisterTestCase(unittest.TestCase):
+    def setUp(self):
+
+        self.clock = ThreadedMemoryReactorClock()
+        self.hs_clock = Clock(self.clock)
+        self.url = "/_matrix/client/r0/admin/register"
+
+        self.registration_handler = Mock()
+        self.identity_handler = Mock()
+        self.login_handler = Mock()
+        self.device_handler = Mock()
+        self.device_handler.check_device_registered = Mock(return_value="FAKE")
+
+        self.datastore = Mock(return_value=Mock())
+        self.datastore.get_current_state_deltas = Mock(return_value=[])
+
+        self.secrets = Mock()
+
+        self.hs = setup_test_homeserver(
+            http_client=None, clock=self.hs_clock, reactor=self.clock
+        )
+
+        self.hs.config.registration_shared_secret = u"shared"
+
+        self.hs.get_media_repository = Mock()
+        self.hs.get_deactivate_account_handler = Mock()
+
+        self.resource = JsonResource(self.hs)
+        register_servlets(self.hs, self.resource)
+
+    def test_disabled(self):
+        """
+        If there is no shared secret, registration through this method will be
+        prevented.
+        """
+        self.hs.config.registration_shared_secret = None
+
+        request, channel = make_request("POST", self.url, b'{}')
+        render(request, self.resource, self.clock)
+
+        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(
+            'Shared secret registration is not enabled', channel.json_body["error"]
+        )
+
+    def test_get_nonce(self):
+        """
+        Calling GET on the endpoint will return a randomised nonce, using the
+        homeserver's secrets provider.
+        """
+        secrets = Mock()
+        secrets.token_hex = Mock(return_value="abcd")
+
+        self.hs.get_secrets = Mock(return_value=secrets)
+
+        request, channel = make_request("GET", self.url)
+        render(request, self.resource, self.clock)
+
+        self.assertEqual(channel.json_body, {"nonce": "abcd"})
+
+    def test_expired_nonce(self):
+        """
+        Calling GET on the endpoint will return a randomised nonce, which will
+        only last for SALT_TIMEOUT (60s).
+        """
+        request, channel = make_request("GET", self.url)
+        render(request, self.resource, self.clock)
+        nonce = channel.json_body["nonce"]
+
+        # 59 seconds
+        self.clock.advance(59)
+
+        body = json.dumps({"nonce": nonce})
+        request, channel = make_request("POST", self.url, body.encode('utf8'))
+        render(request, self.resource, self.clock)
+
+        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual('username must be specified', channel.json_body["error"])
+
+        # 61 seconds
+        self.clock.advance(2)
+
+        request, channel = make_request("POST", self.url, body.encode('utf8'))
+        render(request, self.resource, self.clock)
+
+        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual('unrecognised nonce', channel.json_body["error"])
+
+    def test_register_incorrect_nonce(self):
+        """
+        Only the provided nonce can be used, as it's checked in the MAC.
+        """
+        request, channel = make_request("GET", self.url)
+        render(request, self.resource, self.clock)
+        nonce = channel.json_body["nonce"]
+
+        want_mac = hmac.new(key=b"shared", digestmod=hashlib.sha1)
+        want_mac.update(b"notthenonce\x00bob\x00abc123\x00admin")
+        want_mac = want_mac.hexdigest()
+
+        body = json.dumps(
+            {
+                "nonce": nonce,
+                "username": "bob",
+                "password": "abc123",
+                "admin": True,
+                "mac": want_mac,
+            }
+        ).encode('utf8')
+        request, channel = make_request("POST", self.url, body.encode('utf8'))
+        render(request, self.resource, self.clock)
+
+        self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual("HMAC incorrect", channel.json_body["error"])
+
+    def test_register_correct_nonce(self):
+        """
+        When the correct nonce is provided, and the right key is provided, the
+        user is registered.
+        """
+        request, channel = make_request("GET", self.url)
+        render(request, self.resource, self.clock)
+        nonce = channel.json_body["nonce"]
+
+        want_mac = hmac.new(key=b"shared", digestmod=hashlib.sha1)
+        want_mac.update(nonce.encode('ascii') + b"\x00bob\x00abc123\x00admin")
+        want_mac = want_mac.hexdigest()
+
+        body = json.dumps(
+            {
+                "nonce": nonce,
+                "username": "bob",
+                "password": "abc123",
+                "admin": True,
+                "mac": want_mac,
+            }
+        ).encode('utf8')
+        request, channel = make_request("POST", self.url, body.encode('utf8'))
+        render(request, self.resource, self.clock)
+
+        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual("@bob:test", channel.json_body["user_id"])
+
+    def test_nonce_reuse(self):
+        """
+        A valid unrecognised nonce.
+        """
+        request, channel = make_request("GET", self.url)
+        render(request, self.resource, self.clock)
+        nonce = channel.json_body["nonce"]
+
+        want_mac = hmac.new(key=b"shared", digestmod=hashlib.sha1)
+        want_mac.update(nonce.encode('ascii') + b"\x00bob\x00abc123\x00admin")
+        want_mac = want_mac.hexdigest()
+
+        body = json.dumps(
+            {
+                "nonce": nonce,
+                "username": "bob",
+                "password": "abc123",
+                "admin": True,
+                "mac": want_mac,
+            }
+        ).encode('utf8')
+        request, channel = make_request("POST", self.url, body.encode('utf8'))
+        render(request, self.resource, self.clock)
+
+        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual("@bob:test", channel.json_body["user_id"])
+
+        # Now, try and reuse it
+        request, channel = make_request("POST", self.url, body.encode('utf8'))
+        render(request, self.resource, self.clock)
+
+        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual('unrecognised nonce', channel.json_body["error"])
+
+    def test_missing_parts(self):
+        """
+        Synapse will complain if you don't give nonce, username, password, and
+        mac.  Admin is optional.  Additional checks are done for length and
+        type.
+        """
+        def nonce():
+            request, channel = make_request("GET", self.url)
+            render(request, self.resource, self.clock)
+            return channel.json_body["nonce"]
+
+        #
+        # Nonce check
+        #
+
+        # Must be present
+        body = json.dumps({})
+        request, channel = make_request("POST", self.url, body.encode('utf8'))
+        render(request, self.resource, self.clock)
+
+        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual('nonce must be specified', channel.json_body["error"])
+
+        #
+        # Username checks
+        #
+
+        # Must be present
+        body = json.dumps({"nonce": nonce()})
+        request, channel = make_request("POST", self.url, body.encode('utf8'))
+        render(request, self.resource, self.clock)
+
+        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual('username must be specified', channel.json_body["error"])
+
+        # Must be a string
+        body = json.dumps({"nonce": nonce(), "username": 1234})
+        request, channel = make_request("POST", self.url, body.encode('utf8'))
+        render(request, self.resource, self.clock)
+
+        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual('Invalid username', channel.json_body["error"])
+
+        # Must not have null bytes
+        body = json.dumps({"nonce": nonce(), "username": b"abcd\x00"})
+        request, channel = make_request("POST", self.url, body.encode('utf8'))
+        render(request, self.resource, self.clock)
+
+        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual('Invalid username', channel.json_body["error"])
+
+        # Must not have null bytes
+        body = json.dumps({"nonce": nonce(), "username": "a" * 1000})
+        request, channel = make_request("POST", self.url, body.encode('utf8'))
+        render(request, self.resource, self.clock)
+
+        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual('Invalid username', channel.json_body["error"])
+
+        #
+        # Username checks
+        #
+
+        # Must be present
+        body = json.dumps({"nonce": nonce(), "username": "a"})
+        request, channel = make_request("POST", self.url, body.encode('utf8'))
+        render(request, self.resource, self.clock)
+
+        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual('password must be specified', channel.json_body["error"])
+
+        # Must be a string
+        body = json.dumps({"nonce": nonce(), "username": "a", "password": 1234})
+        request, channel = make_request("POST", self.url, body.encode('utf8'))
+        render(request, self.resource, self.clock)
+
+        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual('Invalid password', channel.json_body["error"])
+
+        # Must not have null bytes
+        body = json.dumps({"nonce": nonce(), "username": "a", "password": b"abcd\x00"})
+        request, channel = make_request("POST", self.url, body.encode('utf8'))
+        render(request, self.resource, self.clock)
+
+        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual('Invalid password', channel.json_body["error"])
+
+        # Super long
+        body = json.dumps({"nonce": nonce(), "username": "a", "password": "A" * 1000})
+        request, channel = make_request("POST", self.url, body.encode('utf8'))
+        render(request, self.resource, self.clock)
+
+        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual('Invalid password', channel.json_body["error"])
diff --git a/tests/utils.py b/tests/utils.py
index e488238bb355..c3dbff85073c 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -71,6 +71,8 @@ def setup_test_homeserver(name="test", datastore=None, config=None, reactor=None
         config.user_directory_search_all_users = False
         config.user_consent_server_notice_content = None
         config.block_events_without_consent_error = None
+        config.media_storage_providers = []
+        config.auto_join_rooms = []
 
         # disable user directory updates, because they get done in the
         # background, which upsets the test runner.
@@ -136,6 +138,7 @@ def setup_test_homeserver(name="test", datastore=None, config=None, reactor=None
             database_engine=db_engine,
             room_list_handler=object(),
             tls_server_context_factory=Mock(),
+            reactor=reactor,
             **kargs
         )
 

From 3d6df846580ce6ef8769945e6990af2f44251e40 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 20 Jul 2018 13:59:55 +0100
Subject: [PATCH 084/830] Test and fix support for cancellation in Linearizer

---
 synapse/util/async.py         | 28 ++++++++++++++++++++++------
 tests/util/test_linearizer.py | 31 +++++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+), 6 deletions(-)

diff --git a/synapse/util/async.py b/synapse/util/async.py
index 5a50d9700fa3..a7094e2fb463 100644
--- a/synapse/util/async.py
+++ b/synapse/util/async.py
@@ -184,13 +184,13 @@ def __init__(self, name=None, max_count=1, clock=None):
 
         # key_to_defer is a map from the key to a 2 element list where
         # the first element is the number of things executing, and
-        # the second element is a deque of deferreds for the things blocked from
-        # executing.
+        # the second element is an OrderedDict, where the keys are deferreds for the
+        # things blocked from executing.
         self.key_to_defer = {}
 
     @defer.inlineCallbacks
     def queue(self, key):
-        entry = self.key_to_defer.setdefault(key, [0, collections.deque()])
+        entry = self.key_to_defer.setdefault(key, [0, collections.OrderedDict()])
 
         # If the number of things executing is greater than the maximum
         # then add a deferred to the list of blocked items
@@ -198,12 +198,28 @@ def queue(self, key):
         # this item so that it can continue executing.
         if entry[0] >= self.max_count:
             new_defer = defer.Deferred()
-            entry[1].append(new_defer)
+            entry[1][new_defer] = 1
 
             logger.info(
                 "Waiting to acquire linearizer lock %r for key %r", self.name, key,
             )
-            yield make_deferred_yieldable(new_defer)
+            try:
+                yield make_deferred_yieldable(new_defer)
+            except Exception as e:
+                if isinstance(e, CancelledError):
+                    logger.info(
+                        "Cancelling wait for linearizer lock %r for key %r",
+                        self.name, key,
+                    )
+                else:
+                    logger.warn(
+                        "Unexpected exception waiting for linearizer lock %r for key %r",
+                        self.name, key,
+                    )
+
+                # we just have to take ourselves back out of the queue.
+                del entry[1][new_defer]
+                raise
 
             logger.info("Acquired linearizer lock %r for key %r", self.name, key)
             entry[0] += 1
@@ -238,7 +254,7 @@ def _ctx_manager():
                 entry[0] -= 1
 
                 if entry[1]:
-                    next_def = entry[1].popleft()
+                    (next_def, _) = entry[1].popitem(last=False)
 
                     # we need to run the next thing in the sentinel context.
                     with PreserveLoggingContext():
diff --git a/tests/util/test_linearizer.py b/tests/util/test_linearizer.py
index c9563124f92b..4729bd5a0a15 100644
--- a/tests/util/test_linearizer.py
+++ b/tests/util/test_linearizer.py
@@ -17,6 +17,7 @@
 from six.moves import range
 
 from twisted.internet import defer, reactor
+from twisted.internet.defer import CancelledError
 
 from synapse.util import Clock, logcontext
 from synapse.util.async import Linearizer
@@ -112,3 +113,33 @@ def test_multiple_entries(self):
         d6 = limiter.queue(key)
         with (yield d6):
             pass
+
+    @defer.inlineCallbacks
+    def test_cancellation(self):
+        linearizer = Linearizer()
+
+        key = object()
+
+        d1 = linearizer.queue(key)
+        cm1 = yield d1
+
+        d2 = linearizer.queue(key)
+        self.assertFalse(d2.called)
+
+        d3 = linearizer.queue(key)
+        self.assertFalse(d3.called)
+
+        d2.cancel()
+
+        with cm1:
+            pass
+
+        self.assertTrue(d2.called)
+        try:
+            yield d2
+            self.fail("Expected d2 to raise CancelledError")
+        except CancelledError:
+            pass
+
+        with (yield d3):
+            pass

From 5c30cb709af1295b2285d802d1e40b91382e7af0 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 20 Jul 2018 14:01:36 +0100
Subject: [PATCH 085/830] Changelog

---
 changelog.d/3572.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3572.misc

diff --git a/changelog.d/3572.misc b/changelog.d/3572.misc
new file mode 100644
index 000000000000..8908324e6803
--- /dev/null
+++ b/changelog.d/3572.misc
@@ -0,0 +1 @@
+Merge Linearizer and Limiter

From 0ecf68aedc09f4037208b613b692a8a98c78b3ea Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 20 Jul 2018 15:30:59 +0100
Subject: [PATCH 086/830] Move check_in_room_or_world_readable to Auth

---
 synapse/api/auth.py         | 34 +++++++++++++++++++++++++++++++
 synapse/handlers/message.py | 40 ++++++-------------------------------
 2 files changed, 40 insertions(+), 34 deletions(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index bc629832d90a..bf9efb170a23 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -737,3 +737,37 @@ def get_access_token_from_request(request, token_not_found_http_status=401):
                 )
 
             return query_params[0]
+
+    @defer.inlineCallbacks
+    def check_in_room_or_world_readable(self, room_id, user_id):
+        """Checks that the user is or was in the room or the room is world
+        readable. If it isn't then an exception is raised.
+
+        Returns:
+            Deferred[tuple[str, str|None]]: Resolves to the current membership of
+            the user in the room and the membership event ID of the user. If
+            the user is not in the room and never has been, then
+            `(Membership.JOIN, None)` is returned.
+        """
+
+        try:
+            # check_user_was_in_room will return the most recent membership
+            # event for the user if:
+            #  * The user is a non-guest user, and was ever in the room
+            #  * The user is a guest user, and has joined the room
+            # else it will throw.
+            member_event = yield self.check_user_was_in_room(room_id, user_id)
+            defer.returnValue((member_event.membership, member_event.event_id))
+        except AuthError:
+            visibility = yield self.state.get_current_state(
+                room_id, EventTypes.RoomHistoryVisibility, ""
+            )
+            if (
+                visibility and
+                visibility.content["history_visibility"] == "world_readable"
+            ):
+                defer.returnValue((Membership.JOIN, None))
+                return
+            raise AuthError(
+                403, "Guest access not allowed", errcode=Codes.GUEST_ACCESS_FORBIDDEN
+            )
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 3c6f9860d53e..c1489cd06619 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -97,7 +97,7 @@ def get_room_data(self, user_id=None, room_id=None,
         Raises:
             SynapseError if something went wrong.
         """
-        membership, membership_event_id = yield self._check_in_room_or_world_readable(
+        membership, membership_event_id = yield self.auth.check_in_room_or_world_readable(
             room_id, user_id
         )
 
@@ -114,31 +114,6 @@ def get_room_data(self, user_id=None, room_id=None,
 
         defer.returnValue(data)
 
-    @defer.inlineCallbacks
-    def _check_in_room_or_world_readable(self, room_id, user_id):
-        try:
-            # check_user_was_in_room will return the most recent membership
-            # event for the user if:
-            #  * The user is a non-guest user, and was ever in the room
-            #  * The user is a guest user, and has joined the room
-            # else it will throw.
-            member_event = yield self.auth.check_user_was_in_room(room_id, user_id)
-            defer.returnValue((member_event.membership, member_event.event_id))
-            return
-        except AuthError:
-            visibility = yield self.state.get_current_state(
-                room_id, EventTypes.RoomHistoryVisibility, ""
-            )
-            if (
-                visibility and
-                visibility.content["history_visibility"] == "world_readable"
-            ):
-                defer.returnValue((Membership.JOIN, None))
-                return
-            raise AuthError(
-                403, "Guest access not allowed", errcode=Codes.GUEST_ACCESS_FORBIDDEN
-            )
-
     @defer.inlineCallbacks
     def get_state_events(self, user_id, room_id, is_guest=False):
         """Retrieve all state events for a given room. If the user is
@@ -151,7 +126,7 @@ def get_state_events(self, user_id, room_id, is_guest=False):
         Returns:
             A list of dicts representing state events. [{}, {}, {}]
         """
-        membership, membership_event_id = yield self._check_in_room_or_world_readable(
+        membership, membership_event_id = yield self.auth.check_in_room_or_world_readable(
             room_id, user_id
         )
 
@@ -184,7 +159,7 @@ def get_joined_members(self, requester, room_id):
         if not requester.app_service:
             # We check AS auth after fetching the room membership, as it
             # requires us to pull out all joined members anyway.
-            membership, _ = yield self._check_in_room_or_world_readable(
+            membership, _ = yield self.auth.check_in_room_or_world_readable(
                 room_id, user_id
             )
             if membership != Membership.JOIN:
@@ -214,19 +189,16 @@ def get_joined_members(self, requester, room_id):
         })
 
 
-class PaginationHandler(MessageHandler):
+class PaginationHandler(object):
     """Handles pagination and purge history requests.
 
     These are in the same handler due to the fact we need to block clients
     paginating during a purge.
-
-    This subclasses MessageHandler to get at _check_in_room_or_world_readable
     """
 
     def __init__(self, hs):
-        super(PaginationHandler, self).__init__(hs)
-
         self.hs = hs
+        self.auth = hs.get_auth()
         self.store = hs.get_datastore()
         self.clock = hs.get_clock()
 
@@ -349,7 +321,7 @@ def get_messages(self, requester, room_id=None, pagin_config=None,
         source_config = pagin_config.get_source_config("room")
 
         with (yield self.pagination_lock.read(room_id)):
-            membership, member_event_id = yield self._check_in_room_or_world_readable(
+            membership, member_event_id = yield self.auth.check_in_room_or_world_readable(
                 room_id, user_id
             )
 

From 5c88bb722f57af1c77f34d77152689425ab95eba Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 20 Jul 2018 15:32:23 +0100
Subject: [PATCH 087/830] Move PaginationHandler to its own file

---
 synapse/handlers/message.py    | 242 +-----------------------------
 synapse/handlers/pagination.py | 265 +++++++++++++++++++++++++++++++++
 synapse/server.py              |   7 +-
 3 files changed, 269 insertions(+), 245 deletions(-)
 create mode 100644 synapse/handlers/pagination.py

diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index c1489cd06619..ba3c4642bcbc 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -23,7 +23,6 @@
 
 from twisted.internet import defer
 from twisted.internet.defer import succeed
-from twisted.python.failure import Failure
 
 from synapse.api.constants import MAX_DEPTH, EventTypes, Membership
 from synapse.api.errors import AuthError, Codes, ConsentNotGivenError, SynapseError
@@ -32,49 +31,17 @@
 from synapse.events.utils import serialize_event
 from synapse.events.validator import EventValidator
 from synapse.replication.http.send_event import send_event_to_master
-from synapse.types import RoomAlias, RoomStreamToken, UserID
-from synapse.util.async import Limiter, ReadWriteLock
+from synapse.types import RoomAlias, UserID
+from synapse.util.async import Limiter
 from synapse.util.frozenutils import frozendict_json_encoder
 from synapse.util.logcontext import run_in_background
 from synapse.util.metrics import measure_func
-from synapse.util.stringutils import random_string
-from synapse.visibility import filter_events_for_client
 
 from ._base import BaseHandler
 
 logger = logging.getLogger(__name__)
 
 
-class PurgeStatus(object):
-    """Object tracking the status of a purge request
-
-    This class contains information on the progress of a purge request, for
-    return by get_purge_status.
-
-    Attributes:
-        status (int): Tracks whether this request has completed. One of
-            STATUS_{ACTIVE,COMPLETE,FAILED}
-    """
-
-    STATUS_ACTIVE = 0
-    STATUS_COMPLETE = 1
-    STATUS_FAILED = 2
-
-    STATUS_TEXT = {
-        STATUS_ACTIVE: "active",
-        STATUS_COMPLETE: "complete",
-        STATUS_FAILED: "failed",
-    }
-
-    def __init__(self):
-        self.status = PurgeStatus.STATUS_ACTIVE
-
-    def asdict(self):
-        return {
-            "status": PurgeStatus.STATUS_TEXT[self.status]
-        }
-
-
 class MessageHandler(object):
     """Contains some read only APIs to get state about a room
     """
@@ -189,211 +156,6 @@ def get_joined_members(self, requester, room_id):
         })
 
 
-class PaginationHandler(object):
-    """Handles pagination and purge history requests.
-
-    These are in the same handler due to the fact we need to block clients
-    paginating during a purge.
-    """
-
-    def __init__(self, hs):
-        self.hs = hs
-        self.auth = hs.get_auth()
-        self.store = hs.get_datastore()
-        self.clock = hs.get_clock()
-
-        self.pagination_lock = ReadWriteLock()
-        self._purges_in_progress_by_room = set()
-        # map from purge id to PurgeStatus
-        self._purges_by_id = {}
-
-    def start_purge_history(self, room_id, token,
-                            delete_local_events=False):
-        """Start off a history purge on a room.
-
-        Args:
-            room_id (str): The room to purge from
-
-            token (str): topological token to delete events before
-            delete_local_events (bool): True to delete local events as well as
-                remote ones
-
-        Returns:
-            str: unique ID for this purge transaction.
-        """
-        if room_id in self._purges_in_progress_by_room:
-            raise SynapseError(
-                400,
-                "History purge already in progress for %s" % (room_id, ),
-            )
-
-        purge_id = random_string(16)
-
-        # we log the purge_id here so that it can be tied back to the
-        # request id in the log lines.
-        logger.info("[purge] starting purge_id %s", purge_id)
-
-        self._purges_by_id[purge_id] = PurgeStatus()
-        run_in_background(
-            self._purge_history,
-            purge_id, room_id, token, delete_local_events,
-        )
-        return purge_id
-
-    @defer.inlineCallbacks
-    def _purge_history(self, purge_id, room_id, token,
-                       delete_local_events):
-        """Carry out a history purge on a room.
-
-        Args:
-            purge_id (str): The id for this purge
-            room_id (str): The room to purge from
-            token (str): topological token to delete events before
-            delete_local_events (bool): True to delete local events as well as
-                remote ones
-
-        Returns:
-            Deferred
-        """
-        self._purges_in_progress_by_room.add(room_id)
-        try:
-            with (yield self.pagination_lock.write(room_id)):
-                yield self.store.purge_history(
-                    room_id, token, delete_local_events,
-                )
-            logger.info("[purge] complete")
-            self._purges_by_id[purge_id].status = PurgeStatus.STATUS_COMPLETE
-        except Exception:
-            logger.error("[purge] failed: %s", Failure().getTraceback().rstrip())
-            self._purges_by_id[purge_id].status = PurgeStatus.STATUS_FAILED
-        finally:
-            self._purges_in_progress_by_room.discard(room_id)
-
-            # remove the purge from the list 24 hours after it completes
-            def clear_purge():
-                del self._purges_by_id[purge_id]
-            self.hs.get_reactor().callLater(24 * 3600, clear_purge)
-
-    def get_purge_status(self, purge_id):
-        """Get the current status of an active purge
-
-        Args:
-            purge_id (str): purge_id returned by start_purge_history
-
-        Returns:
-            PurgeStatus|None
-        """
-        return self._purges_by_id.get(purge_id)
-
-    @defer.inlineCallbacks
-    def get_messages(self, requester, room_id=None, pagin_config=None,
-                     as_client_event=True, event_filter=None):
-        """Get messages in a room.
-
-        Args:
-            requester (Requester): The user requesting messages.
-            room_id (str): The room they want messages from.
-            pagin_config (synapse.api.streams.PaginationConfig): The pagination
-                config rules to apply, if any.
-            as_client_event (bool): True to get events in client-server format.
-            event_filter (Filter): Filter to apply to results or None
-        Returns:
-            dict: Pagination API results
-        """
-        user_id = requester.user.to_string()
-
-        if pagin_config.from_token:
-            room_token = pagin_config.from_token.room_key
-        else:
-            pagin_config.from_token = (
-                yield self.hs.get_event_sources().get_current_token_for_room(
-                    room_id=room_id
-                )
-            )
-            room_token = pagin_config.from_token.room_key
-
-        room_token = RoomStreamToken.parse(room_token)
-
-        pagin_config.from_token = pagin_config.from_token.copy_and_replace(
-            "room_key", str(room_token)
-        )
-
-        source_config = pagin_config.get_source_config("room")
-
-        with (yield self.pagination_lock.read(room_id)):
-            membership, member_event_id = yield self.auth.check_in_room_or_world_readable(
-                room_id, user_id
-            )
-
-            if source_config.direction == 'b':
-                # if we're going backwards, we might need to backfill. This
-                # requires that we have a topo token.
-                if room_token.topological:
-                    max_topo = room_token.topological
-                else:
-                    max_topo = yield self.store.get_max_topological_token(
-                        room_id, room_token.stream
-                    )
-
-                if membership == Membership.LEAVE:
-                    # If they have left the room then clamp the token to be before
-                    # they left the room, to save the effort of loading from the
-                    # database.
-                    leave_token = yield self.store.get_topological_token_for_event(
-                        member_event_id
-                    )
-                    leave_token = RoomStreamToken.parse(leave_token)
-                    if leave_token.topological < max_topo:
-                        source_config.from_key = str(leave_token)
-
-                yield self.hs.get_handlers().federation_handler.maybe_backfill(
-                    room_id, max_topo
-                )
-
-            events, next_key = yield self.store.paginate_room_events(
-                room_id=room_id,
-                from_key=source_config.from_key,
-                to_key=source_config.to_key,
-                direction=source_config.direction,
-                limit=source_config.limit,
-                event_filter=event_filter,
-            )
-
-            next_token = pagin_config.from_token.copy_and_replace(
-                "room_key", next_key
-            )
-
-        if not events:
-            defer.returnValue({
-                "chunk": [],
-                "start": pagin_config.from_token.to_string(),
-                "end": next_token.to_string(),
-            })
-
-        if event_filter:
-            events = event_filter.filter(events)
-
-        events = yield filter_events_for_client(
-            self.store,
-            user_id,
-            events,
-            is_peeking=(member_event_id is None),
-        )
-
-        time_now = self.clock.time_msec()
-
-        chunk = {
-            "chunk": [
-                serialize_event(e, time_now, as_client_event)
-                for e in events
-            ],
-            "start": pagin_config.from_token.to_string(),
-            "end": next_token.to_string(),
-        }
-
-        defer.returnValue(chunk)
-
-
 class EventCreationHandler(object):
     def __init__(self, hs):
         self.hs = hs
diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py
new file mode 100644
index 000000000000..b2849783ed1b
--- /dev/null
+++ b/synapse/handlers/pagination.py
@@ -0,0 +1,265 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014 - 2016 OpenMarket Ltd
+# Copyright 2017 - 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+
+from twisted.internet import defer
+from twisted.python.failure import Failure
+
+from synapse.api.constants import Membership
+from synapse.api.errors import SynapseError
+from synapse.events.utils import serialize_event
+from synapse.types import RoomStreamToken
+from synapse.util.async import ReadWriteLock
+from synapse.util.logcontext import run_in_background
+from synapse.util.stringutils import random_string
+from synapse.visibility import filter_events_for_client
+
+logger = logging.getLogger(__name__)
+
+
+class PurgeStatus(object):
+    """Object tracking the status of a purge request
+
+    This class contains information on the progress of a purge request, for
+    return by get_purge_status.
+
+    Attributes:
+        status (int): Tracks whether this request has completed. One of
+            STATUS_{ACTIVE,COMPLETE,FAILED}
+    """
+
+    STATUS_ACTIVE = 0
+    STATUS_COMPLETE = 1
+    STATUS_FAILED = 2
+
+    STATUS_TEXT = {
+        STATUS_ACTIVE: "active",
+        STATUS_COMPLETE: "complete",
+        STATUS_FAILED: "failed",
+    }
+
+    def __init__(self):
+        self.status = PurgeStatus.STATUS_ACTIVE
+
+    def asdict(self):
+        return {
+            "status": PurgeStatus.STATUS_TEXT[self.status]
+        }
+
+
+class PaginationHandler(object):
+    """Handles pagination and purge history requests.
+
+    These are in the same handler due to the fact we need to block clients
+    paginating during a purge.
+    """
+
+    def __init__(self, hs):
+        self.hs = hs
+        self.auth = hs.get_auth()
+        self.store = hs.get_datastore()
+        self.clock = hs.get_clock()
+
+        self.pagination_lock = ReadWriteLock()
+        self._purges_in_progress_by_room = set()
+        # map from purge id to PurgeStatus
+        self._purges_by_id = {}
+
+    def start_purge_history(self, room_id, token,
+                            delete_local_events=False):
+        """Start off a history purge on a room.
+
+        Args:
+            room_id (str): The room to purge from
+
+            token (str): topological token to delete events before
+            delete_local_events (bool): True to delete local events as well as
+                remote ones
+
+        Returns:
+            str: unique ID for this purge transaction.
+        """
+        if room_id in self._purges_in_progress_by_room:
+            raise SynapseError(
+                400,
+                "History purge already in progress for %s" % (room_id, ),
+            )
+
+        purge_id = random_string(16)
+
+        # we log the purge_id here so that it can be tied back to the
+        # request id in the log lines.
+        logger.info("[purge] starting purge_id %s", purge_id)
+
+        self._purges_by_id[purge_id] = PurgeStatus()
+        run_in_background(
+            self._purge_history,
+            purge_id, room_id, token, delete_local_events,
+        )
+        return purge_id
+
+    @defer.inlineCallbacks
+    def _purge_history(self, purge_id, room_id, token,
+                       delete_local_events):
+        """Carry out a history purge on a room.
+
+        Args:
+            purge_id (str): The id for this purge
+            room_id (str): The room to purge from
+            token (str): topological token to delete events before
+            delete_local_events (bool): True to delete local events as well as
+                remote ones
+
+        Returns:
+            Deferred
+        """
+        self._purges_in_progress_by_room.add(room_id)
+        try:
+            with (yield self.pagination_lock.write(room_id)):
+                yield self.store.purge_history(
+                    room_id, token, delete_local_events,
+                )
+            logger.info("[purge] complete")
+            self._purges_by_id[purge_id].status = PurgeStatus.STATUS_COMPLETE
+        except Exception:
+            logger.error("[purge] failed: %s", Failure().getTraceback().rstrip())
+            self._purges_by_id[purge_id].status = PurgeStatus.STATUS_FAILED
+        finally:
+            self._purges_in_progress_by_room.discard(room_id)
+
+            # remove the purge from the list 24 hours after it completes
+            def clear_purge():
+                del self._purges_by_id[purge_id]
+            self.hs.get_reactor().callLater(24 * 3600, clear_purge)
+
+    def get_purge_status(self, purge_id):
+        """Get the current status of an active purge
+
+        Args:
+            purge_id (str): purge_id returned by start_purge_history
+
+        Returns:
+            PurgeStatus|None
+        """
+        return self._purges_by_id.get(purge_id)
+
+    @defer.inlineCallbacks
+    def get_messages(self, requester, room_id=None, pagin_config=None,
+                     as_client_event=True, event_filter=None):
+        """Get messages in a room.
+
+        Args:
+            requester (Requester): The user requesting messages.
+            room_id (str): The room they want messages from.
+            pagin_config (synapse.api.streams.PaginationConfig): The pagination
+                config rules to apply, if any.
+            as_client_event (bool): True to get events in client-server format.
+            event_filter (Filter): Filter to apply to results or None
+        Returns:
+            dict: Pagination API results
+        """
+        user_id = requester.user.to_string()
+
+        if pagin_config.from_token:
+            room_token = pagin_config.from_token.room_key
+        else:
+            pagin_config.from_token = (
+                yield self.hs.get_event_sources().get_current_token_for_room(
+                    room_id=room_id
+                )
+            )
+            room_token = pagin_config.from_token.room_key
+
+        room_token = RoomStreamToken.parse(room_token)
+
+        pagin_config.from_token = pagin_config.from_token.copy_and_replace(
+            "room_key", str(room_token)
+        )
+
+        source_config = pagin_config.get_source_config("room")
+
+        with (yield self.pagination_lock.read(room_id)):
+            membership, member_event_id = yield self.auth.check_in_room_or_world_readable(
+                room_id, user_id
+            )
+
+            if source_config.direction == 'b':
+                # if we're going backwards, we might need to backfill. This
+                # requires that we have a topo token.
+                if room_token.topological:
+                    max_topo = room_token.topological
+                else:
+                    max_topo = yield self.store.get_max_topological_token(
+                        room_id, room_token.stream
+                    )
+
+                if membership == Membership.LEAVE:
+                    # If they have left the room then clamp the token to be before
+                    # they left the room, to save the effort of loading from the
+                    # database.
+                    leave_token = yield self.store.get_topological_token_for_event(
+                        member_event_id
+                    )
+                    leave_token = RoomStreamToken.parse(leave_token)
+                    if leave_token.topological < max_topo:
+                        source_config.from_key = str(leave_token)
+
+                yield self.hs.get_handlers().federation_handler.maybe_backfill(
+                    room_id, max_topo
+                )
+
+            events, next_key = yield self.store.paginate_room_events(
+                room_id=room_id,
+                from_key=source_config.from_key,
+                to_key=source_config.to_key,
+                direction=source_config.direction,
+                limit=source_config.limit,
+                event_filter=event_filter,
+            )
+
+            next_token = pagin_config.from_token.copy_and_replace(
+                "room_key", next_key
+            )
+
+        if not events:
+            defer.returnValue({
+                "chunk": [],
+                "start": pagin_config.from_token.to_string(),
+                "end": next_token.to_string(),
+            })
+
+        if event_filter:
+            events = event_filter.filter(events)
+
+        events = yield filter_events_for_client(
+            self.store,
+            user_id,
+            events,
+            is_peeking=(member_event_id is None),
+        )
+
+        time_now = self.clock.time_msec()
+
+        chunk = {
+            "chunk": [
+                serialize_event(e, time_now, as_client_event)
+                for e in events
+            ],
+            "start": pagin_config.from_token.to_string(),
+            "end": next_token.to_string(),
+        }
+
+        defer.returnValue(chunk)
diff --git a/synapse/server.py b/synapse/server.py
index a24ea158df4b..83eacccc295b 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -52,11 +52,8 @@
 from synapse.handlers.events import EventHandler, EventStreamHandler
 from synapse.handlers.groups_local import GroupsLocalHandler
 from synapse.handlers.initial_sync import InitialSyncHandler
-from synapse.handlers.message import (
-    EventCreationHandler,
-    MessageHandler,
-    PaginationHandler,
-)
+from synapse.handlers.message import EventCreationHandler, MessageHandler
+from synapse.handlers.pagination import PaginationHandler
 from synapse.handlers.presence import PresenceHandler
 from synapse.handlers.profile import ProfileHandler
 from synapse.handlers.read_marker import ReadMarkerHandler

From 3132b89f12f0386558045683ad198f090b0e2c90 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Sat, 21 Jul 2018 15:47:18 +1000
Subject: [PATCH 088/830] Make the rest of the .iterwhatever go away (#3562)

---
 changelog.d/3562.misc          |  0
 synapse/app/homeserver.py      |  6 ++++--
 synapse/app/synctl.py          |  4 +++-
 synapse/events/snapshot.py     |  4 +++-
 synapse/handlers/federation.py | 18 +++++++++---------
 synapse/state.py               |  6 +++---
 synapse/visibility.py          | 19 ++++++++++---------
 tests/test_federation.py       |  3 +--
 8 files changed, 33 insertions(+), 27 deletions(-)
 create mode 100644 changelog.d/3562.misc

diff --git a/changelog.d/3562.misc b/changelog.d/3562.misc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 14e6dca5225d..2ad1beb8d88c 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -18,6 +18,8 @@
 import os
 import sys
 
+from six import iteritems
+
 from twisted.application import service
 from twisted.internet import defer, reactor
 from twisted.web.resource import EncodingResourceWrapper, NoResource
@@ -442,7 +444,7 @@ def phone_stats_home():
         stats["total_nonbridged_users"] = total_nonbridged_users
 
         daily_user_type_results = yield hs.get_datastore().count_daily_user_type()
-        for name, count in daily_user_type_results.iteritems():
+        for name, count in iteritems(daily_user_type_results):
             stats["daily_user_type_" + name] = count
 
         room_count = yield hs.get_datastore().get_room_count()
@@ -453,7 +455,7 @@ def phone_stats_home():
         stats["daily_messages"] = yield hs.get_datastore().count_daily_messages()
 
         r30_results = yield hs.get_datastore().count_r30_users()
-        for name, count in r30_results.iteritems():
+        for name, count in iteritems(r30_results):
             stats["r30_users_" + name] = count
 
         daily_sent_messages = yield hs.get_datastore().count_daily_sent_messages()
diff --git a/synapse/app/synctl.py b/synapse/app/synctl.py
index 68acc15a9a1b..d658f967ba87 100755
--- a/synapse/app/synctl.py
+++ b/synapse/app/synctl.py
@@ -25,6 +25,8 @@
 import sys
 import time
 
+from six import iteritems
+
 import yaml
 
 SYNAPSE = [sys.executable, "-B", "-m", "synapse.app.homeserver"]
@@ -173,7 +175,7 @@ def main():
         os.environ["SYNAPSE_CACHE_FACTOR"] = str(cache_factor)
 
     cache_factors = config.get("synctl_cache_factors", {})
-    for cache_name, factor in cache_factors.iteritems():
+    for cache_name, factor in iteritems(cache_factors):
         os.environ["SYNAPSE_CACHE_FACTOR_" + cache_name.upper()] = str(factor)
 
     worker_configfiles = []
diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py
index bcd9bb594670..f83a1581a6f1 100644
--- a/synapse/events/snapshot.py
+++ b/synapse/events/snapshot.py
@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from six import iteritems
+
 from frozendict import frozendict
 
 from twisted.internet import defer
@@ -159,7 +161,7 @@ def _encode_state_dict(state_dict):
 
     return [
         (etype, state_key, v)
-        for (etype, state_key), v in state_dict.iteritems()
+        for (etype, state_key), v in iteritems(state_dict)
     ]
 
 
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 65f6041b1057..a6d391c4e889 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -21,8 +21,8 @@
 import sys
 
 import six
-from six import iteritems
-from six.moves import http_client
+from six import iteritems, itervalues
+from six.moves import http_client, zip
 
 from signedjson.key import decode_verify_key_bytes
 from signedjson.sign import verify_signed_json
@@ -731,7 +731,7 @@ def get_domains_from_state(state):
             """
             joined_users = [
                 (state_key, int(event.depth))
-                for (e_type, state_key), event in state.iteritems()
+                for (e_type, state_key), event in iteritems(state)
                 if e_type == EventTypes.Member
                 and event.membership == Membership.JOIN
             ]
@@ -748,7 +748,7 @@ def get_domains_from_state(state):
                 except Exception:
                     pass
 
-            return sorted(joined_domains.iteritems(), key=lambda d: d[1])
+            return sorted(joined_domains.items(), key=lambda d: d[1])
 
         curr_domains = get_domains_from_state(curr_state)
 
@@ -811,7 +811,7 @@ def try_backfill(domains):
         tried_domains = set(likely_domains)
         tried_domains.add(self.server_name)
 
-        event_ids = list(extremities.iterkeys())
+        event_ids = list(extremities.keys())
 
         logger.debug("calling resolve_state_groups in _maybe_backfill")
         resolve = logcontext.preserve_fn(
@@ -827,15 +827,15 @@ def try_backfill(domains):
         states = dict(zip(event_ids, [s.state for s in states]))
 
         state_map = yield self.store.get_events(
-            [e_id for ids in states.itervalues() for e_id in ids.itervalues()],
+            [e_id for ids in itervalues(states) for e_id in itervalues(ids)],
             get_prev_content=False
         )
         states = {
             key: {
                 k: state_map[e_id]
-                for k, e_id in state_dict.iteritems()
+                for k, e_id in iteritems(state_dict)
                 if e_id in state_map
-            } for key, state_dict in states.iteritems()
+            } for key, state_dict in iteritems(states)
         }
 
         for e_id, _ in sorted_extremeties_tuple:
@@ -1515,7 +1515,7 @@ def _handle_new_events(self, origin, event_infos, backfilled=False):
         yield self.store.persist_events(
             [
                 (ev_info["event"], context)
-                for ev_info, context in itertools.izip(event_infos, contexts)
+                for ev_info, context in zip(event_infos, contexts)
             ],
             backfilled=backfilled,
         )
diff --git a/synapse/state.py b/synapse/state.py
index 15a593d41ca7..504caae2f740 100644
--- a/synapse/state.py
+++ b/synapse/state.py
@@ -18,7 +18,7 @@
 import logging
 from collections import namedtuple
 
-from six import iteritems, itervalues
+from six import iteritems, iterkeys, itervalues
 
 from frozendict import frozendict
 
@@ -647,7 +647,7 @@ def resolve_events_with_factory(state_sets, event_map, state_map_factory):
         for event_id in event_ids
     )
     if event_map is not None:
-        needed_events -= set(event_map.iterkeys())
+        needed_events -= set(iterkeys(event_map))
 
     logger.info("Asking for %d conflicted events", len(needed_events))
 
@@ -668,7 +668,7 @@ def resolve_events_with_factory(state_sets, event_map, state_map_factory):
     new_needed_events = set(itervalues(auth_events))
     new_needed_events -= needed_events
     if event_map is not None:
-        new_needed_events -= set(event_map.iterkeys())
+        new_needed_events -= set(iterkeys(event_map))
 
     logger.info("Asking for %d auth events", len(new_needed_events))
 
diff --git a/synapse/visibility.py b/synapse/visibility.py
index 9b97ea2b83a1..ba0499a022e3 100644
--- a/synapse/visibility.py
+++ b/synapse/visibility.py
@@ -12,11 +12,12 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import itertools
+
 import logging
 import operator
 
-import six
+from six import iteritems, itervalues
+from six.moves import map
 
 from twisted.internet import defer
 
@@ -221,7 +222,7 @@ def allowed(event):
         return event
 
     # check each event: gives an iterable[None|EventBase]
-    filtered_events = itertools.imap(allowed, events)
+    filtered_events = map(allowed, events)
 
     # remove the None entries
     filtered_events = filter(operator.truth, filtered_events)
@@ -261,7 +262,7 @@ def redact_disallowed(event, state):
                 # membership states for the requesting server to determine
                 # if the server is either in the room or has been invited
                 # into the room.
-                for ev in state.itervalues():
+                for ev in itervalues(state):
                     if ev.type != EventTypes.Member:
                         continue
                     try:
@@ -295,7 +296,7 @@ def redact_disallowed(event, state):
     )
 
     visibility_ids = set()
-    for sids in event_to_state_ids.itervalues():
+    for sids in itervalues(event_to_state_ids):
         hist = sids.get((EventTypes.RoomHistoryVisibility, ""))
         if hist:
             visibility_ids.add(hist)
@@ -308,7 +309,7 @@ def redact_disallowed(event, state):
         event_map = yield store.get_events(visibility_ids)
         all_open = all(
             e.content.get("history_visibility") in (None, "shared", "world_readable")
-            for e in event_map.itervalues()
+            for e in itervalues(event_map)
         )
 
     if all_open:
@@ -346,7 +347,7 @@ def redact_disallowed(event, state):
     #
     state_key_to_event_id_set = {
         e
-        for key_to_eid in six.itervalues(event_to_state_ids)
+        for key_to_eid in itervalues(event_to_state_ids)
         for e in key_to_eid.items()
     }
 
@@ -369,10 +370,10 @@ def include(typ, state_key):
     event_to_state = {
         e_id: {
             key: event_map[inner_e_id]
-            for key, inner_e_id in key_to_eid.iteritems()
+            for key, inner_e_id in iteritems(key_to_eid)
             if inner_e_id in event_map
         }
-        for e_id, key_to_eid in event_to_state_ids.iteritems()
+        for e_id, key_to_eid in iteritems(event_to_state_ids)
     }
 
     defer.returnValue([
diff --git a/tests/test_federation.py b/tests/test_federation.py
index 159a136971e1..f40ff29b52af 100644
--- a/tests/test_federation.py
+++ b/tests/test_federation.py
@@ -137,7 +137,6 @@ def post_json(destination, path, data, headers=None, timeout=0):
         )
         self.assertEqual(self.successResultOf(extrem)[0], "$join:test.serv")
 
-    @unittest.DEBUG
     def test_cant_hide_past_history(self):
         """
         If you send a message, you must be able to provide the direct
@@ -178,7 +177,7 @@ def get_json(destination, path, args, headers=None):
                             for x, y in d.items()
                             if x == ("m.room.member", "@us:test")
                         ],
-                        "auth_chain_ids": d.values(),
+                        "auth_chain_ids": list(d.values()),
                     }
                 )
 

From c1bf2b587eaa718e28a33f76a7b5f6e288255fca Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Mon, 23 Jul 2018 09:56:23 +0100
Subject: [PATCH 089/830] add trailing comma

---
 synapse/storage/end_to_end_keys.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/storage/end_to_end_keys.py b/synapse/storage/end_to_end_keys.py
index ffe4d7235ab8..523b4360c3cb 100644
--- a/synapse/storage/end_to_end_keys.py
+++ b/synapse/storage/end_to_end_keys.py
@@ -66,7 +66,7 @@ def _set_e2e_device_keys_txn(txn):
     @defer.inlineCallbacks
     def get_e2e_device_keys(
         self, query_list, include_all_devices=False,
-        include_deleted_devices=False
+        include_deleted_devices=False,
     ):
         """Fetch a list of device keys.
         Args:

From acbfdc3442f706150c6312e534ca4ecec8548582 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 23 Jul 2018 12:17:16 +0100
Subject: [PATCH 090/830] Refcator EventContext to accept state during init

---
 synapse/events/snapshot.py | 48 ++++++++++---------
 synapse/state.py           | 97 +++++++++++++++++++++-----------------
 2 files changed, 82 insertions(+), 63 deletions(-)

diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py
index f83a1581a6f1..fbbe8dd49062 100644
--- a/synapse/events/snapshot.py
+++ b/synapse/events/snapshot.py
@@ -60,22 +60,22 @@ class EventContext(object):
         "app_service",
     ]
 
-    def __init__(self):
+    def __init__(self, state_group, current_state_ids, prev_state_ids,
+                 prev_group=None, delta_ids=None):
         # The current state including the current event
-        self.current_state_ids = None
+        self.current_state_ids = current_state_ids
         # The current state excluding the current event
-        self.prev_state_ids = None
-        self.state_group = None
-
-        self.rejected = False
+        self.prev_state_ids = prev_state_ids
+        self.state_group = state_group
 
         # A previously persisted state group and a delta between that
         # and this state.
-        self.prev_group = None
-        self.delta_ids = None
+        self.prev_group = prev_group
+        self.delta_ids = delta_ids
 
-        self.prev_state_events = None
+        self.prev_state_events = []
 
+        self.rejected = False
         self.app_service = None
 
     def serialize(self, event):
@@ -123,27 +123,33 @@ def deserialize(store, input):
         Returns:
             EventContext
         """
-        context = EventContext()
-        context.state_group = input["state_group"]
-        context.rejected = input["rejected"]
-        context.prev_group = input["prev_group"]
-        context.delta_ids = _decode_state_dict(input["delta_ids"])
-        context.prev_state_events = input["prev_state_events"]
-
         # We use the state_group and prev_state_id stuff to pull the
         # current_state_ids out of the DB and construct prev_state_ids.
         prev_state_id = input["prev_state_id"]
         event_type = input["event_type"]
         event_state_key = input["event_state_key"]
 
-        context.current_state_ids = yield store.get_state_ids_for_group(
-            context.state_group,
+        state_group = input["state_group"]
+
+        current_state_ids = yield store.get_state_ids_for_group(
+            state_group,
         )
         if prev_state_id and event_state_key:
-            context.prev_state_ids = dict(context.current_state_ids)
-            context.prev_state_ids[(event_type, event_state_key)] = prev_state_id
+            prev_state_ids = dict(current_state_ids)
+            prev_state_ids[(event_type, event_state_key)] = prev_state_id
         else:
-            context.prev_state_ids = context.current_state_ids
+            prev_state_ids = current_state_ids
+
+        context = EventContext(
+            state_group=state_group,
+            current_state_ids=current_state_ids,
+            prev_state_ids=prev_state_ids,
+            prev_group=input["prev_group"],
+            delta_ids = _decode_state_dict(input["delta_ids"]),
+        )
+
+        context.rejected = input["rejected"]
+        context.prev_state_events = input["prev_state_events"]
 
         app_service_id = input["app_service_id"]
         if app_service_id:
diff --git a/synapse/state.py b/synapse/state.py
index 504caae2f740..a708695006ce 100644
--- a/synapse/state.py
+++ b/synapse/state.py
@@ -203,25 +203,27 @@ def compute_event_context(self, event, old_state=None):
             # If this is an outlier, then we know it shouldn't have any current
             # state. Certainly store.get_current_state won't return any, and
             # persisting the event won't store the state group.
-            context = EventContext()
             if old_state:
-                context.prev_state_ids = {
+                prev_state_ids = {
                     (s.type, s.state_key): s.event_id for s in old_state
                 }
                 if event.is_state():
-                    context.current_state_ids = dict(context.prev_state_ids)
+                    current_state_ids = dict(prev_state_ids)
                     key = (event.type, event.state_key)
-                    context.current_state_ids[key] = event.event_id
+                    current_state_ids[key] = event.event_id
                 else:
-                    context.current_state_ids = context.prev_state_ids
+                    current_state_ids = prev_state_ids
             else:
-                context.current_state_ids = {}
-                context.prev_state_ids = {}
-            context.prev_state_events = []
+                current_state_ids = {}
+                prev_state_ids = {}
 
             # We don't store state for outliers, so we don't generate a state
-            # froup for it.
-            context.state_group = None
+            # group for it.
+            context = EventContext(
+                state_group=None,
+                current_state_ids=current_state_ids,
+                prev_state_ids=prev_state_ids,
+            )
 
             defer.returnValue(context)
 
@@ -230,31 +232,35 @@ def compute_event_context(self, event, old_state=None):
             # Let's just correctly fill out the context and create a
             # new state group for it.
 
-            context = EventContext()
-            context.prev_state_ids = {
+            prev_state_ids = {
                 (s.type, s.state_key): s.event_id for s in old_state
             }
 
             if event.is_state():
                 key = (event.type, event.state_key)
-                if key in context.prev_state_ids:
-                    replaces = context.prev_state_ids[key]
+                if key in prev_state_ids:
+                    replaces = prev_state_ids[key]
                     if replaces != event.event_id:  # Paranoia check
                         event.unsigned["replaces_state"] = replaces
-                context.current_state_ids = dict(context.prev_state_ids)
-                context.current_state_ids[key] = event.event_id
+                current_state_ids = dict(prev_state_ids)
+                current_state_ids[key] = event.event_id
             else:
-                context.current_state_ids = context.prev_state_ids
+                current_state_ids = prev_state_ids
 
-            context.state_group = yield self.store.store_state_group(
+            state_group = yield self.store.store_state_group(
                 event.event_id,
                 event.room_id,
                 prev_group=None,
                 delta_ids=None,
-                current_state_ids=context.current_state_ids,
+                current_state_ids=current_state_ids,
+            )
+
+            context = EventContext(
+                state_group=state_group,
+                current_state_ids=current_state_ids,
+                prev_state_ids=prev_state_ids,
             )
 
-            context.prev_state_events = []
             defer.returnValue(context)
 
         logger.debug("calling resolve_state_groups from compute_event_context")
@@ -262,47 +268,47 @@ def compute_event_context(self, event, old_state=None):
             event.room_id, [e for e, _ in event.prev_events],
         )
 
-        curr_state = entry.state
+        prev_state_ids = entry.state
+        prev_group = None
+        delta_ids = None
 
-        context = EventContext()
-        context.prev_state_ids = curr_state
         if event.is_state():
             # If this is a state event then we need to create a new state
             # group for the state after this event.
 
             key = (event.type, event.state_key)
-            if key in context.prev_state_ids:
-                replaces = context.prev_state_ids[key]
+            if key in prev_state_ids:
+                replaces = prev_state_ids[key]
                 event.unsigned["replaces_state"] = replaces
 
-            context.current_state_ids = dict(context.prev_state_ids)
-            context.current_state_ids[key] = event.event_id
+            current_state_ids = dict(prev_state_ids)
+            current_state_ids[key] = event.event_id
 
             if entry.state_group:
                 # If the state at the event has a state group assigned then
                 # we can use that as the prev group
-                context.prev_group = entry.state_group
-                context.delta_ids = {
+                prev_group = entry.state_group
+                delta_ids = {
                     key: event.event_id
                 }
             elif entry.prev_group:
                 # If the state at the event only has a prev group, then we can
                 # use that as a prev group too.
-                context.prev_group = entry.prev_group
-                context.delta_ids = dict(entry.delta_ids)
-                context.delta_ids[key] = event.event_id
+                prev_group = entry.prev_group
+                delta_ids = dict(entry.delta_ids)
+                delta_ids[key] = event.event_id
 
-            context.state_group = yield self.store.store_state_group(
+            state_group = yield self.store.store_state_group(
                 event.event_id,
                 event.room_id,
-                prev_group=context.prev_group,
-                delta_ids=context.delta_ids,
-                current_state_ids=context.current_state_ids,
+                prev_group=prev_group,
+                delta_ids=delta_ids,
+                current_state_ids=current_state_ids,
             )
         else:
-            context.current_state_ids = context.prev_state_ids
-            context.prev_group = entry.prev_group
-            context.delta_ids = entry.delta_ids
+            current_state_ids = prev_state_ids
+            prev_group = entry.prev_group
+            delta_ids = entry.delta_ids
 
             if entry.state_group is None:
                 entry.state_group = yield self.store.store_state_group(
@@ -310,13 +316,20 @@ def compute_event_context(self, event, old_state=None):
                     event.room_id,
                     prev_group=entry.prev_group,
                     delta_ids=entry.delta_ids,
-                    current_state_ids=context.current_state_ids,
+                    current_state_ids=current_state_ids,
                 )
                 entry.state_id = entry.state_group
 
-            context.state_group = entry.state_group
+            state_group = entry.state_group
+
+        context = EventContext(
+            state_group=state_group,
+            current_state_ids=current_state_ids,
+            prev_state_ids=prev_state_ids,
+            prev_group=prev_group,
+            delta_ids=delta_ids,
+        )
 
-        context.prev_state_events = []
         defer.returnValue(context)
 
     @defer.inlineCallbacks

From 959f4b9074f6e893dc3c8e622e8c17fd229ad319 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 23 Jul 2018 12:22:59 +0100
Subject: [PATCH 091/830] Newsfile

---
 changelog.d/3577.misc | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 changelog.d/3577.misc

diff --git a/changelog.d/3577.misc b/changelog.d/3577.misc
new file mode 100644
index 000000000000..e69de29bb2d1

From 842cdece42e59a4181a496761447f0cb00053c05 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 23 Jul 2018 12:31:35 +0100
Subject: [PATCH 092/830] pep8

---
 synapse/events/snapshot.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py
index fbbe8dd49062..5e02ef1a5c18 100644
--- a/synapse/events/snapshot.py
+++ b/synapse/events/snapshot.py
@@ -145,7 +145,7 @@ def deserialize(store, input):
             current_state_ids=current_state_ids,
             prev_state_ids=prev_state_ids,
             prev_group=input["prev_group"],
-            delta_ids = _decode_state_dict(input["delta_ids"]),
+            delta_ids=_decode_state_dict(input["delta_ids"]),
         )
 
         context.rejected = input["rejected"]

From 440b8845b531db05e7c4f646e48dee7635cf1f0a Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 23 Jul 2018 12:38:46 +0100
Subject: [PATCH 093/830] Make EventContext lazy load state

---
 synapse/events/snapshot.py | 153 +++++++++++++++++++++++++++----------
 synapse/state.py           |   6 +-
 2 files changed, 115 insertions(+), 44 deletions(-)

diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py
index 5e02ef1a5c18..f9568638a153 100644
--- a/synapse/events/snapshot.py
+++ b/synapse/events/snapshot.py
@@ -19,18 +19,12 @@
 
 from twisted.internet import defer
 
+from synapse.util.logcontext import make_deferred_yieldable, run_in_background
+
 
 class EventContext(object):
     """
     Attributes:
-        current_state_ids (dict[(str, str), str]):
-            The current state map including the current event.
-            (type, state_key) -> event_id
-
-        prev_state_ids (dict[(str, str), str]):
-            The current state map excluding the current event.
-            (type, state_key) -> event_id
-
         state_group (int|None): state group id, if the state has been stored
             as a state group. This is usually only None if e.g. the event is
             an outlier.
@@ -47,36 +41,71 @@ class EventContext(object):
 
         prev_state_events (?): XXX: is this ever set to anything other than
             the empty list?
+
+        _current_state_ids (dict[(str, str), str]|None):
+            The current state map including the current event. None if outlier
+            or we haven't fetched the state from DB yet.
+            (type, state_key) -> event_id
+
+        _prev_state_ids (dict[(str, str), str]|None):
+            The current state map excluding the current event. None if outlier
+            or we haven't fetched the state from DB yet.
+            (type, state_key) -> event_id
+
+        _fetching_state_deferred (Deferred|None): Resolves when *_state_ids have
+            been calculated. None if we haven't started calculating yet
+
+        _prev_state_id (str|None): If set then the event associated with the
+            context overrode the _prev_state_id
+
+        _event_type (str): The type of the event the context is associated with
+
+        _event_state_key (str|None): The state_key of the event the context is
+            associated with
     """
 
     __slots__ = [
-        "current_state_ids",
-        "prev_state_ids",
         "state_group",
         "rejected",
         "prev_group",
         "delta_ids",
         "prev_state_events",
         "app_service",
+        "_current_state_ids",
+        "_prev_state_ids",
+        "_prev_state_id",
+        "_event_type",
+        "_event_state_key",
+        "_fetching_state_deferred",
     ]
 
-    def __init__(self, state_group, current_state_ids, prev_state_ids,
-                 prev_group=None, delta_ids=None):
+    @staticmethod
+    def with_state(state_group, current_state_ids, prev_state_ids,
+                   prev_group=None, delta_ids=None):
+        context = EventContext()
+
         # The current state including the current event
-        self.current_state_ids = current_state_ids
+        context._current_state_ids = current_state_ids
         # The current state excluding the current event
-        self.prev_state_ids = prev_state_ids
-        self.state_group = state_group
+        context._prev_state_ids = prev_state_ids
+        context.state_group = state_group
+
+        context._prev_state_id = None
+        context._event_type = None
+        context._event_state_key = None
+        context._fetching_state_deferred = defer.succeed(None)
 
         # A previously persisted state group and a delta between that
         # and this state.
-        self.prev_group = prev_group
-        self.delta_ids = delta_ids
+        context.prev_group = prev_group
+        context.delta_ids = delta_ids
+
+        context.prev_state_events = []
 
-        self.prev_state_events = []
+        context.rejected = False
+        context.app_service = None
 
-        self.rejected = False
-        self.app_service = None
+        return context
 
     def serialize(self, event):
         """Converts self to a type that can be serialized as JSON, and then
@@ -123,30 +152,17 @@ def deserialize(store, input):
         Returns:
             EventContext
         """
+        context = EventContext()
+
         # We use the state_group and prev_state_id stuff to pull the
         # current_state_ids out of the DB and construct prev_state_ids.
-        prev_state_id = input["prev_state_id"]
-        event_type = input["event_type"]
-        event_state_key = input["event_state_key"]
+        context._prev_state_id = input["prev_state_id"]
+        context._event_type = input["event_type"]
+        context._event_state_key = input["event_state_key"]
 
-        state_group = input["state_group"]
-
-        current_state_ids = yield store.get_state_ids_for_group(
-            state_group,
-        )
-        if prev_state_id and event_state_key:
-            prev_state_ids = dict(current_state_ids)
-            prev_state_ids[(event_type, event_state_key)] = prev_state_id
-        else:
-            prev_state_ids = current_state_ids
-
-        context = EventContext(
-            state_group=state_group,
-            current_state_ids=current_state_ids,
-            prev_state_ids=prev_state_ids,
-            prev_group=input["prev_group"],
-            delta_ids=_decode_state_dict(input["delta_ids"]),
-        )
+        context.state_group = input["state_group"]
+        context.prev_group = input["prev_group"]
+        context.delta_ids = _decode_state_dict(input["delta_ids"])
 
         context.rejected = input["rejected"]
         context.prev_state_events = input["prev_state_events"]
@@ -157,6 +173,61 @@ def deserialize(store, input):
 
         defer.returnValue(context)
 
+    @defer.inlineCallbacks
+    def get_current_state_ids(self, store):
+        """Gets the current state IDs
+
+        Returns:
+            Deferred[dict[(str, str), str]|None]: Returns None if state_group
+            is None, which happens when the associated event is an outlier.
+        """
+
+        if not self._fetching_state_deferred:
+            self._fetching_state_deferred = run_in_background(
+                self._fill_out_state, store,
+            )
+
+        yield make_deferred_yieldable(self._fetching_state_deferred)
+
+        defer.returnValue(self._current_state_ids)
+
+    @defer.inlineCallbacks
+    def get_prev_state_ids(self, store):
+        """Gets the prev state IDs
+
+        Returns:
+            Deferred[dict[(str, str), str]|None]: Returns None if state_group
+            is None, which happens when the associated event is an outlier.
+        """
+
+        if not self._fetching_state_deferred:
+            self._fetching_state_deferred = run_in_background(
+                self._fill_out_state, store,
+            )
+
+        yield make_deferred_yieldable(self._fetching_state_deferred)
+
+        defer.returnValue(self._prev_state_ids)
+
+    @defer.inlineCallbacks
+    def _fill_out_state(self, store):
+        """Called to populate the _current_state_ids and _prev_state_ids
+        attributes by loading from the database.
+        """
+        if self.state_group is None:
+            return
+
+        self._current_state_ids = yield store.get_state_ids_for_group(
+            self.state_group,
+        )
+        if self._prev_state_id and self._event_state_key is not None:
+            self._prev_state_ids = dict(self._current_state_ids)
+
+            key = (self._event_type, self._event_state_key)
+            self._prev_state_ids[key] = self._prev_state_id
+        else:
+            self._prev_state_ids = self._current_state_ids
+
 
 def _encode_state_dict(state_dict):
     """Since dicts of (type, state_key) -> event_id cannot be serialized in
diff --git a/synapse/state.py b/synapse/state.py
index a708695006ce..32125c95df49 100644
--- a/synapse/state.py
+++ b/synapse/state.py
@@ -219,7 +219,7 @@ def compute_event_context(self, event, old_state=None):
 
             # We don't store state for outliers, so we don't generate a state
             # group for it.
-            context = EventContext(
+            context = EventContext.with_state(
                 state_group=None,
                 current_state_ids=current_state_ids,
                 prev_state_ids=prev_state_ids,
@@ -255,7 +255,7 @@ def compute_event_context(self, event, old_state=None):
                 current_state_ids=current_state_ids,
             )
 
-            context = EventContext(
+            context = EventContext.with_state(
                 state_group=state_group,
                 current_state_ids=current_state_ids,
                 prev_state_ids=prev_state_ids,
@@ -322,7 +322,7 @@ def compute_event_context(self, event, old_state=None):
 
             state_group = entry.state_group
 
-        context = EventContext(
+        context = EventContext.with_state(
             state_group=state_group,
             current_state_ids=current_state_ids,
             prev_state_ids=prev_state_ids,

From e42510ba635b3e4d83215e4f5634ca51411996e0 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 23 Jul 2018 13:00:22 +0100
Subject: [PATCH 094/830] Use new getters

---
 synapse/api/auth.py                      |  6 ++++--
 synapse/handlers/_base.py                |  3 ++-
 synapse/handlers/federation.py           | 23 ++++++++++++++-------
 synapse/handlers/message.py              | 26 +++++++++++++++---------
 synapse/handlers/room_member.py          |  9 +++++---
 synapse/push/bulk_push_rule_evaluator.py |  7 ++++---
 synapse/storage/events.py                |  2 +-
 synapse/storage/push_rule.py             |  7 +++++--
 synapse/storage/roommember.py            |  7 +++++--
 9 files changed, 59 insertions(+), 31 deletions(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index bc629832d90a..535bdb449df2 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -65,8 +65,9 @@ def __init__(self, hs):
 
     @defer.inlineCallbacks
     def check_from_context(self, event, context, do_sig_check=True):
+        prev_state_ids = yield context.get_prev_state_ids(self.store)
         auth_events_ids = yield self.compute_auth_events(
-            event, context.prev_state_ids, for_verification=True,
+            event, prev_state_ids, for_verification=True,
         )
         auth_events = yield self.store.get_events(auth_events_ids)
         auth_events = {
@@ -544,7 +545,8 @@ def is_server_admin(self, user):
 
     @defer.inlineCallbacks
     def add_auth_events(self, builder, context):
-        auth_ids = yield self.compute_auth_events(builder, context.prev_state_ids)
+        prev_state_ids = yield context.get_prev_state_ids(self.store)
+        auth_ids = yield self.compute_auth_events(builder, prev_state_ids)
 
         auth_events_entries = yield self.store.add_event_hashes(
             auth_ids
diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py
index b6a8b3aa3b31..704181d2d3f8 100644
--- a/synapse/handlers/_base.py
+++ b/synapse/handlers/_base.py
@@ -112,8 +112,9 @@ def maybe_kick_guest_users(self, event, context=None):
             guest_access = event.content.get("guest_access", "forbidden")
             if guest_access != "can_join":
                 if context:
+                    current_state_ids = yield context.get_current_state_ids(self.store)
                     current_state = yield self.store.get_events(
-                        list(context.current_state_ids.values())
+                        list(current_state_ids.values())
                     )
                 else:
                     current_state = yield self.state_handler.get_current_state(
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index a6d391c4e889..98dd4a7fd18b 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -486,7 +486,10 @@ def _process_received_pdu(self, origin, pdu, state, auth_chain):
                 # joined the room. Don't bother if the user is just
                 # changing their profile info.
                 newly_joined = True
-                prev_state_id = context.prev_state_ids.get(
+
+                prev_state_ids = yield context.get_prev_state_ids(self.store)
+
+                prev_state_id = prev_state_ids.get(
                     (event.type, event.state_key)
                 )
                 if prev_state_id:
@@ -1106,10 +1109,12 @@ def on_send_join_request(self, origin, pdu):
                 user = UserID.from_string(event.state_key)
                 yield user_joined_room(self.distributor, user, event.room_id)
 
-        state_ids = list(context.prev_state_ids.values())
+        prev_state_ids = yield context.get_prev_state_ids(self.store)
+
+        state_ids = list(prev_state_ids.values())
         auth_chain = yield self.store.get_auth_chain(state_ids)
 
-        state = yield self.store.get_events(list(context.prev_state_ids.values()))
+        state = yield self.store.get_events(list(prev_state_ids.values()))
 
         defer.returnValue({
             "state": list(state.values()),
@@ -1635,8 +1640,9 @@ def _prep_event(self, origin, event, state=None, auth_events=None):
         )
 
         if not auth_events:
+            prev_state_ids = yield context.get_prev_state_ids(self.store)
             auth_events_ids = yield self.auth.compute_auth_events(
-                event, context.prev_state_ids, for_verification=True,
+                event, prev_state_ids, for_verification=True,
             )
             auth_events = yield self.store.get_events(auth_events_ids)
             auth_events = {
@@ -1876,9 +1882,10 @@ def do_auth(self, origin, event, context, auth_events):
                         break
 
             if do_resolution:
+                prev_state_ids = yield context.get_prev_state_ids(self.store)
                 # 1. Get what we think is the auth chain.
                 auth_ids = yield self.auth.compute_auth_events(
-                    event, context.prev_state_ids
+                    event, prev_state_ids
                 )
                 local_auth_chain = yield self.store.get_auth_chain(
                     auth_ids, include_given=True
@@ -2222,7 +2229,8 @@ def add_display_name_to_third_party_invite(self, event_dict, event, context):
             event.content["third_party_invite"]["signed"]["token"]
         )
         original_invite = None
-        original_invite_id = context.prev_state_ids.get(key)
+        prev_state_ids = yield context.get_prev_state_ids(self.store)
+        original_invite_id = prev_state_ids.get(key)
         if original_invite_id:
             original_invite = yield self.store.get_event(
                 original_invite_id, allow_none=True
@@ -2264,7 +2272,8 @@ def _check_signature(self, event, context):
         signed = event.content["third_party_invite"]["signed"]
         token = signed["token"]
 
-        invite_event_id = context.prev_state_ids.get(
+        prev_state_ids = yield context.get_prev_state_ids(self.store)
+        invite_event_id = prev_state_ids.get(
             (EventTypes.ThirdPartyInvite, token,)
         )
 
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index abc07ea87c88..c4bcd9018b9c 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -630,7 +630,8 @@ def deduplicate_state_event(self, event, context):
         If so, returns the version of the event in context.
         Otherwise, returns None.
         """
-        prev_event_id = context.prev_state_ids.get((event.type, event.state_key))
+        prev_state_ids = yield context.get_prev_state_ids(self.store)
+        prev_event_id = prev_state_ids.get((event.type, event.state_key))
         prev_event = yield self.store.get_event(prev_event_id, allow_none=True)
         if not prev_event:
             return
@@ -752,8 +753,8 @@ def create_new_client_event(self, builder, requester=None,
         event = builder.build()
 
         logger.debug(
-            "Created event %s with state: %s",
-            event.event_id, context.prev_state_ids,
+            "Created event %s",
+            event.event_id,
         )
 
         defer.returnValue(
@@ -884,9 +885,11 @@ def is_inviter_member_event(e):
                         e.sender == event.sender
                     )
 
+                current_state_ids = yield context.get_current_state_ids(self.store)
+
                 state_to_include_ids = [
                     e_id
-                    for k, e_id in iteritems(context.current_state_ids)
+                    for k, e_id in iteritems(current_state_ids)
                     if k[0] in self.hs.config.room_invite_state_types
                     or k == (EventTypes.Member, event.sender)
                 ]
@@ -922,8 +925,9 @@ def is_inviter_member_event(e):
                     )
 
         if event.type == EventTypes.Redaction:
+            prev_state_ids = yield context.get_prev_state_ids(self.store)
             auth_events_ids = yield self.auth.compute_auth_events(
-                event, context.prev_state_ids, for_verification=True,
+                event, prev_state_ids, for_verification=True,
             )
             auth_events = yield self.store.get_events(auth_events_ids)
             auth_events = {
@@ -943,11 +947,13 @@ def is_inviter_member_event(e):
                         "You don't have permission to redact events"
                     )
 
-        if event.type == EventTypes.Create and context.prev_state_ids:
-            raise AuthError(
-                403,
-                "Changing the room create event is forbidden",
-            )
+        if event.type == EventTypes.Create:
+            prev_state_ids = yield context.get_prev_state_ids(self.store)
+            if prev_state_ids:
+                raise AuthError(
+                    403,
+                    "Changing the room create event is forbidden",
+                )
 
         (event_stream_id, max_stream_id) = yield self.store.persist_event(
             event, context=context
diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 00f2e279bc36..a832d9180924 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -201,7 +201,9 @@ def _local_membership_update(
             ratelimit=ratelimit,
         )
 
-        prev_member_event_id = context.prev_state_ids.get(
+        prev_state_ids = yield context.get_prev_state_ids(self.store)
+
+        prev_member_event_id = prev_state_ids.get(
             (EventTypes.Member, target.to_string()),
             None
         )
@@ -496,9 +498,10 @@ def send_membership_event(
         if prev_event is not None:
             return
 
+        prev_state_ids = yield context.get_prev_state_ids(self.store)
         if event.membership == Membership.JOIN:
             if requester.is_guest:
-                guest_can_join = yield self._can_guest_join(context.prev_state_ids)
+                guest_can_join = yield self._can_guest_join(prev_state_ids)
                 if not guest_can_join:
                     # This should be an auth check, but guests are a local concept,
                     # so don't really fit into the general auth process.
@@ -517,7 +520,7 @@ def send_membership_event(
             ratelimit=ratelimit,
         )
 
-        prev_member_event_id = context.prev_state_ids.get(
+        prev_member_event_id = prev_state_ids.get(
             (EventTypes.Member, event.state_key),
             None
         )
diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py
index bb181d94eef4..1d14d3639c1d 100644
--- a/synapse/push/bulk_push_rule_evaluator.py
+++ b/synapse/push/bulk_push_rule_evaluator.py
@@ -112,7 +112,8 @@ def _get_rules_for_room(self, room_id):
 
     @defer.inlineCallbacks
     def _get_power_levels_and_sender_level(self, event, context):
-        pl_event_id = context.prev_state_ids.get(POWER_KEY)
+        prev_state_ids = yield context.get_prev_state_ids(self.store)
+        pl_event_id = prev_state_ids.get(POWER_KEY)
         if pl_event_id:
             # fastpath: if there's a power level event, that's all we need, and
             # not having a power level event is an extreme edge case
@@ -120,7 +121,7 @@ def _get_power_levels_and_sender_level(self, event, context):
             auth_events = {POWER_KEY: pl_event}
         else:
             auth_events_ids = yield self.auth.compute_auth_events(
-                event, context.prev_state_ids, for_verification=False,
+                event, prev_state_ids, for_verification=False,
             )
             auth_events = yield self.store.get_events(auth_events_ids)
             auth_events = {
@@ -304,7 +305,7 @@ def get_rules(self, event, context):
 
                 push_rules_delta_state_cache_metric.inc_hits()
             else:
-                current_state_ids = context.current_state_ids
+                current_state_ids = yield context.get_current_state_ids(self.store)
                 push_rules_delta_state_cache_metric.inc_misses()
 
             push_rules_state_size_counter.inc(len(current_state_ids))
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 4ff0fdc4abab..bf4f3ee92ab4 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -549,7 +549,7 @@ def _get_new_state_after_events(self, room_id, events_context, old_latest_event_
             if ctx.state_group in state_groups_map:
                 continue
 
-            state_groups_map[ctx.state_group] = ctx.current_state_ids
+            state_groups_map[ctx.state_group] = yield ctx.get_current_state_ids(self)
 
         # We need to map the event_ids to their state groups. First, let's
         # check if the event is one we're persisting, in which case we can
diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py
index be655d287bb9..af564b1b4e6b 100644
--- a/synapse/storage/push_rule.py
+++ b/synapse/storage/push_rule.py
@@ -186,6 +186,7 @@ def bulk_get_push_rules(self, user_ids):
 
         defer.returnValue(results)
 
+    @defer.inlineCallbacks
     def bulk_get_push_rules_for_room(self, event, context):
         state_group = context.state_group
         if not state_group:
@@ -195,9 +196,11 @@ def bulk_get_push_rules_for_room(self, event, context):
             # To do this we set the state_group to a new object as object() != object()
             state_group = object()
 
-        return self._bulk_get_push_rules_for_room(
-            event.room_id, state_group, context.current_state_ids, event=event
+        current_state_ids = yield context.get_current_state_ids(self)
+        result = yield self._bulk_get_push_rules_for_room(
+            event.room_id, state_group, current_state_ids, event=event
         )
+        defer.returnValue(result)
 
     @cachedInlineCallbacks(num_args=2, cache_context=True)
     def _bulk_get_push_rules_for_room(self, room_id, state_group, current_state_ids,
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index 02a802bed9a8..a27702a7a0f2 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -232,6 +232,7 @@ def get_users_who_share_room_with_user(self, user_id, cache_context):
 
         defer.returnValue(user_who_share_room)
 
+    @defer.inlineCallbacks
     def get_joined_users_from_context(self, event, context):
         state_group = context.state_group
         if not state_group:
@@ -241,11 +242,13 @@ def get_joined_users_from_context(self, event, context):
             # To do this we set the state_group to a new object as object() != object()
             state_group = object()
 
-        return self._get_joined_users_from_context(
-            event.room_id, state_group, context.current_state_ids,
+        current_state_ids = yield context.get_current_state_ids(self)
+        result = yield self._get_joined_users_from_context(
+            event.room_id, state_group, current_state_ids,
             event=event,
             context=context,
         )
+        defer.returnValue(result)
 
     def get_joined_users_from_state(self, room_id, state_entry):
         state_group = state_entry.state_group

From 027bc01a1bc254fe08140c6e91a9fb945b08486f Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 23 Jul 2018 13:02:09 +0100
Subject: [PATCH 095/830] Add support for updating state

---
 synapse/events/snapshot.py     | 19 +++++++++++++++++++
 synapse/handlers/federation.py | 32 +++++++++++++++++++++++---------
 2 files changed, 42 insertions(+), 9 deletions(-)

diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py
index f9568638a153..b090751bf1c8 100644
--- a/synapse/events/snapshot.py
+++ b/synapse/events/snapshot.py
@@ -228,6 +228,25 @@ def _fill_out_state(self, store):
         else:
             self._prev_state_ids = self._current_state_ids
 
+    @defer.inlineCallbacks
+    def update_state(self, state_group, prev_state_ids, current_state_ids,
+                     delta_ids):
+        """Replace the state in the context
+        """
+
+        # We need to make sure we wait for any ongoing fetching of state
+        # to complete so that the updated state doesn't get clobbered
+        if self._fetching_state_deferred:
+            yield make_deferred_yieldable(self._fetching_state_deferred)
+
+        self.state_group = state_group
+        self._prev_state_ids = prev_state_ids
+        self._current_state_ids = current_state_ids
+        self.delta_ids = delta_ids
+
+        # We need to ensure that that we've marked as having fetched the state
+        self._fetching_state_deferred = defer.succeed(None)
+
 
 def _encode_state_dict(state_dict):
     """Since dicts of (type, state_key) -> event_id cannot be serialized in
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 98dd4a7fd18b..14654d59f1fc 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -1975,21 +1975,35 @@ def _update_context_for_auth_events(self, event, context, auth_events,
             k: a.event_id for k, a in iteritems(auth_events)
             if k != event_key
         }
-        context.current_state_ids = dict(context.current_state_ids)
-        context.current_state_ids.update(state_updates)
+        current_state_ids = yield context.get_current_state_ids(self.store)
+        current_state_ids = dict(current_state_ids)
+
+        current_state_ids.update(state_updates)
+
         if context.delta_ids is not None:
-            context.delta_ids = dict(context.delta_ids)
-            context.delta_ids.update(state_updates)
-        context.prev_state_ids = dict(context.prev_state_ids)
-        context.prev_state_ids.update({
+            delta_ids = dict(context.delta_ids)
+            delta_ids.update(state_updates)
+
+        prev_state_ids = yield context.get_prev_state_ids(self.store)
+        prev_state_ids = dict(prev_state_ids)
+
+        prev_state_ids.update({
             k: a.event_id for k, a in iteritems(auth_events)
         })
-        context.state_group = yield self.store.store_state_group(
+
+        state_group = yield self.store.store_state_group(
             event.event_id,
             event.room_id,
             prev_group=context.prev_group,
-            delta_ids=context.delta_ids,
-            current_state_ids=context.current_state_ids,
+            delta_ids=delta_ids,
+            current_state_ids=current_state_ids,
+        )
+
+        yield context.update_state(
+            state_group=state_group,
+            current_state_ids=current_state_ids,
+            prev_state_ids=prev_state_ids,
+            delta_ids=delta_ids,
         )
 
     @defer.inlineCallbacks

From f3182bb1d0ba496bef710a529c4cd7a99da72061 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 23 Jul 2018 13:19:24 +0100
Subject: [PATCH 096/830] Newsfile

---
 changelog.d/3579.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3579.misc

diff --git a/changelog.d/3579.misc b/changelog.d/3579.misc
new file mode 100644
index 000000000000..2374dc0c44d5
--- /dev/null
+++ b/changelog.d/3579.misc
@@ -0,0 +1 @@
+Lazily load state on master process when using workers to reduce DB consumption

From 4fc52b10378b090fcf96747d8a5f2b261c2d3543 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 23 Jul 2018 13:20:43 +0100
Subject: [PATCH 097/830] Update docs/workers.rst

---
 docs/workers.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/workers.rst b/docs/workers.rst
index 1d521b9ec5e5..c5b37c3ded73 100644
--- a/docs/workers.rst
+++ b/docs/workers.rst
@@ -206,6 +206,10 @@ Handles client API endpoints. It can handle REST endpoints matching the
 following regular expressions::
 
     ^/_matrix/client/(api/v1|r0|unstable)/publicRooms$
+    ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/joined_members$
+    ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/context/.*$
+    ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/members$
+    ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/state$
 
 ``synapse.app.user_dir``
 ~~~~~~~~~~~~~~~~~~~~~~~~

From 8fbe418777a62ea6dd5fd811d63e30c42589650b Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 23 Jul 2018 13:33:49 +0100
Subject: [PATCH 098/830] Fix unit tests

---
 .../replication/slave/storage/test_events.py  |  8 ++--
 tests/test_state.py                           | 47 ++++++++++++++-----
 2 files changed, 40 insertions(+), 15 deletions(-)

diff --git a/tests/replication/slave/storage/test_events.py b/tests/replication/slave/storage/test_events.py
index cea01d93ebb5..f5b47f5ec0c2 100644
--- a/tests/replication/slave/storage/test_events.py
+++ b/tests/replication/slave/storage/test_events.py
@@ -222,9 +222,11 @@ def persist(
             state_ids = {
                 key: e.event_id for key, e in state.items()
             }
-            context = EventContext()
-            context.current_state_ids = state_ids
-            context.prev_state_ids = state_ids
+            context = EventContext.with_state(
+                state_group=None,
+                current_state_ids=state_ids,
+                prev_state_ids=state_ids
+            )
         else:
             state_handler = self.hs.get_state_handler()
             context = yield state_handler.compute_event_context(event)
diff --git a/tests/test_state.py b/tests/test_state.py
index c0f2d1152d9b..429a18cbf75e 100644
--- a/tests/test_state.py
+++ b/tests/test_state.py
@@ -204,7 +204,8 @@ def test_branch_no_conflict(self):
             self.store.register_event_context(event, context)
             context_store[event.event_id] = context
 
-        self.assertEqual(2, len(context_store["D"].prev_state_ids))
+        prev_state_ids = yield context_store["D"].get_prev_state_ids(self.store)
+        self.assertEqual(2, len(prev_state_ids))
 
     @defer.inlineCallbacks
     def test_branch_basic_conflict(self):
@@ -255,9 +256,11 @@ def test_branch_basic_conflict(self):
             self.store.register_event_context(event, context)
             context_store[event.event_id] = context
 
+        prev_state_ids = yield context_store["D"].get_prev_state_ids(self.store)
+
         self.assertSetEqual(
             {"START", "A", "C"},
-            {e_id for e_id in context_store["D"].prev_state_ids.values()}
+            {e_id for e_id in prev_state_ids.values()}
         )
 
     @defer.inlineCallbacks
@@ -318,9 +321,11 @@ def test_branch_have_banned_conflict(self):
             self.store.register_event_context(event, context)
             context_store[event.event_id] = context
 
+        prev_state_ids = yield context_store["E"].get_prev_state_ids(self.store)
+
         self.assertSetEqual(
             {"START", "A", "B", "C"},
-            {e for e in context_store["E"].prev_state_ids.values()}
+            {e for e in prev_state_ids.values()}
         )
 
     @defer.inlineCallbacks
@@ -398,9 +403,11 @@ def test_branch_have_perms_conflict(self):
             self.store.register_event_context(event, context)
             context_store[event.event_id] = context
 
+        prev_state_ids = yield context_store["D"].get_prev_state_ids(self.store)
+
         self.assertSetEqual(
             {"A1", "A2", "A3", "A5", "B"},
-            {e for e in context_store["D"].prev_state_ids.values()}
+            {e for e in prev_state_ids.values()}
         )
 
     def _add_depths(self, nodes, edges):
@@ -429,8 +436,10 @@ def test_annotate_with_old_message(self):
             event, old_state=old_state
         )
 
+        current_state_ids = yield context.get_current_state_ids(self.store)
+
         self.assertEqual(
-            set(e.event_id for e in old_state), set(context.current_state_ids.values())
+            set(e.event_id for e in old_state), set(current_state_ids.values())
         )
 
         self.assertIsNotNone(context.state_group)
@@ -449,8 +458,10 @@ def test_annotate_with_old_state(self):
             event, old_state=old_state
         )
 
+        prev_state_ids = yield context.get_prev_state_ids(self.store)
+
         self.assertEqual(
-            set(e.event_id for e in old_state), set(context.prev_state_ids.values())
+            set(e.event_id for e in old_state), set(prev_state_ids.values())
         )
 
     @defer.inlineCallbacks
@@ -475,9 +486,11 @@ def test_trivial_annotate_message(self):
 
         context = yield self.state.compute_event_context(event)
 
+        current_state_ids = yield context.get_current_state_ids(self.store)
+
         self.assertEqual(
             set([e.event_id for e in old_state]),
-            set(context.current_state_ids.values())
+            set(current_state_ids.values())
         )
 
         self.assertEqual(group_name, context.state_group)
@@ -504,9 +517,11 @@ def test_trivial_annotate_state(self):
 
         context = yield self.state.compute_event_context(event)
 
+        prev_state_ids = yield context.get_prev_state_ids(self.store)
+
         self.assertEqual(
             set([e.event_id for e in old_state]),
-            set(context.prev_state_ids.values())
+            set(prev_state_ids.values())
         )
 
         self.assertIsNotNone(context.state_group)
@@ -545,7 +560,9 @@ def test_resolve_message_conflict(self):
             event, prev_event_id1, old_state_1, prev_event_id2, old_state_2,
         )
 
-        self.assertEqual(len(context.current_state_ids), 6)
+        current_state_ids = yield context.get_current_state_ids(self.store)
+
+        self.assertEqual(len(current_state_ids), 6)
 
         self.assertIsNotNone(context.state_group)
 
@@ -585,7 +602,9 @@ def test_resolve_state_conflict(self):
             event, prev_event_id1, old_state_1, prev_event_id2, old_state_2,
         )
 
-        self.assertEqual(len(context.current_state_ids), 6)
+        current_state_ids = yield context.get_current_state_ids(self.store)
+
+        self.assertEqual(len(current_state_ids), 6)
 
         self.assertIsNotNone(context.state_group)
 
@@ -642,8 +661,10 @@ def test_standard_depth_conflict(self):
             event, prev_event_id1, old_state_1, prev_event_id2, old_state_2,
         )
 
+        current_state_ids = yield context.get_current_state_ids(self.store)
+
         self.assertEqual(
-            old_state_2[3].event_id, context.current_state_ids[("test1", "1")]
+            old_state_2[3].event_id, current_state_ids[("test1", "1")]
         )
 
         # Reverse the depth to make sure we are actually using the depths
@@ -670,8 +691,10 @@ def test_standard_depth_conflict(self):
             event, prev_event_id1, old_state_1, prev_event_id2, old_state_2,
         )
 
+        current_state_ids = yield context.get_current_state_ids(self.store)
+
         self.assertEqual(
-            old_state_1[3].event_id, context.current_state_ids[("test1", "1")]
+            old_state_1[3].event_id, current_state_ids[("test1", "1")]
         )
 
     def _get_context(self, event, prev_event_id_1, old_state_1, prev_event_id_2,

From 4797ed000e612b68c418ce8c342bdd7ecc16b198 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 23 Jul 2018 15:05:40 +0100
Subject: [PATCH 099/830] Update docstrings to make sense

---
 synapse/events/snapshot.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py
index b090751bf1c8..a6d7bf57006d 100644
--- a/synapse/events/snapshot.py
+++ b/synapse/events/snapshot.py
@@ -55,13 +55,16 @@ class EventContext(object):
         _fetching_state_deferred (Deferred|None): Resolves when *_state_ids have
             been calculated. None if we haven't started calculating yet
 
-        _prev_state_id (str|None): If set then the event associated with the
-            context overrode the _prev_state_id
-
-        _event_type (str): The type of the event the context is associated with
+        _event_type (str): The type of the event the context is associated with.
+            Only set when state has not been fetched yet.
 
         _event_state_key (str|None): The state_key of the event the context is
-            associated with
+            associated with. Only set when state has not been fetched yet.
+
+        _prev_state_id (str|None): If the event associated with the context is
+            a state event, then `_prev_state_id` is the event_id of the state
+            that was replaced.
+            Only set when state has not been fetched yet.
     """
 
     __slots__ = [

From 999bcf9d016fb7fd9ad5a9daf4f0ec6d25a10717 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 23 Jul 2018 15:24:21 +0100
Subject: [PATCH 100/830] Fix EventContext when using workers

We were:
  1. Not correctly setting all attributes
  2. Using defer.inlineCallbacks in a non-generator
---
 synapse/events/snapshot.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py
index a6d7bf57006d..e31eceb921a7 100644
--- a/synapse/events/snapshot.py
+++ b/synapse/events/snapshot.py
@@ -82,6 +82,11 @@ class EventContext(object):
         "_fetching_state_deferred",
     ]
 
+    def __init__(self):
+        self.prev_state_events = []
+        self.rejected = False
+        self.app_service = None
+
     @staticmethod
     def with_state(state_group, current_state_ids, prev_state_ids,
                    prev_group=None, delta_ids=None):
@@ -103,11 +108,6 @@ def with_state(state_group, current_state_ids, prev_state_ids,
         context.prev_group = prev_group
         context.delta_ids = delta_ids
 
-        context.prev_state_events = []
-
-        context.rejected = False
-        context.app_service = None
-
         return context
 
     def serialize(self, event):
@@ -143,7 +143,6 @@ def serialize(self, event):
         }
 
     @staticmethod
-    @defer.inlineCallbacks
     def deserialize(store, input):
         """Converts a dict that was produced by `serialize` back into a
         EventContext.
@@ -162,6 +161,7 @@ def deserialize(store, input):
         context._prev_state_id = input["prev_state_id"]
         context._event_type = input["event_type"]
         context._event_state_key = input["event_state_key"]
+        context._fetching_state_deferred = None
 
         context.state_group = input["state_group"]
         context.prev_group = input["prev_group"]
@@ -174,7 +174,7 @@ def deserialize(store, input):
         if app_service_id:
             context.app_service = store.get_app_service_by_id(app_service_id)
 
-        defer.returnValue(context)
+        return context
 
     @defer.inlineCallbacks
     def get_current_state_ids(self, store):

From a4d24781bf2290311af8f901862d83134bbd6229 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 23 Jul 2018 15:28:51 +0100
Subject: [PATCH 101/830] Newsfile

---
 changelog.d/3581.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3581.misc

diff --git a/changelog.d/3581.misc b/changelog.d/3581.misc
new file mode 100644
index 000000000000..2374dc0c44d5
--- /dev/null
+++ b/changelog.d/3581.misc
@@ -0,0 +1 @@
+Lazily load state on master process when using workers to reduce DB consumption

From 0faa3223cdf996aa18376a7420a43061a6691638 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 23 Jul 2018 16:28:00 +0100
Subject: [PATCH 102/830] Fix missing attributes on workers.

This was missed during the transition from attribute to getter for
getting state from context.
---
 synapse/events/snapshot.py             | 10 ++++++----
 synapse/handlers/message.py            |  5 +++--
 synapse/replication/http/send_event.py |  7 +++++--
 3 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py
index e31eceb921a7..a59064b416a5 100644
--- a/synapse/events/snapshot.py
+++ b/synapse/events/snapshot.py
@@ -110,7 +110,8 @@ def with_state(state_group, current_state_ids, prev_state_ids,
 
         return context
 
-    def serialize(self, event):
+    @defer.inlineCallbacks
+    def serialize(self, event, store):
         """Converts self to a type that can be serialized as JSON, and then
         deserialized by `deserialize`
 
@@ -126,11 +127,12 @@ def serialize(self, event):
         # the prev_state_ids, so if we're a state event we include the event
         # id that we replaced in the state.
         if event.is_state():
-            prev_state_id = self.prev_state_ids.get((event.type, event.state_key))
+            prev_state_ids = yield self.get_prev_state_ids(store)
+            prev_state_id = prev_state_ids.get((event.type, event.state_key))
         else:
             prev_state_id = None
 
-        return {
+        defer.returnValue({
             "prev_state_id": prev_state_id,
             "event_type": event.type,
             "event_state_key": event.state_key if event.is_state() else None,
@@ -140,7 +142,7 @@ def serialize(self, event):
             "delta_ids": _encode_state_dict(self.delta_ids),
             "prev_state_events": self.prev_state_events,
             "app_service_id": self.app_service.id if self.app_service else None
-        }
+        })
 
     @staticmethod
     def deserialize(store, input):
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index c4bcd9018b9c..7571975c22f1 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -807,8 +807,9 @@ def handle_new_client_event(
             # If we're a worker we need to hit out to the master.
             if self.config.worker_app:
                 yield send_event_to_master(
-                    self.hs.get_clock(),
-                    self.http_client,
+                    clock=self.hs.get_clock(),
+                    store=self.store,
+                    client=self.http_client,
                     host=self.config.worker_replication_host,
                     port=self.config.worker_replication_http_port,
                     requester=requester,
diff --git a/synapse/replication/http/send_event.py b/synapse/replication/http/send_event.py
index 2eede547921e..5227bc333d19 100644
--- a/synapse/replication/http/send_event.py
+++ b/synapse/replication/http/send_event.py
@@ -34,12 +34,13 @@
 
 
 @defer.inlineCallbacks
-def send_event_to_master(clock, client, host, port, requester, event, context,
+def send_event_to_master(clock, store, client, host, port, requester, event, context,
                          ratelimit, extra_users):
     """Send event to be handled on the master
 
     Args:
         clock (synapse.util.Clock)
+        store (DataStore)
         client (SimpleHttpClient)
         host (str): host of master
         port (int): port on master listening for HTTP replication
@@ -53,11 +54,13 @@ def send_event_to_master(clock, client, host, port, requester, event, context,
         host, port, event.event_id,
     )
 
+    serialized_context = yield context.serialize(event, store)
+
     payload = {
         "event": event.get_pdu_json(),
         "internal_metadata": event.internal_metadata.get_dict(),
         "rejected_reason": event.rejected_reason,
-        "context": context.serialize(event),
+        "context": serialized_context,
         "requester": requester.serialize(),
         "ratelimit": ratelimit,
         "extra_users": [u.to_string() for u in extra_users],

From 9f41ad491dd950058562e25045bc6ae3539ddc79 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 23 Jul 2018 16:31:46 +0100
Subject: [PATCH 103/830] Newsfile

---
 changelog.d/3582.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3582.misc

diff --git a/changelog.d/3582.misc b/changelog.d/3582.misc
new file mode 100644
index 000000000000..2374dc0c44d5
--- /dev/null
+++ b/changelog.d/3582.misc
@@ -0,0 +1 @@
+Lazily load state on master process when using workers to reduce DB consumption

From dae6dc1e776cc6198dc1f71575876fc16693c170 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 23 Jul 2018 17:13:34 +0100
Subject: [PATCH 104/830] Remove redundant checks on room forgottenness

Fixes #3550
---
 changelog.d/3350.misc         |  1 +
 synapse/storage/push_rule.py  | 13 -------------
 synapse/storage/roommember.py | 15 ---------------
 synapse/visibility.py         | 19 +------------------
 4 files changed, 2 insertions(+), 46 deletions(-)
 create mode 100644 changelog.d/3350.misc

diff --git a/changelog.d/3350.misc b/changelog.d/3350.misc
new file mode 100644
index 000000000000..3713cd6d635e
--- /dev/null
+++ b/changelog.d/3350.misc
@@ -0,0 +1 @@
+Remove redundant checks on who_forgot_in_room
\ No newline at end of file
diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py
index be655d287bb9..d25b39ec0265 100644
--- a/synapse/storage/push_rule.py
+++ b/synapse/storage/push_rule.py
@@ -21,7 +21,6 @@
 
 from twisted.internet import defer
 
-from synapse.api.constants import EventTypes
 from synapse.push.baserules import list_with_base_rules
 from synapse.storage.appservice import ApplicationServiceWorkerStore
 from synapse.storage.pusher import PusherWorkerStore
@@ -247,18 +246,6 @@ def _bulk_get_push_rules_for_room(self, room_id, state_group, current_state_ids,
             if uid in local_users_in_room:
                 user_ids.add(uid)
 
-        forgotten = yield self.who_forgot_in_room(
-            event.room_id, on_invalidate=cache_context.invalidate,
-        )
-
-        for row in forgotten:
-            user_id = row["user_id"]
-            event_id = row["event_id"]
-
-            mem_id = current_state_ids.get((EventTypes.Member, user_id), None)
-            if event_id == mem_id:
-                user_ids.discard(user_id)
-
         rules_by_user = yield self.bulk_get_push_rules(
             user_ids, on_invalidate=cache_context.invalidate,
         )
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index 02a802bed9a8..9eb5b181445e 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -458,18 +458,6 @@ def _get_joined_hosts(self, room_id, state_group, current_state_ids, state_entry
     def _get_joined_hosts_cache(self, room_id):
         return _JoinedHostsCache(self, room_id)
 
-    @cached()
-    def who_forgot_in_room(self, room_id):
-        return self._simple_select_list(
-            table="room_memberships",
-            retcols=("user_id", "event_id"),
-            keyvalues={
-                "room_id": room_id,
-                "forgotten": 1,
-            },
-            desc="who_forgot"
-        )
-
 
 class RoomMemberStore(RoomMemberWorkerStore):
     def __init__(self, db_conn, hs):
@@ -578,9 +566,6 @@ def f(txn):
             txn.execute(sql, (user_id, room_id))
 
             txn.call_after(self.did_forget.invalidate, (user_id, room_id))
-            self._invalidate_cache_and_stream(
-                txn, self.who_forgot_in_room, (room_id,)
-            )
         return self.runInteraction("forget_membership", f)
 
     @cachedInlineCallbacks(num_args=2)
diff --git a/synapse/visibility.py b/synapse/visibility.py
index 9b97ea2b83a1..e0b2eccceb8f 100644
--- a/synapse/visibility.py
+++ b/synapse/visibility.py
@@ -23,7 +23,6 @@
 from synapse.api.constants import EventTypes, Membership
 from synapse.events.utils import prune_event
 from synapse.types import get_domain_from_id
-from synapse.util.logcontext import make_deferred_yieldable, preserve_fn
 
 logger = logging.getLogger(__name__)
 
@@ -75,19 +74,6 @@ def filter_events_for_client(store, user_id, events, is_peeking=False,
         types=types,
     )
 
-    forgotten = yield make_deferred_yieldable(defer.gatherResults([
-        defer.maybeDeferred(
-            preserve_fn(store.who_forgot_in_room),
-            room_id,
-        )
-        for room_id in frozenset(e.room_id for e in events)
-    ], consumeErrors=True))
-
-    # Set of membership event_ids that have been forgotten
-    event_id_forgotten = frozenset(
-        row["event_id"] for rows in forgotten for row in rows
-    )
-
     ignore_dict_content = yield store.get_global_account_data_by_type_for_user(
         "m.ignored_user_list", user_id,
     )
@@ -176,10 +162,7 @@ def allowed(event):
         if membership is None:
             membership_event = state.get((EventTypes.Member, user_id), None)
             if membership_event:
-                # XXX why do we do this?
-                # https://github.com/matrix-org/synapse/issues/3350
-                if membership_event.event_id not in event_id_forgotten:
-                    membership = membership_event.membership
+                membership = membership_event.membership
 
         # if the user was a member of the room at the time of the event,
         # they can see it.

From 50c60e5fadbefff6785c17dda9eecf88286dba30 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 23 Jul 2018 17:21:40 +0100
Subject: [PATCH 105/830] Only get cached state from context in persist_event

We don't want to bother pulling out the current state from the DB since
until we know we have to. Checking the context for state is just an
optimisation.
---
 synapse/events/snapshot.py | 13 +++++++++++++
 synapse/storage/events.py  |  4 +++-
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py
index a59064b416a5..c439b53801d8 100644
--- a/synapse/events/snapshot.py
+++ b/synapse/events/snapshot.py
@@ -163,6 +163,9 @@ def deserialize(store, input):
         context._prev_state_id = input["prev_state_id"]
         context._event_type = input["event_type"]
         context._event_state_key = input["event_state_key"]
+
+        context._current_state_ids = None
+        context._prev_state_ids = None
         context._fetching_state_deferred = None
 
         context.state_group = input["state_group"]
@@ -214,6 +217,16 @@ def get_prev_state_ids(self, store):
 
         defer.returnValue(self._prev_state_ids)
 
+    def get_cached_current_state_ids(self):
+        """Gets the current state IDs if we have them already cached.
+
+        Returns:
+            dict[(str, str), str]|None: Returns None if state_group
+            is None, which happens when the associated event is an outlier.
+        """
+
+        return self._current_state_ids
+
     @defer.inlineCallbacks
     def _fill_out_state(self, store):
         """Called to populate the _current_state_ids and _prev_state_ids
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index bf4f3ee92ab4..dc0b3c2ebad0 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -549,7 +549,9 @@ def _get_new_state_after_events(self, room_id, events_context, old_latest_event_
             if ctx.state_group in state_groups_map:
                 continue
 
-            state_groups_map[ctx.state_group] = yield ctx.get_current_state_ids(self)
+            current_state_ids = ctx.get_cached_current_state_ids()
+            if current_state_ids is not None:
+                state_groups_map[ctx.state_group] = current_state_ids
 
         # We need to map the event_ids to their state groups. First, let's
         # check if the event is one we're persisting, in which case we can

From 2d5bba151bec414ea6f16417c5eae335205c0df0 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 23 Jul 2018 17:29:32 +0100
Subject: [PATCH 106/830] Newsfile

---
 changelog.d/3584.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3584.misc

diff --git a/changelog.d/3584.misc b/changelog.d/3584.misc
new file mode 100644
index 000000000000..2374dc0c44d5
--- /dev/null
+++ b/changelog.d/3584.misc
@@ -0,0 +1 @@
+Lazily load state on master process when using workers to reduce DB consumption

From 8b9f164fff6cf821ff5bc702f3660c0f0eb320e7 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 23 Jul 2018 17:43:01 +0100
Subject: [PATCH 107/830] Comments

---
 synapse/events/snapshot.py | 5 +++--
 synapse/storage/events.py  | 3 +++
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py
index c439b53801d8..189212b0fa0b 100644
--- a/synapse/events/snapshot.py
+++ b/synapse/events/snapshot.py
@@ -221,8 +221,9 @@ def get_cached_current_state_ids(self):
         """Gets the current state IDs if we have them already cached.
 
         Returns:
-            dict[(str, str), str]|None: Returns None if state_group
-            is None, which happens when the associated event is an outlier.
+            dict[(str, str), str]|None: Returns None if we haven't cached the
+            state or if state_group is None, which happens when the associated
+            event is an outlier.
         """
 
         return self._current_state_ids
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index dc0b3c2ebad0..c2910094d07d 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -549,6 +549,9 @@ def _get_new_state_after_events(self, room_id, events_context, old_latest_event_
             if ctx.state_group in state_groups_map:
                 continue
 
+            # We're only interested in pulling out state that has already
+            # been cached in the context. We'll pull stuff out of the DB later
+            # if necessary.
             current_state_ids = ctx.get_cached_current_state_ids()
             if current_state_ids is not None:
                 state_groups_map[ctx.state_group] = current_state_ids

From 5c705f70c9489427a7985ea10ec60552965b9a1c Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 23 Jul 2018 19:00:16 +0100
Subject: [PATCH 108/830] Fixes and optimisations for resolve_state_groups

First of all, fix the logic which looks for identical input state groups so
that we actually use them. This turned out to be most easily done by factoring
the relevant code out to a separate function so that we could do an early
return.

Secondly, avoid building the whole `conflicted_state` dict (which was only ever
used as a boolean flag).

Thirdly, replace the construction of the `state` dict (which mapped from keys
to events that set them), with an optimistic construction of the resolution
result assuming there will be no conflicts. This should be no slower than
building the old `state` dict, and:
  - in the conflicted case, we'll short-cut it, saving part of the work
  - in the unconflicted case, it saves rebuilding the resolution from the
    `state` dict.

Finally, do a couple of s/values/itervalues/.
---
 synapse/state.py | 143 +++++++++++++++++++++++++++++------------------
 1 file changed, 89 insertions(+), 54 deletions(-)

diff --git a/synapse/state.py b/synapse/state.py
index 32125c95df49..033f55d967f1 100644
--- a/synapse/state.py
+++ b/synapse/state.py
@@ -471,69 +471,39 @@ def resolve_state_groups(
                 "Resolving state for %s with %d groups", room_id, len(state_groups_ids)
             )
 
-            # build a map from state key to the event_ids which set that state.
-            # dict[(str, str), set[str])
-            state = {}
+            # start by assuming we won't have any conflicted state, and build up the new
+            # state map by iterating through the state groups. If we discover a conflict,
+            # we give up and instead use `resolve_events_with_factory`.
+            #
+            # XXX: is this actually worthwhile, or should we just let
+            # resolve_events_with_factory do it?
+            new_state = {}
+            conflicted_state = False
             for st in itervalues(state_groups_ids):
                 for key, e_id in iteritems(st):
-                    state.setdefault(key, set()).add(e_id)
-
-            # build a map from state key to the event_ids which set that state,
-            # including only those where there are state keys in conflict.
-            conflicted_state = {
-                k: list(v)
-                for k, v in iteritems(state)
-                if len(v) > 1
-            }
+                    if key in new_state:
+                        conflicted_state = True
+                        break
+                    new_state[key] = e_id
+                if conflicted_state:
+                    break
 
             if conflicted_state:
                 logger.info("Resolving conflicted state for %r", room_id)
                 with Measure(self.clock, "state._resolve_events"):
                     new_state = yield resolve_events_with_factory(
-                        list(state_groups_ids.values()),
+                        list(itervalues(state_groups_ids)),
                         event_map=event_map,
                         state_map_factory=state_map_factory,
                     )
-            else:
-                new_state = {
-                    key: e_ids.pop() for key, e_ids in iteritems(state)
-                }
 
-            with Measure(self.clock, "state.create_group_ids"):
-                # if the new state matches any of the input state groups, we can
-                # use that state group again. Otherwise we will generate a state_id
-                # which will be used as a cache key for future resolutions, but
-                # not get persisted.
-                state_group = None
-                new_state_event_ids = frozenset(itervalues(new_state))
-                for sg, events in iteritems(state_groups_ids):
-                    if new_state_event_ids == frozenset(e_id for e_id in events):
-                        state_group = sg
-                        break
+            # if the new state matches any of the input state groups, we can
+            # use that state group again. Otherwise we will generate a state_id
+            # which will be used as a cache key for future resolutions, but
+            # not get persisted.
 
-                # TODO: We want to create a state group for this set of events, to
-                # increase cache hits, but we need to make sure that it doesn't
-                # end up as a prev_group without being added to the database
-
-                prev_group = None
-                delta_ids = None
-                for old_group, old_ids in iteritems(state_groups_ids):
-                    if not set(new_state) - set(old_ids):
-                        n_delta_ids = {
-                            k: v
-                            for k, v in iteritems(new_state)
-                            if old_ids.get(k) != v
-                        }
-                        if not delta_ids or len(n_delta_ids) < len(delta_ids):
-                            prev_group = old_group
-                            delta_ids = n_delta_ids
-
-            cache = _StateCacheEntry(
-                state=new_state,
-                state_group=state_group,
-                prev_group=prev_group,
-                delta_ids=delta_ids,
-            )
+            with Measure(self.clock, "state.create_group_ids"):
+                cache = _make_state_cache_entry(new_state, state_groups_ids)
 
             if self._state_cache is not None:
                 self._state_cache[group_names] = cache
@@ -541,6 +511,70 @@ def resolve_state_groups(
             defer.returnValue(cache)
 
 
+def _make_state_cache_entry(
+    new_state,
+    state_groups_ids,
+):
+    """Given a resolved state, and a set of input state groups, pick one to base
+    a new state group on (if any), and return an appropriately-constructed
+    _StateCacheEntry.
+
+    Args:
+        new_state (dict[(str, str), str]): resolved state map (mapping from
+           (type, state_key) to event_id)
+
+        state_groups_ids (dict[int, dict[(str, str), str]]):
+                 map from state group id to the state in that state group
+                (where 'state' is a map from state key to event id)
+
+    Returns:
+        _StateCacheEntry
+    """
+    # if the new state matches any of the input state groups, we can
+    # use that state group again. Otherwise we will generate a state_id
+    # which will be used as a cache key for future resolutions, but
+    # not get persisted.
+
+    # first look for exact matches
+    new_state_event_ids = set(itervalues(new_state))
+    for sg, state in iteritems(state_groups_ids):
+        if len(new_state_event_ids) != len(state):
+            continue
+
+        old_state_event_ids = set(itervalues(state))
+        if new_state_event_ids == old_state_event_ids:
+            # got an exact match.
+            return _StateCacheEntry(
+                state=new_state,
+                state_group=sg,
+            )
+
+    # TODO: We want to create a state group for this set of events, to
+    # increase cache hits, but we need to make sure that it doesn't
+    # end up as a prev_group without being added to the database
+
+    # failing that, look for the closest match.
+    prev_group = None
+    delta_ids = None
+
+    for old_group, old_state in iteritems(state_groups_ids):
+        n_delta_ids = {
+            k: v
+            for k, v in iteritems(new_state)
+            if old_state.get(k) != v
+        }
+        if not delta_ids or len(n_delta_ids) < len(delta_ids):
+            prev_group = old_group
+            delta_ids = n_delta_ids
+
+    return _StateCacheEntry(
+        state=new_state,
+        state_group=None,
+        prev_group=prev_group,
+        delta_ids=delta_ids,
+    )
+
+
 def _ordered_events(events):
     def key_func(e):
         return -int(e.depth), hashlib.sha1(e.event_id.encode()).hexdigest()
@@ -582,7 +616,7 @@ def _seperate(state_sets):
     with them in different state sets.
 
     Args:
-        state_sets(list[dict[(str, str), str]]):
+        state_sets(iterable[dict[(str, str), str]]):
             List of dicts of (type, state_key) -> event_id, which are the
             different state groups to resolve.
 
@@ -596,10 +630,11 @@ def _seperate(state_sets):
             conflicted_state is a dict mapping (type, state_key) to a set of
             event ids for conflicted state keys.
     """
-    unconflicted_state = dict(state_sets[0])
+    state_set_iterator = iter(state_sets)
+    unconflicted_state = dict(next(state_set_iterator))
     conflicted_state = {}
 
-    for state_set in state_sets[1:]:
+    for state_set in state_set_iterator:
         for key, value in iteritems(state_set):
             # Check if there is an unconflicted entry for the state key.
             unconflicted_value = unconflicted_state.get(key)

From cc99256e90c620b01ae255db304170479bfe8523 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 23 Jul 2018 19:10:50 +0100
Subject: [PATCH 109/830] newsfile

---
 changelog.d/3586.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3586.misc

diff --git a/changelog.d/3586.misc b/changelog.d/3586.misc
new file mode 100644
index 000000000000..e853e2481bcd
--- /dev/null
+++ b/changelog.d/3586.misc
@@ -0,0 +1 @@
+Fixes and optimisations for resolve_state_groups

From 254fb430d1662c93c56c2abbd6984e07fb04c36b Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Mon, 23 Jul 2018 19:21:20 +0100
Subject: [PATCH 110/830] incorporate review

---
 synapse/handlers/sync.py | 67 ++++++++++++++++------------------------
 synapse/storage/state.py | 20 +++++-------
 2 files changed, 35 insertions(+), 52 deletions(-)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index b597f94cf641..5689ad2f580a 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -543,17 +543,6 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
 
                     state_ids = current_state_ids
 
-                # track the membership state events as of the beginning of this
-                # timeline sequence, so they can be filtered out of the state
-                # if we are lazy loading members.
-                if lazy_load_members:
-                    member_state_ids = {
-                        t: state_ids[t]
-                        for t in state_ids if t[0] == EventTypes.Member
-                    }
-                else:
-                    member_state_ids = {}
-
                 timeline_state = {
                     (event.type, event.state_key): event.event_id
                     for event in batch.events if event.is_state()
@@ -562,9 +551,9 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                 state_ids = _calculate_state(
                     timeline_contains=timeline_state,
                     timeline_start=state_ids,
-                    timeline_start_members=member_state_ids,
                     previous={},
                     current=current_state_ids,
+                    lazy_load_members=lazy_load_members,
                 )
             elif batch.limited:
                 state_at_previous_sync = yield self.get_state_at(
@@ -582,37 +571,27 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                     filtered_types=filtered_types,
                 )
 
-                # track the membership state events as of the beginning of this
-                # timeline sequence, so they can be filtered out of the state
-                # if we are lazy loading members.
-                if lazy_load_members:
-                    # TODO: optionally filter out redundant membership events at this
-                    # point, to stop repeatedly sending members in every /sync as if
-                    # the client isn't tracking them.
-                    # When implement, this should filter using event_ids (not mxids).
-                    # In practice, limited syncs are
-                    # relatively rare so it's not a total disaster to send redundant
-                    # members down at this point. Redundant members are ones which
-                    # repeatedly get sent down /sync because we don't know if the client
-                    # is caching them or not.
-                    member_state_ids = {
-                        t: state_at_timeline_start[t]
-                        for t in state_at_timeline_start if t[0] == EventTypes.Member
-                    }
-                else:
-                    member_state_ids = {}
-
                 timeline_state = {
                     (event.type, event.state_key): event.event_id
                     for event in batch.events if event.is_state()
                 }
 
+                # TODO: optionally filter out redundant membership events at this
+                # point, to stop repeatedly sending members in every /sync as if
+                # the client isn't tracking them.
+                # When implemented, this should filter using event_ids (not mxids).
+                # In practice, limited syncs are
+                # relatively rare so it's not a total disaster to send redundant
+                # members down at this point. Redundant members are ones which
+                # repeatedly get sent down /sync because we don't know if the client
+                # is caching them or not.
+
                 state_ids = _calculate_state(
                     timeline_contains=timeline_state,
                     timeline_start=state_at_timeline_start,
-                    timeline_start_members=member_state_ids,
                     previous=state_at_previous_sync,
                     current=current_state_ids,
+                    lazy_load_members=lazy_load_members,
                 )
             else:
                 state_ids = {}
@@ -1536,16 +1515,14 @@ def _action_has_highlight(actions):
     return False
 
 
-def _calculate_state(timeline_contains, timeline_start, timeline_start_members,
-                     previous, current):
+def _calculate_state(
+    timeline_contains, timeline_start, previous, current, lazy_load_members,
+):
     """Works out what state to include in a sync response.
 
     Args:
         timeline_contains (dict): state in the timeline
         timeline_start (dict): state at the start of the timeline
-        timeline_start_members (dict): state at the start of the timeline
-            for room members who participate in this chunk of timeline.
-            Should always be a subset of timeline_start.
         previous (dict): state at the end of the previous sync (or empty dict
             if this is an initial sync)
         current (dict): state at the end of the timeline
@@ -1565,11 +1542,21 @@ def _calculate_state(timeline_contains, timeline_start, timeline_start_members,
 
     c_ids = set(e for e in current.values())
     ts_ids = set(e for e in timeline_start.values())
-    tsm_ids = set(e for e in timeline_start_members.values())
     p_ids = set(e for e in previous.values())
     tc_ids = set(e for e in timeline_contains.values())
 
-    state_ids = (((c_ids | ts_ids) - p_ids) - tc_ids) | tsm_ids
+    # track the membership events in the state as of the start of the timeline
+    # so we can add them back in to the state if we're lazyloading.  We don't
+    # add them into state if they're already contained in the timeline.
+    if lazy_load_members:
+        ll_ids = set(
+            e for t, e in timeline_start.iteritems()
+            if t[0] == EventTypes.Member and e not in tc_ids
+        )
+    else:
+        ll_ids = set()
+
+    state_ids = (((c_ids | ts_ids) - p_ids) - tc_ids) | ll_ids
 
     return {
         event_id_to_key[e]: e for e in state_ids
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index f09be7172dc0..40ca8bd2a2a4 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -191,10 +191,10 @@ def _get_state_groups_from_groups(self, groups, types, filtered_types=None):
 
         Args:
             groups(list[int]): list of state group IDs to query
-            types(list[str|None, str|None])|None: List of 2-tuples of the form
+            types (Iterable[str, str|None]|None): list of 2-tuples of the form
                 (`type`, `state_key`), where a `state_key` of `None` matches all
                 state_keys for the `type`. If None, all types are returned.
-            filtered_types(list[str]|None): Only apply filtering via `types` to this
+            filtered_types(Iterable[str]|None): Only apply filtering via `types` to this
                 list of event types.  Other types of events are returned unfiltered.
                 If None, `types` filtering is applied to all events.
 
@@ -207,19 +207,17 @@ def _get_state_groups_from_groups(self, groups, types, filtered_types=None):
         for chunk in chunks:
             res = yield self.runInteraction(
                 "_get_state_groups_from_groups",
-                self._get_state_groups_from_groups_txn, chunk, types, filtered_types
+                self._get_state_groups_from_groups_txn, chunk, types, filtered_types,
             )
             results.update(res)
 
         defer.returnValue(results)
 
     def _get_state_groups_from_groups_txn(
-        self, txn, groups, types=None, filtered_types=None
+        self, txn, groups, types=None, filtered_types=None,
     ):
         results = {group: {} for group in groups}
 
-        include_other_types = False if filtered_types is None else True
-
         if types is not None:
             types = list(set(types))  # deduplicate types list
 
@@ -269,7 +267,7 @@ def _get_state_groups_from_groups_txn(
                     for etype, state_key in types
                 ]
 
-                if include_other_types:
+                if filtered_types is not None:
                     # XXX: check whether this slows postgres down like a list of
                     # ORs does too?
                     unique_types = set(filtered_types)
@@ -308,7 +306,7 @@ def _get_state_groups_from_groups_txn(
                         where_clauses.append("(type = ? AND state_key = ?)")
                         where_args.extend([typ[0], typ[1]])
 
-                if include_other_types:
+                if filtered_types is not None:
                     unique_types = set(filtered_types)
                     where_clauses.append(
                         "(" + " AND ".join(["type <> ?"] * len(unique_types)) + ")"
@@ -538,8 +536,6 @@ def _get_some_state_from_cache(self, group, types, filtered_types=None):
         # tracks which of the requested types are missing from our cache
         missing_types = set()
 
-        include_other_types = False if filtered_types is None else True
-
         for typ, state_key in types:
             key = (typ, state_key)
 
@@ -562,7 +558,7 @@ def _get_some_state_from_cache(self, group, types, filtered_types=None):
         def include(typ, state_key):
             valid_state_keys = type_to_key.get(typ, sentinel)
             if valid_state_keys is sentinel:
-                return include_other_types and typ not in filtered_types
+                return filtered_types is not None and typ not in filtered_types
             if valid_state_keys is None:
                 return True
             if state_key in valid_state_keys:
@@ -598,7 +594,7 @@ def _get_state_for_groups(self, groups, types=None, filtered_types=None):
         Args:
             groups (iterable[int]): list of state groups for which we want
                 to get the state.
-            types (None|iterable[(None, None|str)]):
+            types (None|iterable[(str, None|str)]):
                 indicates the state type/keys required. If None, the whole
                 state is fetched and returned.
 

From c1f80effbe17b1572161cc50838e60b495fb45a4 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 23 Jul 2018 22:06:50 +0100
Subject: [PATCH 111/830] Handle delta_ids being None in
 _update_context_for_auth_events

it's easier to create the new state group as a delta from the existing one.

(There's an outside chance this will help with
https://github.com/matrix-org/synapse/issues/3364)
---
 synapse/events/snapshot.py     |  3 ++-
 synapse/handlers/federation.py | 13 ++++++-------
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py
index 189212b0fa0b..368b5f6ae449 100644
--- a/synapse/events/snapshot.py
+++ b/synapse/events/snapshot.py
@@ -249,7 +249,7 @@ def _fill_out_state(self, store):
 
     @defer.inlineCallbacks
     def update_state(self, state_group, prev_state_ids, current_state_ids,
-                     delta_ids):
+                     prev_group, delta_ids):
         """Replace the state in the context
         """
 
@@ -260,6 +260,7 @@ def update_state(self, state_group, prev_state_ids, current_state_ids,
 
         self.state_group = state_group
         self._prev_state_ids = prev_state_ids
+        self.prev_group = prev_group
         self._current_state_ids = current_state_ids
         self.delta_ids = delta_ids
 
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 14654d59f1fc..145c1a21d4c8 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -1980,10 +1980,6 @@ def _update_context_for_auth_events(self, event, context, auth_events,
 
         current_state_ids.update(state_updates)
 
-        if context.delta_ids is not None:
-            delta_ids = dict(context.delta_ids)
-            delta_ids.update(state_updates)
-
         prev_state_ids = yield context.get_prev_state_ids(self.store)
         prev_state_ids = dict(prev_state_ids)
 
@@ -1991,11 +1987,13 @@ def _update_context_for_auth_events(self, event, context, auth_events,
             k: a.event_id for k, a in iteritems(auth_events)
         })
 
+        # create a new state group as a delta from the existing one.
+        prev_group = context.state_group
         state_group = yield self.store.store_state_group(
             event.event_id,
             event.room_id,
-            prev_group=context.prev_group,
-            delta_ids=delta_ids,
+            prev_group=prev_group,
+            delta_ids=state_updates,
             current_state_ids=current_state_ids,
         )
 
@@ -2003,7 +2001,8 @@ def _update_context_for_auth_events(self, event, context, auth_events,
             state_group=state_group,
             current_state_ids=current_state_ids,
             prev_state_ids=prev_state_ids,
-            delta_ids=delta_ids,
+            prev_group=prev_group,
+            delta_ids=state_updates,
         )
 
     @defer.inlineCallbacks

From ce0c18dec5d04626018b8e401273c8f30e88b0c7 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 23 Jul 2018 22:13:19 +0100
Subject: [PATCH 112/830] Improve logging for exceptions handling PDUs

when we get an exception handling a federation PDU, log the whole stacktrace.
---
 synapse/federation/federation_server.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index 48f26db67ca0..e501251b6e96 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -24,6 +24,7 @@
 
 from twisted.internet import defer
 from twisted.internet.abstract import isIPAddress
+from twisted.python import failure
 
 from synapse.api.constants import EventTypes
 from synapse.api.errors import AuthError, FederationError, NotFoundError, SynapseError
@@ -186,8 +187,12 @@ def process_pdus_for_room(room_id):
                     logger.warn("Error handling PDU %s: %s", event_id, e)
                     pdu_results[event_id] = {"error": str(e)}
                 except Exception as e:
+                    f = failure.Failure()
                     pdu_results[event_id] = {"error": str(e)}
-                    logger.exception("Failed to handle PDU %s", event_id)
+                    logger.error(
+                        "Failed to handle PDU %s: %s",
+                        event_id, f.getTraceback().rstrip(),
+                    )
 
         yield async.concurrently_execute(
             process_pdus_for_room, pdus_by_room.keys(),
@@ -203,8 +208,8 @@ def process_pdus_for_room(room_id):
                 )
 
         pdu_failures = getattr(transaction, "pdu_failures", [])
-        for failure in pdu_failures:
-            logger.info("Got failure %r", failure)
+        for fail in pdu_failures:
+            logger.info("Got failure %r", fail)
 
         response = {
             "pdus": pdu_results,

From d8709df7391901ac13ed57375b986d1b9de583ea Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 23 Jul 2018 22:16:22 +0100
Subject: [PATCH 113/830] changelog

---
 changelog.d/3587.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3587.misc

diff --git a/changelog.d/3587.misc b/changelog.d/3587.misc
new file mode 100644
index 000000000000..75a3479910b8
--- /dev/null
+++ b/changelog.d/3587.misc
@@ -0,0 +1 @@
+Improve logging for exceptions when handling PDUs
\ No newline at end of file

From 004a83b43a5c75ec6ca3378c8ef5a4c616e33e8c Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Mon, 23 Jul 2018 22:32:35 +0100
Subject: [PATCH 114/830] changelog

---
 changelog.d/2970.feature | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/2970.feature

diff --git a/changelog.d/2970.feature b/changelog.d/2970.feature
new file mode 100644
index 000000000000..5eb928563fea
--- /dev/null
+++ b/changelog.d/2970.feature
@@ -0,0 +1 @@
+add support for the lazy_loaded_members filter as per MSC1227

From efcdacad7d1b7f52f879179701c7e0d9b763511f Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Mon, 23 Jul 2018 22:41:05 +0100
Subject: [PATCH 115/830] handle case where types is [] on postgres correctly

---
 synapse/storage/state.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index 40ca8bd2a2a4..f99d3871e4a1 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -255,7 +255,7 @@ def _get_state_groups_from_groups_txn(
             # Turns out that postgres doesn't like doing a list of OR's and
             # is about 1000x slower, so we just issue a query for each specific
             # type seperately.
-            if types:
+            if types is not None:
                 clause_to_args = [
                     (
                         "AND type = ? AND state_key = ?",

From 1938cffaea465375d5dd410baa94a6b9b6ded0cc Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 23 Jul 2018 23:48:19 +0100
Subject: [PATCH 116/830] Add some measure blocks to persist_events

... to help us figure out where 40% of CPU is going
---
 synapse/storage/events.py | 26 +++++++++++++++-----------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index c2910094d07d..d13d099d1e9a 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -417,19 +417,23 @@ def _persist_events(self, events_and_contexts, backfilled=False,
                             logger.info(
                                 "Calculating state delta for room %s", room_id,
                             )
-                            current_state = yield self._get_new_state_after_events(
-                                room_id,
-                                ev_ctx_rm,
-                                latest_event_ids,
-                                new_latest_event_ids,
-                            )
+
+                            with Measure("persist_events.get_new_state_after_events"):
+                                current_state = yield self._get_new_state_after_events(
+                                    room_id,
+                                    ev_ctx_rm,
+                                    latest_event_ids,
+                                    new_latest_event_ids,
+                                )
+
                             if current_state is not None:
                                 current_state_for_room[room_id] = current_state
-                                delta = yield self._calculate_state_delta(
-                                    room_id, current_state,
-                                )
-                                if delta is not None:
-                                    state_delta_for_room[room_id] = delta
+                                with Measure("persist_events.calculate_state_delta"):
+                                    delta = yield self._calculate_state_delta(
+                                        room_id, current_state,
+                                    )
+                                    if delta is not None:
+                                        state_delta_for_room[room_id] = delta
 
                 yield self.runInteraction(
                     "persist_events",

From 69fb5dbdabf50dffe34e2a0d012b5a227fc16c55 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 24 Jul 2018 00:04:44 +0100
Subject: [PATCH 117/830] fix idiocy

---
 synapse/storage/events.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index d13d099d1e9a..392935cacf4e 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -418,7 +418,10 @@ def _persist_events(self, events_and_contexts, backfilled=False,
                                 "Calculating state delta for room %s", room_id,
                             )
 
-                            with Measure("persist_events.get_new_state_after_events"):
+                            with Measure(
+                                    self._clock,
+                                    "persist_events.get_new_state_after_events",
+                            ):
                                 current_state = yield self._get_new_state_after_events(
                                     room_id,
                                     ev_ctx_rm,
@@ -428,7 +431,10 @@ def _persist_events(self, events_and_contexts, backfilled=False,
 
                             if current_state is not None:
                                 current_state_for_room[room_id] = current_state
-                                with Measure("persist_events.calculate_state_delta"):
+                                with Measure(
+                                        self._clock,
+                                        "persist_events.calculate_state_delta",
+                                ):
                                     delta = yield self._calculate_state_delta(
                                         room_id, current_state,
                                     )

From 30957a941a9a48b9afcfadbabdd4118b08abd81f Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 24 Jul 2018 00:24:38 +0100
Subject: [PATCH 118/830] changelog

---
 changelog.d/3590.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3590.misc

diff --git a/changelog.d/3590.misc b/changelog.d/3590.misc
new file mode 100644
index 000000000000..0f1688fd0f73
--- /dev/null
+++ b/changelog.d/3590.misc
@@ -0,0 +1 @@
+Add some measure blocks to persist_events

From 8dff6e0322718ec9c446465c1e10ab331a417b8a Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 24 Jul 2018 00:37:17 +0100
Subject: [PATCH 119/830] Logcontext fixes

Fix some random logcontext leaks.
---
 synapse/handlers/initial_sync.py | 28 +++++++++++++++-------------
 synapse/storage/events.py        |  5 +++--
 synapse/storage/pusher.py        |  2 +-
 3 files changed, 19 insertions(+), 16 deletions(-)

diff --git a/synapse/handlers/initial_sync.py b/synapse/handlers/initial_sync.py
index fb11716eb894..50b13d882002 100644
--- a/synapse/handlers/initial_sync.py
+++ b/synapse/handlers/initial_sync.py
@@ -387,19 +387,21 @@ def get_receipts():
                 receipts = []
             defer.returnValue(receipts)
 
-        presence, receipts, (messages, token) = yield defer.gatherResults(
-            [
-                run_in_background(get_presence),
-                run_in_background(get_receipts),
-                run_in_background(
-                    self.store.get_recent_events_for_room,
-                    room_id,
-                    limit=limit,
-                    end_token=now_token.room_key,
-                )
-            ],
-            consumeErrors=True,
-        ).addErrback(unwrapFirstError)
+        presence, receipts, (messages, token) = yield make_deferred_yieldable(
+            defer.gatherResults(
+                [
+                    run_in_background(get_presence),
+                    run_in_background(get_receipts),
+                    run_in_background(
+                        self.store.get_recent_events_for_room,
+                        room_id,
+                        limit=limit,
+                        end_token=now_token.room_key,
+                    )
+                ],
+                consumeErrors=True,
+            ).addErrback(unwrapFirstError),
+        )
 
         messages = yield filter_events_for_client(
             self.store, user_id, messages, is_peeking=is_peeking,
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index c2910094d07d..c06dbb376891 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -39,7 +39,7 @@
 from synapse.util.async import ObservableDeferred
 from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
 from synapse.util.frozenutils import frozendict_json_encoder
-from synapse.util.logcontext import make_deferred_yieldable
+from synapse.util.logcontext import PreserveLoggingContext, make_deferred_yieldable
 from synapse.util.logutils import log_function
 from synapse.util.metrics import Measure
 
@@ -147,7 +147,8 @@ def handle_queue_loop():
                     # callbacks on the deferred.
                     try:
                         ret = yield per_item_callback(item)
-                        item.deferred.callback(ret)
+                        with PreserveLoggingContext():
+                            item.deferred.callback(ret)
                     except Exception:
                         item.deferred.errback()
             finally:
diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py
index cc273a57b2c6..8443bd4c1b70 100644
--- a/synapse/storage/pusher.py
+++ b/synapse/storage/pusher.py
@@ -233,7 +233,7 @@ def add_pusher(self, user_id, access_token, kind, app_id,
             )
 
             if newly_inserted:
-                self.runInteraction(
+                yield self.runInteraction(
                     "add_pusher",
                     self._invalidate_cache_and_stream,
                     self.get_if_user_has_pusher, (user_id,)

From c6e66821a991c45b8b6a78f00f52ae65304225ea Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 24 Jul 2018 00:38:39 +0100
Subject: [PATCH 120/830] Changelog

---
 changelog.d/3591.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3591.misc

diff --git a/changelog.d/3591.misc b/changelog.d/3591.misc
new file mode 100644
index 000000000000..f0137766a066
--- /dev/null
+++ b/changelog.d/3591.misc
@@ -0,0 +1 @@
+Fix some random logcontext leaks.
\ No newline at end of file

From cf2d15c6a953d42207fb2c8fe5dc57ee7fdae7ce Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 24 Jul 2018 00:57:48 +0100
Subject: [PATCH 121/830] another couple of logcontext leaks

---
 synapse/handlers/appservice.py   |  5 ++++-
 synapse/handlers/initial_sync.py | 10 ++++++----
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py
index ec9fe01a5a94..ee41aed69e46 100644
--- a/synapse/handlers/appservice.py
+++ b/synapse/handlers/appservice.py
@@ -23,6 +23,7 @@
 
 import synapse
 from synapse.api.constants import EventTypes
+from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.util.logcontext import make_deferred_yieldable, run_in_background
 from synapse.util.metrics import Measure
 
@@ -106,7 +107,9 @@ def handle_event(event):
                             yield self._check_user_exists(event.state_key)
 
                         if not self.started_scheduler:
-                            self.scheduler.start().addErrback(log_failure)
+                            def start_scheduler():
+                                return self.scheduler.start().addErrback(log_failure)
+                            run_as_background_process("as_scheduler", start_scheduler)
                             self.started_scheduler = True
 
                         # Fork off pushes to these services
diff --git a/synapse/handlers/initial_sync.py b/synapse/handlers/initial_sync.py
index 50b13d882002..40e7580a61dd 100644
--- a/synapse/handlers/initial_sync.py
+++ b/synapse/handlers/initial_sync.py
@@ -148,13 +148,15 @@ def handle_room(event):
             try:
                 if event.membership == Membership.JOIN:
                     room_end_token = now_token.room_key
-                    deferred_room_state = self.state_handler.get_current_state(
-                        event.room_id
+                    deferred_room_state = run_in_background(
+                        self.state_handler.get_current_state,
+                        event.room_id,
                     )
                 elif event.membership == Membership.LEAVE:
                     room_end_token = "s%d" % (event.stream_ordering,)
-                    deferred_room_state = self.store.get_state_for_events(
-                        [event.event_id], None
+                    deferred_room_state = run_in_background(
+                        self.store.get_state_for_events,
+                        [event.event_id], None,
                     )
                     deferred_room_state.addCallback(
                         lambda states: states[event.event_id]

From ff5426f6b8263b416f412725a6d7be2fac284824 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 24 Jul 2018 10:55:11 +0100
Subject: [PATCH 122/830] Speed up _calculate_state_delta

---
 synapse/storage/events.py | 13 +++----------
 1 file changed, 3 insertions(+), 10 deletions(-)

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index c2910094d07d..1b075e6cc6fa 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -644,21 +644,14 @@ def _calculate_state_delta(self, room_id, current_state):
         """
         existing_state = yield self.get_current_state_ids(room_id)
 
-        existing_events = set(itervalues(existing_state))
-        new_events = set(ev_id for ev_id in itervalues(current_state))
-        changed_events = existing_events ^ new_events
-
-        if not changed_events:
-            return
-
         to_delete = {
             key: ev_id for key, ev_id in iteritems(existing_state)
-            if ev_id in changed_events
+            if ev_id != current_state.get(key)
         }
-        events_to_insert = (new_events - existing_events)
+
         to_insert = {
             key: ev_id for key, ev_id in iteritems(current_state)
-            if ev_id in events_to_insert
+            if ev_id != existing_state.get(key)
         }
 
         defer.returnValue((to_delete, to_insert))

From 2581eb3e1d9f7d767315432f2c2e381f31a8716c Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 24 Jul 2018 11:14:28 +0100
Subject: [PATCH 123/830] Newsfile

---
 changelog.d/3592.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3592.misc

diff --git a/changelog.d/3592.misc b/changelog.d/3592.misc
new file mode 100644
index 000000000000..60129569c278
--- /dev/null
+++ b/changelog.d/3592.misc
@@ -0,0 +1 @@
+Speed up calculating state deltas in persist_event loop

From 0fa73e4a63d6cd5d402ea5213eac6b766e650321 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 24 Jul 2018 11:19:23 +0100
Subject: [PATCH 124/830] Remove unnecessary if

---
 synapse/storage/events.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 1b075e6cc6fa..cc8ab5165b23 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -428,8 +428,7 @@ def _persist_events(self, events_and_contexts, backfilled=False,
                                 delta = yield self._calculate_state_delta(
                                     room_id, current_state,
                                 )
-                                if delta is not None:
-                                    state_delta_for_room[room_id] = delta
+                                state_delta_for_room[room_id] = delta
 
                 yield self.runInteraction(
                     "persist_events",

From cd241d6bda01a761fbe1ca29727dacd918fb8975 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Tue, 24 Jul 2018 12:39:40 +0100
Subject: [PATCH 125/830] incorporate more review

---
 synapse/handlers/sync.py    | 12 +++++++++---
 synapse/storage/state.py    | 36 +++++++++---------------------------
 tests/storage/test_state.py |  9 +++++++++
 3 files changed, 27 insertions(+), 30 deletions(-)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 5689ad2f580a..e5a2329d7308 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -1526,6 +1526,9 @@ def _calculate_state(
         previous (dict): state at the end of the previous sync (or empty dict
             if this is an initial sync)
         current (dict): state at the end of the timeline
+        lazy_load_members (bool): whether to return members from timeline_start
+            or not.  assumes that timeline_start has already been filtered to
+            include only the members the client needs to know about.
 
     Returns:
         dict
@@ -1545,9 +1548,12 @@ def _calculate_state(
     p_ids = set(e for e in previous.values())
     tc_ids = set(e for e in timeline_contains.values())
 
-    # track the membership events in the state as of the start of the timeline
-    # so we can add them back in to the state if we're lazyloading.  We don't
-    # add them into state if they're already contained in the timeline.
+    # If we are lazyloading room members, we explicitly add the membership events
+    # for the senders in the timeline into the state block returned by /sync,
+    # as we may not have sent them to the client before.  We find these membership
+    # events by filtering them out of timeline_start, which has already been filtered
+    # to only include membership events for the senders in the timeline.
+
     if lazy_load_members:
         ll_ids = set(
             e for t, e in timeline_start.iteritems()
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index f99d3871e4a1..1413a6f91048 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -185,7 +185,7 @@ def get_state_groups(self, room_id, event_ids):
         })
 
     @defer.inlineCallbacks
-    def _get_state_groups_from_groups(self, groups, types, filtered_types=None):
+    def _get_state_groups_from_groups(self, groups, types):
         """Returns the state groups for a given set of groups, filtering on
         types of state events.
 
@@ -194,9 +194,6 @@ def _get_state_groups_from_groups(self, groups, types, filtered_types=None):
             types (Iterable[str, str|None]|None): list of 2-tuples of the form
                 (`type`, `state_key`), where a `state_key` of `None` matches all
                 state_keys for the `type`. If None, all types are returned.
-            filtered_types(Iterable[str]|None): Only apply filtering via `types` to this
-                list of event types.  Other types of events are returned unfiltered.
-                If None, `types` filtering is applied to all events.
 
         Returns:
             dictionary state_group -> (dict of (type, state_key) -> event id)
@@ -207,14 +204,14 @@ def _get_state_groups_from_groups(self, groups, types, filtered_types=None):
         for chunk in chunks:
             res = yield self.runInteraction(
                 "_get_state_groups_from_groups",
-                self._get_state_groups_from_groups_txn, chunk, types, filtered_types,
+                self._get_state_groups_from_groups_txn, chunk, types,
             )
             results.update(res)
 
         defer.returnValue(results)
 
     def _get_state_groups_from_groups_txn(
-        self, txn, groups, types=None, filtered_types=None,
+        self, txn, groups, types=None,
     ):
         results = {group: {} for group in groups}
 
@@ -266,17 +263,6 @@ def _get_state_groups_from_groups_txn(
                     )
                     for etype, state_key in types
                 ]
-
-                if filtered_types is not None:
-                    # XXX: check whether this slows postgres down like a list of
-                    # ORs does too?
-                    unique_types = set(filtered_types)
-                    clause_to_args.append(
-                        (
-                            "AND type <> ? " * len(unique_types),
-                            list(unique_types)
-                        )
-                    )
             else:
                 # If types is None we fetch all the state, and so just use an
                 # empty where clause with no extra args.
@@ -306,13 +292,6 @@ def _get_state_groups_from_groups_txn(
                         where_clauses.append("(type = ? AND state_key = ?)")
                         where_args.extend([typ[0], typ[1]])
 
-                if filtered_types is not None:
-                    unique_types = set(filtered_types)
-                    where_clauses.append(
-                        "(" + " AND ".join(["type <> ?"] * len(unique_types)) + ")"
-                    )
-                    where_args.extend(list(unique_types))
-
                 where_clause = "AND (%s)" % (" OR ".join(where_clauses))
             else:
                 where_clause = ""
@@ -643,13 +622,13 @@ def _get_state_for_groups(self, groups, types=None, filtered_types=None):
             # cache. Hence, if we are doing a wildcard lookup, populate the
             # cache fully so that we can do an efficient lookup next time.
 
-            if types and any(k is None for (t, k) in types):
+            if filtered_types or (types and any(k is None for (t, k) in types)):
                 types_to_fetch = None
             else:
                 types_to_fetch = types
 
             group_to_state_dict = yield self._get_state_groups_from_groups(
-                missing_groups, types_to_fetch, filtered_types
+                missing_groups, types_to_fetch
             )
 
             for group, group_state_dict in iteritems(group_to_state_dict):
@@ -659,7 +638,10 @@ def _get_state_for_groups(self, groups, types=None, filtered_types=None):
                 if types:
                     for k, v in iteritems(group_state_dict):
                         (typ, _) = k
-                        if k in types or (typ, None) in types:
+                        if (
+                            (k in types or (typ, None) in types) or
+                            (filtered_types and typ not in filtered_types)
+                        ):
                             state_dict[k] = v
                 else:
                     state_dict.update(group_state_dict)
diff --git a/tests/storage/test_state.py b/tests/storage/test_state.py
index 8924ba9f7f90..b2f314e9db02 100644
--- a/tests/storage/test_state.py
+++ b/tests/storage/test_state.py
@@ -158,3 +158,12 @@ def test_get_state_for_event(self):
             (e2.type, e2.state_key): e2,
             (e3.type, e3.state_key): e3,
         }, state)
+
+        state = yield self.store.get_state_for_event(
+            e5.event_id, [], filtered_types=[EventTypes.Member],
+        )
+
+        self.assertStateMapEqual({
+            (e1.type, e1.state_key): e1,
+            (e2.type, e2.state_key): e2,
+        }, state)

From eb1d911ab743e85154f7c4b2db8a954d152020dc Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Tue, 24 Jul 2018 13:40:49 +0100
Subject: [PATCH 126/830] rather than adding ll_ids, remove them from p_ids

---
 synapse/handlers/sync.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index e5a2329d7308..1422843af829 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -1553,16 +1553,17 @@ def _calculate_state(
     # as we may not have sent them to the client before.  We find these membership
     # events by filtering them out of timeline_start, which has already been filtered
     # to only include membership events for the senders in the timeline.
+    # In practice, we can do this by removing them from the p_ids list.
+    # see https://github.com/matrix-org/synapse/pull/2970
+    #            /files/efcdacad7d1b7f52f879179701c7e0d9b763511f#r204732809
 
     if lazy_load_members:
-        ll_ids = set(
+        p_ids.difference_update(
             e for t, e in timeline_start.iteritems()
-            if t[0] == EventTypes.Member and e not in tc_ids
+            if t[0] == EventTypes.Member
         )
-    else:
-        ll_ids = set()
 
-    state_ids = (((c_ids | ts_ids) - p_ids) - tc_ids) | ll_ids
+    state_ids = ((c_ids | ts_ids) - p_ids) - tc_ids
 
     return {
         event_id_to_key[e]: e for e in state_ids

From 7417951117fb096d433b620264f2c034500e41d3 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Tue, 24 Jul 2018 13:46:39 +0100
Subject: [PATCH 127/830] Update ISSUE_TEMPLATE.md

request backticks for logs
---
 .github/ISSUE_TEMPLATE.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index d2050a3e44ab..21acb3202a94 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -27,8 +27,9 @@ Describe here the problem that you are experiencing, or the feature you are requ
 
 Describe how what happens differs from what you expected.
 
-If you can identify any relevant log snippets from _homeserver.log_, please include
-those here (please be careful to remove any personal or private data):
+<!-- If you can identify any relevant log snippets from _homeserver.log_, please include
+those (please be careful to remove any personal or private data). Please surround them with
+``` (three backticks, on a line on their own), so that they are formatted legibly. -->
 
 ### Version information
 

From e22700c3dd929f9f0953b3d2b37e503eece82b38 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Tue, 24 Jul 2018 13:59:07 +0100
Subject: [PATCH 128/830] consider non-filter_type types as wildcards, thus
 missing from the state-group-cache

---
 synapse/storage/state.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index 1413a6f91048..86f2c2e6b18a 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -518,7 +518,10 @@ def _get_some_state_from_cache(self, group, types, filtered_types=None):
         for typ, state_key in types:
             key = (typ, state_key)
 
-            if state_key is None:
+            if (
+                state_key is None or
+                filtered_types is not None and typ not in filtered_types
+            ):
                 type_to_key[typ] = None
                 # we mark the type as missing from the cache because
                 # when the cache was populated it might have been done with a

From 223341205edf725c5583a73d5680245a24790452 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 24 Jul 2018 11:59:16 +0100
Subject: [PATCH 129/830] Don't require to_delete to have event_ids

---
 synapse/storage/events.py | 91 +++++++++++++++++++++++----------------
 1 file changed, 55 insertions(+), 36 deletions(-)

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 19c05dc8d6da..87d0b78a2de9 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -19,7 +19,7 @@
 from collections import OrderedDict, deque, namedtuple
 from functools import wraps
 
-from six import iteritems, itervalues
+from six import iteritems
 from six.moves import range
 
 from canonicaljson import json
@@ -347,8 +347,9 @@ def _persist_events(self, events_and_contexts, backfilled=False,
                 # state in each room after adding these events
                 current_state_for_room = {}
 
-                # map room_id->(to_delete, to_insert) where each entry is
-                # a map (type,key)->event_id giving the state delta in each
+                # map room_id->(to_delete, to_insert) where to_delete is a list
+                # of type/state keys to remove from current state, and to_insert
+                # is a map (type,key)->event_id giving the state delta in each
                 # room
                 state_delta_for_room = {}
 
@@ -647,17 +648,16 @@ def _calculate_state_delta(self, room_id, current_state):
         Assumes that we are only persisting events for one room at a time.
 
         Returns:
-            2-tuple (to_delete, to_insert) where both are state dicts,
-            i.e. (type, state_key) -> event_id. `to_delete` are the entries to
-            first be deleted from current_state_events, `to_insert` are entries
-            to insert.
+            tuple[list, dict] (to_delete, to_insert): where to_delete are the
+            type/state_keys to remove from current_state_events and `to_insert`
+            are the updates to current_state_events.
         """
         existing_state = yield self.get_current_state_ids(room_id)
 
-        to_delete = {
-            key: ev_id for key, ev_id in iteritems(existing_state)
-            if ev_id != current_state.get(key)
-        }
+        to_delete = [
+            key for key, ev_id in iteritems(existing_state)
+            if key not in current_state
+        ]
 
         to_insert = {
             key: ev_id for key, ev_id in iteritems(current_state)
@@ -684,10 +684,10 @@ def _persist_events_txn(self, txn, events_and_contexts, backfilled,
             delete_existing (bool): True to purge existing table rows for the
                 events from the database. This is useful when retrying due to
                 IntegrityError.
-            state_delta_for_room (dict[str, (list[str], list[str])]):
+            state_delta_for_room (dict[str, (list, dict)]):
                 The current-state delta for each room. For each room, a tuple
-                (to_delete, to_insert), being a list of event ids to be removed
-                from the current state, and a list of event ids to be added to
+                (to_delete, to_insert), being a list of type/state keys to be
+                removed from the current state, and a state set to be added to
                 the current state.
             new_forward_extremeties (dict[str, list[str]]):
                 The new forward extremities for each room. For each room, a
@@ -765,9 +765,46 @@ def _persist_events_txn(self, txn, events_and_contexts, backfilled,
     def _update_current_state_txn(self, txn, state_delta_by_room, max_stream_order):
         for room_id, current_state_tuple in iteritems(state_delta_by_room):
                 to_delete, to_insert = current_state_tuple
+
+                # First we add entries to the current_state_delta_stream. We
+                # do this before updating the current_state_events table so
+                # that we can use it to calculate the `prev_event_id`. (This
+                # allows us to not have to pull out the existing state
+                # unnecessarily).
+                sql = """
+                    INSERT INTO current_state_delta_stream
+                    (stream_id, room_id, type, state_key, event_id, prev_event_id)
+                    SELECT ?, ?, ?, ?, ?, (
+                        SELECT event_id FROM current_state_events
+                        WHERE room_id = ? AND type = ? AND state_key = ?
+                    )
+                """
+                txn.executemany(sql, (
+                    (
+                        max_stream_order, room_id, etype, state_key, None,
+                        room_id, etype, state_key,
+                    )
+                    for etype, state_key in to_delete
+                    # We sanity check that we're deleting rather than updating
+                    if (etype, state_key) not in to_insert
+                ))
+                txn.executemany(sql, (
+                    (
+                        max_stream_order, room_id, etype, state_key, ev_id,
+                        room_id, etype, state_key,
+                    )
+                    for (etype, state_key), ev_id in iteritems(to_insert)
+                ))
+
+                # Now we actually update the current_state_events table
+
                 txn.executemany(
-                    "DELETE FROM current_state_events WHERE event_id = ?",
-                    [(ev_id,) for ev_id in itervalues(to_delete)],
+                    "DELETE FROM current_state_events"
+                    " WHERE room_id = ? AND type = ? AND state_key = ?",
+                    (
+                        (room_id, etype, state_key)
+                        for etype, state_key in itertools.chain(to_delete, to_insert)
+                    ),
                 )
 
                 self._simple_insert_many_txn(
@@ -784,25 +821,6 @@ def _update_current_state_txn(self, txn, state_delta_by_room, max_stream_order):
                     ],
                 )
 
-                state_deltas = {key: None for key in to_delete}
-                state_deltas.update(to_insert)
-
-                self._simple_insert_many_txn(
-                    txn,
-                    table="current_state_delta_stream",
-                    values=[
-                        {
-                            "stream_id": max_stream_order,
-                            "room_id": room_id,
-                            "type": key[0],
-                            "state_key": key[1],
-                            "event_id": ev_id,
-                            "prev_event_id": to_delete.get(key, None),
-                        }
-                        for key, ev_id in iteritems(state_deltas)
-                    ]
-                )
-
                 txn.call_after(
                     self._curr_state_delta_stream_cache.entity_has_changed,
                     room_id, max_stream_order,
@@ -816,7 +834,8 @@ def _update_current_state_txn(self, txn, state_delta_by_room, max_stream_order):
                 # and which we have added, then we invlidate the caches for all
                 # those users.
                 members_changed = set(
-                    state_key for ev_type, state_key in state_deltas
+                    state_key
+                    for ev_type, state_key in itertools.chain(to_delete, to_insert)
                     if ev_type == EventTypes.Member
                 )
 

From 1a01a5b964d3ea373355684a91b9f7fd95726fbc Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Tue, 24 Jul 2018 14:03:15 +0100
Subject: [PATCH 130/830] clarify comment on p_ids

---
 synapse/handlers/sync.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 1422843af829..4ced3144c8eb 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -1553,7 +1553,8 @@ def _calculate_state(
     # as we may not have sent them to the client before.  We find these membership
     # events by filtering them out of timeline_start, which has already been filtered
     # to only include membership events for the senders in the timeline.
-    # In practice, we can do this by removing them from the p_ids list.
+    # In practice, we can do this by removing them from the p_ids list,
+    # which is the list of relevant state we know we have already sent to the client.
     # see https://github.com/matrix-org/synapse/pull/2970
     #            /files/efcdacad7d1b7f52f879179701c7e0d9b763511f#r204732809
 

From a79410e7b8dce3fb8b4b40901a66fd9fde9c47a4 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 24 Jul 2018 11:13:47 +0100
Subject: [PATCH 131/830] Have _get_new_state_after_events return delta

If we have a delta from the existing to new current state, then we can
reuse that rather than manually working it out by fetching both lots of
state.
---
 synapse/storage/events.py | 65 +++++++++++++++++++++++++++++----------
 1 file changed, 48 insertions(+), 17 deletions(-)

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 87d0b78a2de9..6d2aae9c4a4a 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -344,7 +344,9 @@ def _persist_events(self, events_and_contexts, backfilled=False,
                 new_forward_extremeties = {}
 
                 # map room_id->(type,state_key)->event_id tracking the full
-                # state in each room after adding these events
+                # state in each room after adding these events.
+                # This is simply used to prefill the get_current_state_ids
+                # cache
                 current_state_for_room = {}
 
                 # map room_id->(to_delete, to_insert) where to_delete is a list
@@ -419,28 +421,40 @@ def _persist_events(self, events_and_contexts, backfilled=False,
                             logger.info(
                                 "Calculating state delta for room %s", room_id,
                             )
-
                             with Measure(
-                                    self._clock,
-                                    "persist_events.get_new_state_after_events",
+                                self._clock,
+                                "persist_events.get_new_state_after_events",
                             ):
-                                current_state = yield self._get_new_state_after_events(
+                                res = yield self._get_new_state_after_events(
                                     room_id,
                                     ev_ctx_rm,
                                     latest_event_ids,
                                     new_latest_event_ids,
                                 )
-
-                            if current_state is not None:
-                                current_state_for_room[room_id] = current_state
+                                current_state, delta_ids = res
+
+                            # If either are not None then there has been a change,
+                            # and we need to work out the delta (or use that
+                            # given)
+                            if delta_ids is not None:
+                                # If there is a delta we know that we've
+                                # only added or replaced state, never
+                                # removed keys entirely.
+                                state_delta_for_room[room_id] = ([], delta_ids)
+                            elif current_state is not None:
                                 with Measure(
-                                        self._clock,
-                                        "persist_events.calculate_state_delta",
+                                    self._clock,
+                                    "persist_events.calculate_state_delta",
                                 ):
                                     delta = yield self._calculate_state_delta(
                                         room_id, current_state,
                                     )
-                                    state_delta_for_room[room_id] = delta
+                                state_delta_for_room[room_id] = delta
+
+                            # If we have the current_state then lets prefill
+                            # the cache with it.
+                            if current_state is not None:
+                                current_state_for_room[room_id] = current_state
 
                 yield self.runInteraction(
                     "persist_events",
@@ -539,9 +553,10 @@ def _get_new_state_after_events(self, room_id, events_context, old_latest_event_
                 the new forward extremities for the room.
 
         Returns:
-            Deferred[dict[(str,str), str]|None]:
-                None if there are no changes to the room state, or
-                a dict of (type, state_key) -> event_id].
+            Deferred[tuple[dict[(str,str), str]|None, dict[(str,str), str]|None]]:
+            Returns a tuple of two state maps, the first being the full new current
+            state and the second being the delta to the existing current state.
+            If both are None then there has been no change.
         """
 
         if not new_latest_event_ids:
@@ -549,6 +564,9 @@ def _get_new_state_after_events(self, room_id, events_context, old_latest_event_
 
         # map from state_group to ((type, key) -> event_id) state map
         state_groups_map = {}
+
+        state_group_deltas = {}
+
         for ev, ctx in events_context:
             if ctx.state_group is None:
                 # I don't think this can happen, but let's double-check
@@ -567,6 +585,9 @@ def _get_new_state_after_events(self, room_id, events_context, old_latest_event_
             if current_state_ids is not None:
                 state_groups_map[ctx.state_group] = current_state_ids
 
+            if ctx.prev_group:
+                state_group_deltas[(ctx.prev_group, ctx.state_group)] = ctx.delta_ids
+
         # We need to map the event_ids to their state groups. First, let's
         # check if the event is one we're persisting, in which case we can
         # pull the state group from its context.
@@ -608,7 +629,7 @@ def _get_new_state_after_events(self, room_id, events_context, old_latest_event_
         # If they old and new groups are the same then we don't need to do
         # anything.
         if old_state_groups == new_state_groups:
-            return
+            defer.returnValue((None, None))
 
         # Now that we have calculated new_state_groups we need to get
         # their state IDs so we can resolve to a single state set.
@@ -620,7 +641,17 @@ def _get_new_state_after_events(self, room_id, events_context, old_latest_event_
         if len(new_state_groups) == 1:
             # If there is only one state group, then we know what the current
             # state is.
-            defer.returnValue(state_groups_map[new_state_groups.pop()])
+            new_state_group = new_state_groups.pop()
+
+            delta_ids = None
+            if len(old_state_groups) == 1:
+                old_state_group = old_state_groups.pop()
+
+                delta_ids = state_group_deltas.get(
+                    (old_state_group, new_state_group,), None
+                )
+
+            defer.returnValue((state_groups_map[new_state_group], delta_ids))
 
         # Ok, we need to defer to the state handler to resolve our state sets.
 
@@ -639,7 +670,7 @@ def get_events(ev_ids):
             room_id, state_groups, events_map, get_events
         )
 
-        defer.returnValue(res.state)
+        defer.returnValue((res.state, None))
 
     @defer.inlineCallbacks
     def _calculate_state_delta(self, room_id, current_state):

From 811ac73a42db16db7211a7517ad5651a1f6aee71 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 24 Jul 2018 13:40:42 +0100
Subject: [PATCH 132/830] Don't fetch state from the database unless needed

---
 synapse/storage/events.py | 30 ++++++++++++++++++------------
 1 file changed, 18 insertions(+), 12 deletions(-)

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 6d2aae9c4a4a..eb8bbe13ab86 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -631,27 +631,33 @@ def _get_new_state_after_events(self, room_id, events_context, old_latest_event_
         if old_state_groups == new_state_groups:
             defer.returnValue((None, None))
 
-        # Now that we have calculated new_state_groups we need to get
-        # their state IDs so we can resolve to a single state set.
-        missing_state = new_state_groups - set(state_groups_map)
-        if missing_state:
-            group_to_state = yield self._get_state_for_groups(missing_state)
-            state_groups_map.update(group_to_state)
-
         if len(new_state_groups) == 1:
             # If there is only one state group, then we know what the current
             # state is.
-            new_state_group = new_state_groups.pop()
-
-            delta_ids = None
             if len(old_state_groups) == 1:
-                old_state_group = old_state_groups.pop()
+                new_state_group = next(iter(new_state_groups))
+                old_state_group = next(iter(old_state_groups))
 
                 delta_ids = state_group_deltas.get(
                     (old_state_group, new_state_group,), None
                 )
+                if delta_ids is not None:
+                    # We have a delta from the existing to new current state,
+                    # so lets just return that. If we happen to already have
+                    # the current state in memory then lets also return that,
+                    # but it doesn't matter if we don't.
+                    new_state = state_groups_map.get(new_state_group)
+                    defer.returnValue((new_state, delta_ids))
 
-            defer.returnValue((state_groups_map[new_state_group], delta_ids))
+        # Now that we have calculated new_state_groups we need to get
+        # their state IDs so we can resolve to a single state set.
+        missing_state = new_state_groups - set(state_groups_map)
+        if missing_state:
+            group_to_state = yield self._get_state_for_groups(missing_state)
+            state_groups_map.update(group_to_state)
+
+        if len(new_state_groups) == 1:
+            defer.returnValue((state_groups_map[new_state_groups.pop()], None))
 
         # Ok, we need to defer to the state handler to resolve our state sets.
 

From f33c596533e46335a3fc4d02236087412615ee1a Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 24 Jul 2018 14:28:04 +0100
Subject: [PATCH 133/830] Newsfile

---
 changelog.d/3595.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3595.misc

diff --git a/changelog.d/3595.misc b/changelog.d/3595.misc
new file mode 100644
index 000000000000..85903504cc2f
--- /dev/null
+++ b/changelog.d/3595.misc
@@ -0,0 +1 @@
+Attempt to reduce amount of state pulled out of DB during persist_events

From ed0dd68731fa1549b4ea9b2dab08c2afb6ccb64e Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 24 Jul 2018 14:31:38 +0100
Subject: [PATCH 134/830] Fixup comment (and indent)

---
 synapse/storage/events.py | 36 ++++++++++++++++++++----------------
 1 file changed, 20 insertions(+), 16 deletions(-)

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index eb8bbe13ab86..76cfbc90fe39 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -565,6 +565,7 @@ def _get_new_state_after_events(self, room_id, events_context, old_latest_event_
         # map from state_group to ((type, key) -> event_id) state map
         state_groups_map = {}
 
+        # Map from (prev state group, new state group) -> delta state dict
         state_group_deltas = {}
 
         for ev, ctx in events_context:
@@ -631,23 +632,24 @@ def _get_new_state_after_events(self, room_id, events_context, old_latest_event_
         if old_state_groups == new_state_groups:
             defer.returnValue((None, None))
 
-        if len(new_state_groups) == 1:
-            # If there is only one state group, then we know what the current
-            # state is.
-            if len(old_state_groups) == 1:
-                new_state_group = next(iter(new_state_groups))
-                old_state_group = next(iter(old_state_groups))
+        if len(new_state_groups) == 1 and len(old_state_groups) == 1:
+            # If we're going from one state group to another, lets check if
+            # we have a delta for that transition. If we do then we can just
+            # return that.
 
-                delta_ids = state_group_deltas.get(
-                    (old_state_group, new_state_group,), None
-                )
-                if delta_ids is not None:
-                    # We have a delta from the existing to new current state,
-                    # so lets just return that. If we happen to already have
-                    # the current state in memory then lets also return that,
-                    # but it doesn't matter if we don't.
-                    new_state = state_groups_map.get(new_state_group)
-                    defer.returnValue((new_state, delta_ids))
+            new_state_group = next(iter(new_state_groups))
+            old_state_group = next(iter(old_state_groups))
+
+            delta_ids = state_group_deltas.get(
+                (old_state_group, new_state_group,), None
+            )
+            if delta_ids is not None:
+                # We have a delta from the existing to new current state,
+                # so lets just return that. If we happen to already have
+                # the current state in memory then lets also return that,
+                # but it doesn't matter if we don't.
+                new_state = state_groups_map.get(new_state_group)
+                defer.returnValue((new_state, delta_ids))
 
         # Now that we have calculated new_state_groups we need to get
         # their state IDs so we can resolve to a single state set.
@@ -657,6 +659,8 @@ def _get_new_state_after_events(self, room_id, events_context, old_latest_event_
             state_groups_map.update(group_to_state)
 
         if len(new_state_groups) == 1:
+            # If there is only one state group, then we know what the current
+            # state is.
             defer.returnValue((state_groups_map[new_state_groups.pop()], None))
 
         # Ok, we need to defer to the state handler to resolve our state sets.

From 8f65ab98d2ab35bbb29c23cc7978d54b12fd3db0 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 24 Jul 2018 15:08:01 +0100
Subject: [PATCH 135/830] Remove unnecessary iteritems

---
 synapse/storage/events.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 76cfbc90fe39..9c94d3454f06 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -696,7 +696,7 @@ def _calculate_state_delta(self, room_id, current_state):
         existing_state = yield self.get_current_state_ids(room_id)
 
         to_delete = [
-            key for key, ev_id in iteritems(existing_state)
+            key for key in existing_state
             if key not in current_state
         ]
 

From 709c309b0e1756afd8b4514b95b0e901c567a0f1 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 24 Jul 2018 15:12:50 +0100
Subject: [PATCH 136/830] Expand on docstring comment about return value

---
 synapse/storage/events.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 9c94d3454f06..906a4050310a 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -557,6 +557,11 @@ def _get_new_state_after_events(self, room_id, events_context, old_latest_event_
             Returns a tuple of two state maps, the first being the full new current
             state and the second being the delta to the existing current state.
             If both are None then there has been no change.
+
+            If there has been a change then we only return the delta if its
+            already been calculated. Conversely if we do know the delta then
+            the new current state is only returned if we've already calculated
+            it.
         """
 
         if not new_latest_event_ids:

From 3188973857d5bab5c1faf968917ad0ced42b5a0d Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 24 Jul 2018 16:20:14 +0100
Subject: [PATCH 137/830] Pull out did_forget to worker store

---
 synapse/app/synchrotron.py    |  5 +---
 synapse/storage/roommember.py | 52 ++++++++++++++++++-----------------
 2 files changed, 28 insertions(+), 29 deletions(-)

diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py
index 26b9ec85f2e1..e201f18efdc5 100644
--- a/synapse/app/synchrotron.py
+++ b/synapse/app/synchrotron.py
@@ -55,7 +55,6 @@
 from synapse.server import HomeServer
 from synapse.storage.engines import create_engine
 from synapse.storage.presence import UserPresenceState
-from synapse.storage.roommember import RoomMemberStore
 from synapse.util.httpresourcetree import create_resource_tree
 from synapse.util.logcontext import LoggingContext, run_in_background
 from synapse.util.manhole import manhole
@@ -81,9 +80,7 @@ class SynchrotronSlavedStore(
     RoomStore,
     BaseSlavedStore,
 ):
-    did_forget = (
-        RoomMemberStore.__dict__["did_forget"]
-    )
+    pass
 
 
 UPDATE_SYNCING_USERS_MS = 10 * 1000
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index 01697ab2c977..027bf8c85ee2 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -461,6 +461,30 @@ def _get_joined_hosts(self, room_id, state_group, current_state_ids, state_entry
     def _get_joined_hosts_cache(self, room_id):
         return _JoinedHostsCache(self, room_id)
 
+    @cachedInlineCallbacks(num_args=2)
+    def did_forget(self, user_id, room_id):
+        """Returns whether user_id has elected to discard history for room_id.
+
+        Returns False if they have since re-joined."""
+        def f(txn):
+            sql = (
+                "SELECT"
+                "  COUNT(*)"
+                " FROM"
+                "  room_memberships"
+                " WHERE"
+                "  user_id = ?"
+                " AND"
+                "  room_id = ?"
+                " AND"
+                "  forgotten = 0"
+            )
+            txn.execute(sql, (user_id, room_id))
+            rows = txn.fetchall()
+            return rows[0][0]
+        count = yield self.runInteraction("did_forget_membership", f)
+        defer.returnValue(count == 0)
+
 
 class RoomMemberStore(RoomMemberWorkerStore):
     def __init__(self, db_conn, hs):
@@ -568,32 +592,10 @@ def f(txn):
             )
             txn.execute(sql, (user_id, room_id))
 
-            txn.call_after(self.did_forget.invalidate, (user_id, room_id))
-        return self.runInteraction("forget_membership", f)
-
-    @cachedInlineCallbacks(num_args=2)
-    def did_forget(self, user_id, room_id):
-        """Returns whether user_id has elected to discard history for room_id.
-
-        Returns False if they have since re-joined."""
-        def f(txn):
-            sql = (
-                "SELECT"
-                "  COUNT(*)"
-                " FROM"
-                "  room_memberships"
-                " WHERE"
-                "  user_id = ?"
-                " AND"
-                "  room_id = ?"
-                " AND"
-                "  forgotten = 0"
+            self._invalidate_cache_and_stream(
+                txn, self.did_forget, (user_id, room_id,),
             )
-            txn.execute(sql, (user_id, room_id))
-            rows = txn.fetchall()
-            return rows[0][0]
-        count = yield self.runInteraction("did_forget_membership", f)
-        defer.returnValue(count == 0)
+        return self.runInteraction("forget_membership", f)
 
     @defer.inlineCallbacks
     def _background_add_membership_profile(self, progress, batch_size):

From 8b8c4f34a336376610bf353f6aa5d71c5ef69980 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 24 Jul 2018 16:46:30 +0100
Subject: [PATCH 138/830] Replace usage of get_current_toke with
 StreamToken.START

This allows us to handle /context/ requests on the client_reader worker
without having to pull in all the various stream handlers (e.g.
precence, typing, pushers etc). The only thing the token gets used for
is pagination, and that ignores everything but the room portion of the
token.
---
 synapse/app/client_reader.py |  2 ++
 synapse/handlers/room.py     | 12 +++++++-----
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/synapse/app/client_reader.py b/synapse/app/client_reader.py
index 398bb36602dc..e2c91123db8c 100644
--- a/synapse/app/client_reader.py
+++ b/synapse/app/client_reader.py
@@ -31,6 +31,7 @@
 from synapse.metrics import RegistryProxy
 from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.replication.slave.storage._base import BaseSlavedStore
+from synapse.replication.slave.storage.account_data import SlavedAccountDataStore
 from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore
 from synapse.replication.slave.storage.client_ips import SlavedClientIpStore
 from synapse.replication.slave.storage.directory import DirectoryStore
@@ -58,6 +59,7 @@
 
 
 class ClientReaderSlavedStore(
+    SlavedAccountDataStore,
     SlavedEventStore,
     SlavedKeyStore,
     RoomStore,
diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index 6150b7e22650..003b848c0012 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -24,7 +24,7 @@
 
 from synapse.api.constants import EventTypes, JoinRules, RoomCreationPreset
 from synapse.api.errors import AuthError, Codes, StoreError, SynapseError
-from synapse.types import RoomAlias, RoomID, RoomStreamToken, UserID
+from synapse.types import RoomAlias, RoomID, RoomStreamToken, StreamToken, UserID
 from synapse.util import stringutils
 from synapse.visibility import filter_events_for_client
 
@@ -418,8 +418,6 @@ def get_event_context(self, user, room_id, event_id, limit):
         before_limit = math.floor(limit / 2.)
         after_limit = limit - before_limit
 
-        now_token = yield self.hs.get_event_sources().get_current_token()
-
         users = yield self.store.get_users_in_room(room_id)
         is_peeking = user.to_string() not in users
 
@@ -462,11 +460,15 @@ def filter_evts(events):
         )
         results["state"] = list(state[last_event_id].values())
 
-        results["start"] = now_token.copy_and_replace(
+        # We use a dummy token here as we only care about the room portion of
+        # the token, which we replace.
+        token = StreamToken.START
+
+        results["start"] = token.copy_and_replace(
             "room_key", results["start"]
         ).to_string()
 
-        results["end"] = now_token.copy_and_replace(
+        results["end"] = token.copy_and_replace(
             "room_key", results["end"]
         ).to_string()
 

From f14c866e376b3a99f324e5d205a9922086ebcf70 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 24 Jul 2018 16:51:19 +0100
Subject: [PATCH 139/830] Newsfile

---
 changelog.d/3597.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3597.misc

diff --git a/changelog.d/3597.misc b/changelog.d/3597.misc
new file mode 100644
index 000000000000..ea4a85e0aee7
--- /dev/null
+++ b/changelog.d/3597.misc
@@ -0,0 +1 @@
+Add support for client_reader to handle more APIs

From 1674a852382faf2eadc8b9a428d83b9cd2defe7e Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 24 Jul 2018 17:20:27 +0100
Subject: [PATCH 140/830] Move newsfile

---
 changelog.d/{3597.misc => 3597.feature} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename changelog.d/{3597.misc => 3597.feature} (100%)

diff --git a/changelog.d/3597.misc b/changelog.d/3597.feature
similarity index 100%
rename from changelog.d/3597.misc
rename to changelog.d/3597.feature

From 38eaa5280d49df8d0e52f93693a576b7e936bde5 Mon Sep 17 00:00:00 2001
From: Michael Telatynski <7t3chguy@gmail.com>
Date: Tue, 24 Jul 2018 17:25:42 +0100
Subject: [PATCH 141/830] add changelog entry for PR#3391

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>
---
 changelog.d/3391.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3391.bugfix

diff --git a/changelog.d/3391.bugfix b/changelog.d/3391.bugfix
new file mode 100644
index 000000000000..88eeb50df233
--- /dev/null
+++ b/changelog.d/3391.bugfix
@@ -0,0 +1 @@
+Default inviter_display_name to mxid for email invites
\ No newline at end of file

From cb5c37a57c387a74f6079008870cf024e674dfe5 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Tue, 24 Jul 2018 20:34:45 +0100
Subject: [PATCH 142/830] handle the edge case for _get_some_state_from_cache
 where types is []

---
 synapse/storage/state.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index 86f2c2e6b18a..989977c644ae 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -547,7 +547,13 @@ def include(typ, state_key):
                 return True
             return False
 
-        got_all = is_all or not missing_types
+        if types == [] and filtered_types is not None:
+            # special wildcard case for empty type-list but an explicit filtered_types
+            # which means that we'll try to return all types which aren't in the
+            # filtered_types list.  missing_types will always be empty, so we ignore it.
+            got_all = is_all
+        else:
+            got_all = is_all or not missing_types
 
         return {
             k: v for k, v in iteritems(state_dict_ids)

From ec56121b0d099824a1455260e05904f081b6d14a Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 25 Jul 2018 09:35:02 +0100
Subject: [PATCH 143/830] Correctly handle outliers during persist events

We incorrectly asserted that all contexts must have a non None state
group without consider outliers. This would usually be fine as the
assertion would never be hit, as there is a shortcut during persistence
if the forward extremities don't change.

However, if the outlier is being persisted with non-outlier events, the
function would be called and the assertion would be hit.

Fixes #3601
---
 synapse/storage/events.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 906a4050310a..bd05f23b064a 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -575,11 +575,12 @@ def _get_new_state_after_events(self, room_id, events_context, old_latest_event_
 
         for ev, ctx in events_context:
             if ctx.state_group is None:
-                # I don't think this can happen, but let's double-check
-                raise Exception(
-                    "Context for new extremity event %s has no state "
-                    "group" % (ev.event_id, ),
-                )
+                # This should only happen for outlier events.
+                if not event.internal_metadata.is_outlier():
+                    raise Exception(
+                        "Context for new event %s has no state "
+                        "group" % (ev.event_id, ),
+                    )
 
             if ctx.state_group in state_groups_map:
                 continue
@@ -607,7 +608,7 @@ def _get_new_state_after_events(self, room_id, events_context, old_latest_event_
         for event_id in new_latest_event_ids:
             # First search in the list of new events we're adding.
             for ev, ctx in events_context:
-                if event_id == ev.event_id:
+                if event_id == ev.event_id and ctx.state_group is not None:
                     event_id_to_state_group[event_id] = ctx.state_group
                     break
             else:

From 0b300d323a35920c081b9fba4dbf0f639c2f6d4d Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 25 Jul 2018 09:39:45 +0100
Subject: [PATCH 144/830] Newsfile

---
 changelog.d/3601.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3601.bugfix

diff --git a/changelog.d/3601.bugfix b/changelog.d/3601.bugfix
new file mode 100644
index 000000000000..1678b261d049
--- /dev/null
+++ b/changelog.d/3601.bugfix
@@ -0,0 +1 @@
+Fix failure to persist events over federation under load

From 371da42ae4ec093f3887e9485d149cdfbedd4e58 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 25 Jul 2018 09:41:12 +0100
Subject: [PATCH 145/830] Wrap a number of things that run in the background

This will reduce the number of "Starting db connection from sentinel context"
warnings, and will help with our metrics.
---
 synapse/app/homeserver.py                     | 13 ++++++++++---
 synapse/groups/attestations.py                |  6 +++++-
 synapse/replication/tcp/resource.py           | 14 ++++++++------
 synapse/rest/media/v1/media_repository.py     |  8 +++++++-
 synapse/rest/media/v1/preview_url_resource.py |  8 +++++++-
 synapse/storage/devices.py                    |  8 ++++++--
 synapse/storage/event_federation.py           |  9 ++++++---
 synapse/storage/event_push_actions.py         | 13 +++++++++----
 synapse/storage/transactions.py               |  6 +++++-
 9 files changed, 63 insertions(+), 22 deletions(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 2ad1beb8d88c..b7e7718290af 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -49,6 +49,7 @@
 from synapse.http.server import RootRedirect
 from synapse.http.site import SynapseSite
 from synapse.metrics import RegistryProxy
+from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.module_api import ModuleApi
 from synapse.python_dependencies import CONDITIONAL_REQUIREMENTS, check_requirements
@@ -427,6 +428,9 @@ def profiled(*args, **kargs):
     # currently either 0 or 1
     stats_process = []
 
+    def start_phone_stats_home():
+        run_as_background_process("phone_stats_home", phone_stats_home)
+
     @defer.inlineCallbacks
     def phone_stats_home():
         logger.info("Gathering stats for reporting")
@@ -498,7 +502,10 @@ def performance_stats_init():
             )
 
     def generate_user_daily_visit_stats():
-        hs.get_datastore().generate_user_daily_visits()
+        run_as_background_process(
+            "generate_user_daily_visits",
+            hs.get_datastore().generate_user_daily_visits,
+        )
 
     # Rather than update on per session basis, batch up the requests.
     # If you increase the loop period, the accuracy of user_daily_visits
@@ -507,7 +514,7 @@ def generate_user_daily_visit_stats():
 
     if hs.config.report_stats:
         logger.info("Scheduling stats reporting for 3 hour intervals")
-        clock.looping_call(phone_stats_home, 3 * 60 * 60 * 1000)
+        clock.looping_call(start_phone_stats_home, 3 * 60 * 60 * 1000)
 
         # We need to defer this init for the cases that we daemonize
         # otherwise the process ID we get is that of the non-daemon process
@@ -515,7 +522,7 @@ def generate_user_daily_visit_stats():
 
         # We wait 5 minutes to send the first set of stats as the server can
         # be quite busy the first few minutes
-        clock.call_later(5 * 60, phone_stats_home)
+        clock.call_later(5 * 60, start_phone_stats_home)
 
     if hs.config.daemonize and hs.config.print_pidfile:
         print (hs.config.pid_file)
diff --git a/synapse/groups/attestations.py b/synapse/groups/attestations.py
index 47452700a877..4216af0a2743 100644
--- a/synapse/groups/attestations.py
+++ b/synapse/groups/attestations.py
@@ -43,6 +43,7 @@
 from twisted.internet import defer
 
 from synapse.api.errors import SynapseError
+from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.types import get_domain_from_id
 from synapse.util.logcontext import run_in_background
 
@@ -129,7 +130,7 @@ def __init__(self, hs):
         self.attestations = hs.get_groups_attestation_signing()
 
         self._renew_attestations_loop = self.clock.looping_call(
-            self._renew_attestations, 30 * 60 * 1000,
+            self._start_renew_attestations, 30 * 60 * 1000,
         )
 
     @defer.inlineCallbacks
@@ -151,6 +152,9 @@ def on_renew_attestation(self, group_id, user_id, content):
 
         defer.returnValue({})
 
+    def _start_renew_attestations(self):
+        run_as_background_process("renew_attestations", self._renew_attestations)
+
     @defer.inlineCallbacks
     def _renew_attestations(self):
         """Called periodically to check if we need to update any of our attestations
diff --git a/synapse/replication/tcp/resource.py b/synapse/replication/tcp/resource.py
index 611fb66e1da1..fd59f1595f4e 100644
--- a/synapse/replication/tcp/resource.py
+++ b/synapse/replication/tcp/resource.py
@@ -25,6 +25,7 @@
 from twisted.internet.protocol import Factory
 
 from synapse.metrics import LaterGauge
+from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.util.metrics import Measure, measure_func
 
 from .protocol import ServerReplicationStreamProtocol
@@ -117,7 +118,6 @@ def on_shutdown(self):
         for conn in self.connections:
             conn.send_error("server shutting down")
 
-    @defer.inlineCallbacks
     def on_notifier_poke(self):
         """Checks if there is actually any new data and sends it to the
         connections if there are.
@@ -132,14 +132,16 @@ def on_notifier_poke(self):
                 stream.discard_updates_and_advance()
             return
 
-        # If we're in the process of checking for new updates, mark that fact
-        # and return
+        self.pending_updates = True
+
         if self.is_looping:
-            logger.debug("Noitifier poke loop already running")
-            self.pending_updates = True
+            logger.debug("Notifier poke loop already running")
             return
 
-        self.pending_updates = True
+        run_as_background_process("replication_notifier", self._run_notifier_loop)
+
+    @defer.inlineCallbacks
+    def _run_notifier_loop(self):
         self.is_looping = True
 
         try:
diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py
index 30242c525afc..5b13378caaf4 100644
--- a/synapse/rest/media/v1/media_repository.py
+++ b/synapse/rest/media/v1/media_repository.py
@@ -35,6 +35,7 @@
     SynapseError,
 )
 from synapse.http.matrixfederationclient import MatrixFederationHttpClient
+from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.util.async import Linearizer
 from synapse.util.logcontext import make_deferred_yieldable
 from synapse.util.retryutils import NotRetryingDestination
@@ -100,10 +101,15 @@ def __init__(self, hs):
         )
 
         self.clock.looping_call(
-            self._update_recently_accessed,
+            self._start_update_recently_accessed,
             UPDATE_RECENTLY_ACCESSED_TS,
         )
 
+    def _start_update_recently_accessed(self):
+        run_as_background_process(
+            "update_recently_accessed_media", self._update_recently_accessed,
+        )
+
     @defer.inlineCallbacks
     def _update_recently_accessed(self):
         remote_media = self.recently_accessed_remotes
diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py
index b70b15c4c29b..4efd5339a455 100644
--- a/synapse/rest/media/v1/preview_url_resource.py
+++ b/synapse/rest/media/v1/preview_url_resource.py
@@ -41,6 +41,7 @@
     wrap_json_request_handler,
 )
 from synapse.http.servlet import parse_integer, parse_string
+from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.util.async import ObservableDeferred
 from synapse.util.caches.expiringcache import ExpiringCache
 from synapse.util.logcontext import make_deferred_yieldable, run_in_background
@@ -81,7 +82,7 @@ def __init__(self, hs, media_repo, media_storage):
         self._cache.start()
 
         self._cleaner_loop = self.clock.looping_call(
-            self._expire_url_cache_data, 10 * 1000
+            self._start_expire_url_cache_data, 10 * 1000,
         )
 
     def render_OPTIONS(self, request):
@@ -371,6 +372,11 @@ def _download_url(self, url, user):
             "etag": headers["ETag"][0] if "ETag" in headers else None,
         })
 
+    def _start_expire_url_cache_data(self):
+        run_as_background_process(
+            "expire_url_cache_data", self._expire_url_cache_data,
+        )
+
     @defer.inlineCallbacks
     def _expire_url_cache_data(self):
         """Clean up expired url cache content, media and thumbnails.
diff --git a/synapse/storage/devices.py b/synapse/storage/devices.py
index cc3cdf2ebccc..52dccb150744 100644
--- a/synapse/storage/devices.py
+++ b/synapse/storage/devices.py
@@ -21,6 +21,7 @@
 from twisted.internet import defer
 
 from synapse.api.errors import StoreError
+from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.util.caches.descriptors import cached, cachedInlineCallbacks, cachedList
 
 from ._base import Cache, SQLBaseStore
@@ -711,6 +712,9 @@ def _prune_txn(txn):
 
             logger.info("Pruned %d device list outbound pokes", txn.rowcount)
 
-        return self.runInteraction(
-            "_prune_old_outbound_device_pokes", _prune_txn
+        run_as_background_process(
+            "prune_old_outbound_device_pokes",
+            self.runInteraction,
+            "_prune_old_outbound_device_pokes",
+            _prune_txn,
         )
diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py
index 8d366d1b915e..65f2d19e208b 100644
--- a/synapse/storage/event_federation.py
+++ b/synapse/storage/event_federation.py
@@ -23,6 +23,7 @@
 from twisted.internet import defer
 
 from synapse.api.errors import StoreError
+from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.storage._base import SQLBaseStore
 from synapse.storage.events import EventsWorkerStore
 from synapse.storage.signatures import SignatureWorkerStore
@@ -446,7 +447,7 @@ def __init__(self, db_conn, hs):
         )
 
         hs.get_clock().looping_call(
-            self._delete_old_forward_extrem_cache, 60 * 60 * 1000
+            self._delete_old_forward_extrem_cache, 60 * 60 * 1000,
         )
 
     def _update_min_depth_for_room_txn(self, txn, room_id, depth):
@@ -548,9 +549,11 @@ def _delete_old_forward_extrem_cache_txn(txn):
                 sql,
                 (self.stream_ordering_month_ago, self.stream_ordering_month_ago,)
             )
-        return self.runInteraction(
+        run_as_background_process(
+            "delete_old_forward_extrem_cache",
+            self.runInteraction,
             "_delete_old_forward_extrem_cache",
-            _delete_old_forward_extrem_cache_txn
+            _delete_old_forward_extrem_cache_txn,
         )
 
     def clean_room_for_join(self, room_id):
diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py
index 29b511ae5e2a..4f44b0ad47da 100644
--- a/synapse/storage/event_push_actions.py
+++ b/synapse/storage/event_push_actions.py
@@ -22,6 +22,7 @@
 
 from twisted.internet import defer
 
+from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.storage._base import LoggingTransaction, SQLBaseStore
 from synapse.util.caches.descriptors import cachedInlineCallbacks
 
@@ -458,11 +459,12 @@ def remove_push_actions_from_staging(self, event_id):
                 "Error removing push actions after event persistence failure",
             )
 
-    @defer.inlineCallbacks
     def _find_stream_orderings_for_times(self):
-        yield self.runInteraction(
+        run_as_background_process(
+            "event_push_action_stream_orderings",
+            self.runInteraction,
             "_find_stream_orderings_for_times",
-            self._find_stream_orderings_for_times_txn
+            self._find_stream_orderings_for_times_txn,
         )
 
     def _find_stream_orderings_for_times_txn(self, txn):
@@ -604,7 +606,7 @@ def __init__(self, db_conn, hs):
 
         self._doing_notif_rotation = False
         self._rotate_notif_loop = self._clock.looping_call(
-            self._rotate_notifs, 30 * 60 * 1000
+            self._start_rotate_notifs, 30 * 60 * 1000,
         )
 
     def _set_push_actions_for_event_and_users_txn(self, txn, events_and_contexts,
@@ -787,6 +789,9 @@ def _remove_old_push_actions_before_txn(self, txn, room_id, user_id,
             WHERE room_id = ? AND user_id = ? AND stream_ordering <= ?
         """, (room_id, user_id, stream_ordering))
 
+    def _start_rotate_notifs(self):
+        run_as_background_process("rotate_notifs", self._rotate_notifs)
+
     @defer.inlineCallbacks
     def _rotate_notifs(self):
         if self._doing_notif_rotation or self.stream_ordering_day_ago is None:
diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py
index c3bc94f56d8a..b4b479d94ced 100644
--- a/synapse/storage/transactions.py
+++ b/synapse/storage/transactions.py
@@ -22,6 +22,7 @@
 
 from twisted.internet import defer
 
+from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.util.caches.descriptors import cached
 
 from ._base import SQLBaseStore
@@ -57,7 +58,7 @@ class TransactionStore(SQLBaseStore):
     def __init__(self, db_conn, hs):
         super(TransactionStore, self).__init__(db_conn, hs)
 
-        self._clock.looping_call(self._cleanup_transactions, 30 * 60 * 1000)
+        self._clock.looping_call(self._start_cleanup_transactions, 30 * 60 * 1000)
 
     def get_received_txn_response(self, transaction_id, origin):
         """For an incoming transaction from a given origin, check if we have
@@ -271,6 +272,9 @@ def _get_destinations_needing_retry(self, txn):
         txn.execute(query, (self._clock.time_msec(),))
         return self.cursor_to_dict(txn)
 
+    def _start_cleanup_transactions(self):
+        run_as_background_process("cleanup_transactions", self._cleanup_transactions)
+
     def _cleanup_transactions(self):
         now = self._clock.time_msec()
         month_ago = now - 30 * 24 * 60 * 60 * 1000

From 3f11d84534be93593251100118d928eeeb3c4d83 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 25 Jul 2018 09:43:25 +0100
Subject: [PATCH 146/830] Changelog

---
 changelog.d/3604.feature | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3604.feature

diff --git a/changelog.d/3604.feature b/changelog.d/3604.feature
new file mode 100644
index 000000000000..77a294cb9f42
--- /dev/null
+++ b/changelog.d/3604.feature
@@ -0,0 +1 @@
+Add metrics to track resource usage by background processes

From a297ff2b16094f358cc8432f1db660ca539f8b4d Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 25 Jul 2018 09:48:01 +0100
Subject: [PATCH 147/830] Fix typo in conditional

---
 synapse/storage/events.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index bd05f23b064a..5024c4714de6 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -576,7 +576,7 @@ def _get_new_state_after_events(self, room_id, events_context, old_latest_event_
         for ev, ctx in events_context:
             if ctx.state_group is None:
                 # This should only happen for outlier events.
-                if not event.internal_metadata.is_outlier():
+                if not ev.internal_metadata.is_outlier():
                     raise Exception(
                         "Context for new event %s has no state "
                         "group" % (ev.event_id, ),

From f59be4eb0e4c96d6744b4a479a04991bd53a50a6 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 25 Jul 2018 10:30:36 +0100
Subject: [PATCH 148/830] Fix unit tests

on_notifier_poke no longer runs synchonously, so we have to do a different hack
to make sure that the replication data has been sent. Let's actually listen for
its arrival.
---
 synapse/replication/tcp/client.py        |  2 +-
 tests/replication/slave/storage/_base.py | 37 +++++++++++++++++++-----
 2 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py
index e592ab57bfaf..970e94313e31 100644
--- a/synapse/replication/tcp/client.py
+++ b/synapse/replication/tcp/client.py
@@ -192,7 +192,7 @@ def await_sync(self, data):
         """Returns a deferred that is resolved when we receive a SYNC command
         with given data.
 
-        Used by tests.
+        [Not currently] used by tests.
         """
         return self.awaiting_syncs.setdefault(data, defer.Deferred())
 
diff --git a/tests/replication/slave/storage/_base.py b/tests/replication/slave/storage/_base.py
index 8708c8a19615..a103e7be802e 100644
--- a/tests/replication/slave/storage/_base.py
+++ b/tests/replication/slave/storage/_base.py
@@ -11,23 +11,44 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 import tempfile
 
 from mock import Mock, NonCallableMock
 
 from twisted.internet import defer, reactor
+from twisted.internet.defer import Deferred
 
 from synapse.replication.tcp.client import (
     ReplicationClientFactory,
     ReplicationClientHandler,
 )
 from synapse.replication.tcp.resource import ReplicationStreamProtocolFactory
+from synapse.util.logcontext import PreserveLoggingContext, make_deferred_yieldable
 
 from tests import unittest
 from tests.utils import setup_test_homeserver
 
 
+class TestReplicationClientHandler(ReplicationClientHandler):
+    """Overrides on_rdata so that we can wait for it to happen"""
+    def __init__(self, store):
+        super(TestReplicationClientHandler, self).__init__(store)
+        self._rdata_awaiters = []
+
+    def await_replication(self):
+        d = Deferred()
+        self._rdata_awaiters.append(d)
+        return make_deferred_yieldable(d)
+
+    def on_rdata(self, stream_name, token, rows):
+        awaiters = self._rdata_awaiters
+        self._rdata_awaiters = []
+        super(TestReplicationClientHandler, self).on_rdata(stream_name, token, rows)
+        with PreserveLoggingContext():
+            for a in awaiters:
+                a.callback(None)
+
+
 class BaseSlavedStoreTestCase(unittest.TestCase):
     @defer.inlineCallbacks
     def setUp(self):
@@ -52,7 +73,7 @@ def setUp(self):
         self.addCleanup(listener.stopListening)
         self.streamer = server_factory.streamer
 
-        self.replication_handler = ReplicationClientHandler(self.slaved_store)
+        self.replication_handler = TestReplicationClientHandler(self.slaved_store)
         client_factory = ReplicationClientFactory(
             self.hs, "client_name", self.replication_handler
         )
@@ -60,12 +81,14 @@ def setUp(self):
         self.addCleanup(client_factory.stopTrying)
         self.addCleanup(client_connector.disconnect)
 
-    @defer.inlineCallbacks
     def replicate(self):
-        yield self.streamer.on_notifier_poke()
-        d = self.replication_handler.await_sync("replication_test")
-        self.streamer.send_sync_to_all_connections("replication_test")
-        yield d
+        """Tell the master side of replication that something has happened, and then
+        wait for the replication to occur.
+        """
+        # xxx: should we be more specific in what we wait for?
+        d = self.replication_handler.await_replication()
+        self.streamer.on_notifier_poke()
+        return d
 
     @defer.inlineCallbacks
     def check(self, method, args, expected_result=None):

From 55acd6856cd86feb34edcbb5d9ce30e55a04e27f Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 25 Jul 2018 10:34:48 +0100
Subject: [PATCH 149/830] Fix updating of cached remote profiles

_update_remote_profile_cache was missing its `defer.inlineCallbacks`, so when
it was called, would just return a generator object, without actually running
any of the method body.
---
 synapse/handlers/profile.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py
index 859f6d2b2e08..43692b83a817 100644
--- a/synapse/handlers/profile.py
+++ b/synapse/handlers/profile.py
@@ -18,6 +18,7 @@
 from twisted.internet import defer
 
 from synapse.api.errors import AuthError, CodeMessageException, SynapseError
+from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.types import UserID, get_domain_from_id
 
 from ._base import BaseHandler
@@ -41,7 +42,7 @@ def __init__(self, hs):
 
         if hs.config.worker_app is None:
             self.clock.looping_call(
-                self._update_remote_profile_cache, self.PROFILE_UPDATE_MS,
+                self._start_update_remote_profile_cache, self.PROFILE_UPDATE_MS,
             )
 
     @defer.inlineCallbacks
@@ -254,6 +255,12 @@ def _update_join_states(self, requester, target_user):
                     room_id, str(e.message)
                 )
 
+    def _start_update_remote_profile_cache(self):
+        run_as_background_process(
+            "Update remote profile", self._update_remote_profile_cache,
+        )
+
+    @defer.inlineCallbacks
     def _update_remote_profile_cache(self):
         """Called periodically to check profiles of remote users we haven't
         checked in a while.

From 9c237a7ab567f8e0c1b088fdb918294998946e3a Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 25 Jul 2018 10:46:01 +0100
Subject: [PATCH 150/830] changelog

---
 changelog.d/3605.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3605.bugfix

diff --git a/changelog.d/3605.bugfix b/changelog.d/3605.bugfix
new file mode 100644
index 000000000000..786da546ebe9
--- /dev/null
+++ b/changelog.d/3605.bugfix
@@ -0,0 +1 @@
+Fix updating of cached remote profiles

From 07defd5fe6cc6f645195ddf0d679290bb214ca73 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 25 Jul 2018 10:47:57 +0100
Subject: [PATCH 151/830] Fix another logcontext leak in _persist_events

We need to run the errback in the sentinel context to avoid losing our own
context.

Also: add logging to runInteraction to help identify where "Starting db
connection from sentinel context" warnings are coming from
---
 changelog.d/3606.misc     | 1 +
 synapse/storage/_base.py  | 6 ++++++
 synapse/storage/events.py | 9 ++++-----
 3 files changed, 11 insertions(+), 5 deletions(-)
 create mode 100644 changelog.d/3606.misc

diff --git a/changelog.d/3606.misc b/changelog.d/3606.misc
new file mode 100644
index 000000000000..f0137766a066
--- /dev/null
+++ b/changelog.d/3606.misc
@@ -0,0 +1 @@
+Fix some random logcontext leaks.
\ No newline at end of file
diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 1d41d8d44535..44f37b4c1ee8 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -311,6 +311,12 @@ def runInteraction(self, desc, func, *args, **kwargs):
         after_callbacks = []
         exception_callbacks = []
 
+        if LoggingContext.current_context() == LoggingContext.sentinel:
+            logger.warn(
+                "Starting db txn '%s' from sentinel context",
+                desc,
+            )
+
         try:
             result = yield self.runWithConnection(
                 self._new_transaction,
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 906a4050310a..ee9e4d4b6585 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -142,15 +142,14 @@ def handle_queue_loop():
             try:
                 queue = self._get_drainining_queue(room_id)
                 for item in queue:
-                    # handle_queue_loop runs in the sentinel logcontext, so
-                    # there is no need to preserve_fn when running the
-                    # callbacks on the deferred.
                     try:
                         ret = yield per_item_callback(item)
+                    except Exception:
+                        with PreserveLoggingContext():
+                            item.deferred.errback()
+                    else:
                         with PreserveLoggingContext():
                             item.deferred.callback(ret)
-                    except Exception:
-                        item.deferred.errback()
             finally:
                 queue = self._event_persist_queues.pop(room_id, None)
                 if queue:

From 1be94440d38ad6af64486ce31c650d0540d4049c Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 25 Jul 2018 11:01:36 +0100
Subject: [PATCH 152/830] Fix occasional 'tuple index out of range' error

This fixes a bug in _delete_existing_rows_txn which was introduced in #3435
(though it's been on matrix-org-hotfixes for *years*). This code is only called
when there is some sort of conflict the first time we try to persist an event,
so it only happens rarely. Still, the exceptions are annoying.
---
 changelog.d/3607.bugfix   | 1 +
 synapse/storage/events.py | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/3607.bugfix

diff --git a/changelog.d/3607.bugfix b/changelog.d/3607.bugfix
new file mode 100644
index 000000000000..7ad64593b8f2
--- /dev/null
+++ b/changelog.d/3607.bugfix
@@ -0,0 +1 @@
+Fix 'tuple index out of range' error
\ No newline at end of file
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 906a4050310a..f7c4226ea20e 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -1137,7 +1137,7 @@ def _delete_existing_rows_txn(cls, txn, events_and_contexts):
         ):
             txn.executemany(
                 "DELETE FROM %s WHERE room_id = ? AND event_id = ?" % (table,),
-                [(ev.event_id,) for ev, _ in events_and_contexts]
+                [(ev.room_id, ev.event_id) for ev, _ in events_and_contexts]
             )
 
     def _store_event_txn(self, txn, events_and_contexts):

From 7780a7b47ca0fa039886dda8f6ca84bb197e9425 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 25 Jul 2018 11:13:20 +0100
Subject: [PATCH 153/830] Actually fix it by adding continue

---
 synapse/storage/events.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 5024c4714de6..d7fea0986b1e 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -581,6 +581,7 @@ def _get_new_state_after_events(self, room_id, events_context, old_latest_event_
                         "Context for new event %s has no state "
                         "group" % (ev.event_id, ),
                     )
+                continue
 
             if ctx.state_group in state_groups_map:
                 continue

From 78a691d005b925b6211a3d090add34c6efb1c0f4 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 25 Jul 2018 16:00:38 +0100
Subject: [PATCH 154/830] Split out DB writes in federation handler

This will allow us to easily add an internal replication API to proxy
these reqeusts to master, so that we can move federation APIs to
workers.
---
 synapse/handlers/federation.py | 165 ++++++++++++++++-----------------
 synapse/storage/events.py      |  14 ++-
 2 files changed, 94 insertions(+), 85 deletions(-)

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 145c1a21d4c8..06700d5038f4 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -444,7 +444,7 @@ def _process_received_pdu(self, origin, pdu, state, auth_chain):
                 yield self._handle_new_events(origin, event_infos)
 
             try:
-                context, event_stream_id, max_stream_id = yield self._handle_new_event(
+                context = yield self._handle_new_event(
                     origin,
                     event,
                     state=state,
@@ -469,17 +469,6 @@ def _process_received_pdu(self, origin, pdu, state, auth_chain):
             except StoreError:
                 logger.exception("Failed to store room.")
 
-        extra_users = []
-        if event.type == EventTypes.Member:
-            target_user_id = event.state_key
-            target_user = UserID.from_string(target_user_id)
-            extra_users.append(target_user)
-
-        self.notifier.on_new_room_event(
-            event, event_stream_id, max_stream_id,
-            extra_users=extra_users
-        )
-
         if event.type == EventTypes.Member:
             if event.membership == Membership.JOIN:
                 # Only fire user_joined_room if the user has acutally
@@ -501,7 +490,7 @@ def _process_received_pdu(self, origin, pdu, state, auth_chain):
 
                 if newly_joined:
                     user = UserID.from_string(event.state_key)
-                    yield user_joined_room(self.distributor, user, event.room_id)
+                    yield self.user_joined_room(user, event.room_id)
 
     @log_function
     @defer.inlineCallbacks
@@ -942,7 +931,7 @@ def do_invite_join(self, target_hosts, room_id, joinee, content):
 
         self.room_queues[room_id] = []
 
-        yield self.store.clean_room_for_join(room_id)
+        yield self._clean_room_for_join(room_id)
 
         handled_events = set()
 
@@ -981,15 +970,10 @@ def do_invite_join(self, target_hosts, room_id, joinee, content):
                 # FIXME
                 pass
 
-            event_stream_id, max_stream_id = yield self._persist_auth_tree(
+            yield self._persist_auth_tree(
                 origin, auth_chain, state, event
             )
 
-            self.notifier.on_new_room_event(
-                event, event_stream_id, max_stream_id,
-                extra_users=[joinee]
-            )
-
             logger.debug("Finished joining %s to %s", joinee, room_id)
         finally:
             room_queue = self.room_queues[room_id]
@@ -1084,7 +1068,7 @@ def on_send_join_request(self, origin, pdu):
         # would introduce the danger of backwards-compatibility problems.
         event.internal_metadata.send_on_behalf_of = origin
 
-        context, event_stream_id, max_stream_id = yield self._handle_new_event(
+        context = yield self._handle_new_event(
             origin, event
         )
 
@@ -1094,20 +1078,10 @@ def on_send_join_request(self, origin, pdu):
             event.signatures,
         )
 
-        extra_users = []
-        if event.type == EventTypes.Member:
-            target_user_id = event.state_key
-            target_user = UserID.from_string(target_user_id)
-            extra_users.append(target_user)
-
-        self.notifier.on_new_room_event(
-            event, event_stream_id, max_stream_id, extra_users=extra_users
-        )
-
         if event.type == EventTypes.Member:
             if event.content["membership"] == Membership.JOIN:
                 user = UserID.from_string(event.state_key)
-                yield user_joined_room(self.distributor, user, event.room_id)
+                yield self.user_joined_room(user, event.room_id)
 
         prev_state_ids = yield context.get_prev_state_ids(self.store)
 
@@ -1176,17 +1150,7 @@ def on_invite_request(self, origin, pdu):
         )
 
         context = yield self.state_handler.compute_event_context(event)
-
-        event_stream_id, max_stream_id = yield self.store.persist_event(
-            event,
-            context=context,
-        )
-
-        target_user = UserID.from_string(event.state_key)
-        self.notifier.on_new_room_event(
-            event, event_stream_id, max_stream_id,
-            extra_users=[target_user],
-        )
+        yield self._persist_events([(event, context)])
 
         defer.returnValue(event)
 
@@ -1217,17 +1181,7 @@ def do_remotely_reject_invite(self, target_hosts, room_id, user_id):
         )
 
         context = yield self.state_handler.compute_event_context(event)
-
-        event_stream_id, max_stream_id = yield self.store.persist_event(
-            event,
-            context=context,
-        )
-
-        target_user = UserID.from_string(event.state_key)
-        self.notifier.on_new_room_event(
-            event, event_stream_id, max_stream_id,
-            extra_users=[target_user],
-        )
+        yield self._persist_events([(event, context)])
 
         defer.returnValue(event)
 
@@ -1318,7 +1272,7 @@ def on_send_leave_request(self, origin, pdu):
 
         event.internal_metadata.outlier = False
 
-        context, event_stream_id, max_stream_id = yield self._handle_new_event(
+        yield self._handle_new_event(
             origin, event
         )
 
@@ -1328,16 +1282,6 @@ def on_send_leave_request(self, origin, pdu):
             event.signatures,
         )
 
-        extra_users = []
-        if event.type == EventTypes.Member:
-            target_user_id = event.state_key
-            target_user = UserID.from_string(target_user_id)
-            extra_users.append(target_user)
-
-        self.notifier.on_new_room_event(
-            event, event_stream_id, max_stream_id, extra_users=extra_users
-        )
-
         defer.returnValue(None)
 
     @defer.inlineCallbacks
@@ -1472,9 +1416,8 @@ def _handle_new_event(self, origin, event, state=None, auth_events=None,
                     event, context
                 )
 
-            event_stream_id, max_stream_id = yield self.store.persist_event(
-                event,
-                context=context,
+            yield self._persist_events(
+                [(event, context)],
                 backfilled=backfilled,
             )
         except:  # noqa: E722, as we reraise the exception this is fine.
@@ -1487,15 +1430,7 @@ def _handle_new_event(self, origin, event, state=None, auth_events=None,
 
             six.reraise(tp, value, tb)
 
-        if not backfilled:
-            # this intentionally does not yield: we don't care about the result
-            # and don't need to wait for it.
-            logcontext.run_in_background(
-                self.pusher_pool.on_new_notifications,
-                event_stream_id, max_stream_id,
-            )
-
-        defer.returnValue((context, event_stream_id, max_stream_id))
+        defer.returnValue(context)
 
     @defer.inlineCallbacks
     def _handle_new_events(self, origin, event_infos, backfilled=False):
@@ -1517,7 +1452,7 @@ def _handle_new_events(self, origin, event_infos, backfilled=False):
             ], consumeErrors=True,
         ))
 
-        yield self.store.persist_events(
+        yield self._persist_events(
             [
                 (ev_info["event"], context)
                 for ev_info, context in zip(event_infos, contexts)
@@ -1605,7 +1540,7 @@ def _persist_auth_tree(self, origin, auth_events, state, event):
                     raise
                 events_to_context[e.event_id].rejected = RejectedReason.AUTH_ERROR
 
-        yield self.store.persist_events(
+        yield self._persist_events(
             [
                 (e, events_to_context[e.event_id])
                 for e in itertools.chain(auth_events, state)
@@ -1616,12 +1551,10 @@ def _persist_auth_tree(self, origin, auth_events, state, event):
             event, old_state=state
         )
 
-        event_stream_id, max_stream_id = yield self.store.persist_event(
-            event, new_event_context,
+        yield self._persist_events(
+            [(event, new_event_context)],
         )
 
-        defer.returnValue((event_stream_id, max_stream_id))
-
     @defer.inlineCallbacks
     def _prep_event(self, origin, event, state=None, auth_events=None):
         """
@@ -2347,3 +2280,69 @@ def _check_key_revocation(self, public_key, url):
             )
         if "valid" not in response or not response["valid"]:
             raise AuthError(403, "Third party certificate was invalid")
+
+    @defer.inlineCallbacks
+    def _persist_events(self, event_and_contexts, backfilled=False):
+        """Persists events and tells the notifier/pushers about them, if
+        necessary.
+
+        Args:
+            event_and_contexts(list[tuple[FrozenEvent, EventContext]])
+            backfilled (bool): Whether these events are a result of
+                backfilling or not
+
+        Returns:
+            Deferred
+        """
+        max_stream_id = yield self.store.persist_events(
+            event_and_contexts,
+            backfilled=backfilled,
+        )
+
+        if not backfilled:  # Never notify for backfilled events
+            for event, _ in event_and_contexts:
+                self._notify_persisted_event(event, max_stream_id)
+
+    def _notify_persisted_event(self, event, max_stream_id):
+        """Checks to see if notifier/pushers should be notified about the
+        event or not.
+
+        Args:
+            event (FrozenEvent)
+            max_stream_id (int): The max_stream_id returned by persist_events
+        """
+
+        extra_users = []
+        if event.type == EventTypes.Member:
+            target_user_id = event.state_key
+
+            # We notify for memberships if its an invite for one of our
+            # users
+            if event.internal_metadata.is_outlier():
+                if event.membership != Membership.INVITE:
+                    if not self.is_mine_id(target_user_id):
+                        return
+
+            target_user = UserID.from_string(target_user_id)
+            extra_users.append(target_user)
+        elif event.internal_metadata.is_outlier():
+            return
+
+        event_stream_id = event.internal_metadata.stream_ordering
+        self.notifier.on_new_room_event(
+            event, event_stream_id, max_stream_id,
+            extra_users=extra_users
+        )
+
+        logcontext.run_in_background(
+            self.pusher_pool.on_new_notifications,
+            event_stream_id, max_stream_id,
+        )
+
+    def _clean_room_for_join(self, room_id):
+        return self.store.clean_room_for_join(room_id)
+
+    def user_joined_room(self, user, room_id):
+        """Called when a new user has joined the room
+        """
+        return user_joined_room(self.distributor, user, room_id)
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 200f5ec95f15..e3910ed28257 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -231,12 +231,18 @@ def __init__(self, db_conn, hs):
 
         self._state_resolution_handler = hs.get_state_resolution_handler()
 
+    @defer.inlineCallbacks
     def persist_events(self, events_and_contexts, backfilled=False):
         """
         Write events to the database
         Args:
             events_and_contexts: list of tuples of (event, context)
-            backfilled: ?
+            backfilled (bool): Whether the results are retrieved from federation
+                via backfill or not. Used to determine if they're "new" events
+                which might update the current state etc.
+
+        Returns:
+            Deferred[int]: he stream ordering of the latest persisted event
         """
         partitioned = {}
         for event, ctx in events_and_contexts:
@@ -253,10 +259,14 @@ def persist_events(self, events_and_contexts, backfilled=False):
         for room_id in partitioned:
             self._maybe_start_persisting(room_id)
 
-        return make_deferred_yieldable(
+        yield make_deferred_yieldable(
             defer.gatherResults(deferreds, consumeErrors=True)
         )
 
+        max_persisted_id = yield self._stream_id_gen.get_current_token()
+
+        defer.returnValue(max_persisted_id)
+
     @defer.inlineCallbacks
     @log_function
     def persist_event(self, event, context, backfilled=False):

From 7d9fb88617f475ac7e064c5cccc8d78dbd78d2a3 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Wed, 25 Jul 2018 16:15:33 +0100
Subject: [PATCH 155/830] incorporate more review.

---
 synapse/storage/state.py | 30 ++++++++++++++----------------
 1 file changed, 14 insertions(+), 16 deletions(-)

diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index 989977c644ae..e38427bf9d42 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -493,12 +493,6 @@ def _get_state_group_for_events(self, event_ids):
     def _get_some_state_from_cache(self, group, types, filtered_types=None):
         """Checks if group is in cache. See `_get_state_for_groups`
 
-        Returns 3-tuple (`state_dict`, `missing_types`, `got_all`).
-        `missing_types` is the list of types that aren't in the cache for that
-        group. `got_all` is a bool indicating if we successfully retrieved all
-        requests state from the cache, if False we need to query the DB for the
-        missing state.
-
         Args:
             group(int): The state group to lookup
             types(list[str, str|None]): List of 2-tuples of the form
@@ -507,6 +501,11 @@ def _get_some_state_from_cache(self, group, types, filtered_types=None):
             filtered_types(list[str]|None): Only apply filtering via `types` to this
                 list of event types.  Other types of events are returned unfiltered.
                 If None, `types` filtering is applied to all events.
+
+        Returns 2-tuple (`state_dict`, `got_all`).
+        `got_all` is a bool indicating if we successfully retrieved all
+        requests state from the cache, if False we need to query the DB for the
+        missing state.
         """
         is_all, known_absent, state_dict_ids = self._state_group_cache.get(group)
 
@@ -520,7 +519,7 @@ def _get_some_state_from_cache(self, group, types, filtered_types=None):
 
             if (
                 state_key is None or
-                filtered_types is not None and typ not in filtered_types
+                (filtered_types is not None and typ not in filtered_types)
             ):
                 type_to_key[typ] = None
                 # we mark the type as missing from the cache because
@@ -547,18 +546,17 @@ def include(typ, state_key):
                 return True
             return False
 
-        if types == [] and filtered_types is not None:
-            # special wildcard case for empty type-list but an explicit filtered_types
-            # which means that we'll try to return all types which aren't in the
-            # filtered_types list.  missing_types will always be empty, so we ignore it.
-            got_all = is_all
-        else:
-            got_all = is_all or not missing_types
+        got_all = is_all
+        if not got_all:
+            # the cache is incomplete. We may still have got all the results we need, if
+            # we don't have any wildcards in the match list.
+            if not missing_types and filtered_types is None:
+                got_all = True
 
         return {
             k: v for k, v in iteritems(state_dict_ids)
             if include(k[0], k[1])
-        }, missing_types, got_all
+        }, got_all
 
     def _get_all_state_from_cache(self, group):
         """Checks if group is in cache. See `_get_state_for_groups`
@@ -603,7 +601,7 @@ def _get_state_for_groups(self, groups, types=None, filtered_types=None):
         missing_groups = []
         if types is not None:
             for group in set(groups):
-                state_dict_ids, _, got_all = self._get_some_state_from_cache(
+                state_dict_ids, got_all = self._get_some_state_from_cache(
                     group, types, filtered_types
                 )
                 results[group] = state_dict_ids

From 0a7ee0ab8b0794c8633177edca8f839d34c6a42a Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Wed, 25 Jul 2018 16:10:34 +0100
Subject: [PATCH 156/830] add tests for _get_some_state_from_cache

---
 tests/storage/test_state.py | 150 ++++++++++++++++++++++++++++++++++++
 1 file changed, 150 insertions(+)

diff --git a/tests/storage/test_state.py b/tests/storage/test_state.py
index b2f314e9db02..b7797c45e8fc 100644
--- a/tests/storage/test_state.py
+++ b/tests/storage/test_state.py
@@ -159,6 +159,8 @@ def test_get_state_for_event(self):
             (e3.type, e3.state_key): e3,
         }, state)
 
+        # check that types=[], filtered_types=[EventTypes.Member]
+        # doesn't return all members
         state = yield self.store.get_state_for_event(
             e5.event_id, [], filtered_types=[EventTypes.Member],
         )
@@ -167,3 +169,151 @@ def test_get_state_for_event(self):
             (e1.type, e1.state_key): e1,
             (e2.type, e2.state_key): e2,
         }, state)
+
+        ##################################
+        # _get_some_state_from_cache tests
+        ##################################
+
+        room_id = self.room.to_string()
+        group_ids = yield self.store.get_state_groups_ids(room_id, [e5.event_id])
+        group = group_ids.keys()[0]
+
+        # test that _get_some_state_from_cache correctly filters out members with types=[]
+        (state_dict, is_all) = yield self.store._get_some_state_from_cache(
+            group, [], filtered_types=[EventTypes.Member]
+        )
+
+        self.assertEqual(is_all, True)
+        self.assertDictEqual({
+            (e1.type, e1.state_key): e1.event_id,
+            (e2.type, e2.state_key): e2.event_id,
+        }, state_dict)
+
+        # test that _get_some_state_from_cache correctly filters out members with types=wildcard
+        (state_dict, is_all) = yield self.store._get_some_state_from_cache(
+            group, [(EventTypes.Member, None)], filtered_types=[EventTypes.Member]
+        )
+
+        self.assertEqual(is_all, True)
+        self.assertDictEqual({
+            (e1.type, e1.state_key): e1.event_id,
+            (e2.type, e2.state_key): e2.event_id,
+            (e3.type, e3.state_key): e3.event_id,
+            # e4 is overwritten by e5
+            (e5.type, e5.state_key): e5.event_id,
+        }, state_dict)
+
+        # test that _get_some_state_from_cache correctly filters out members with types=specific
+        (state_dict, is_all) = yield self.store._get_some_state_from_cache(
+            group, [(EventTypes.Member, e5.state_key)], filtered_types=[EventTypes.Member]
+        )
+
+        self.assertEqual(is_all, True)
+        self.assertDictEqual({
+            (e1.type, e1.state_key): e1.event_id,
+            (e2.type, e2.state_key): e2.event_id,
+            (e5.type, e5.state_key): e5.event_id,
+        }, state_dict)
+
+        # test that _get_some_state_from_cache correctly filters out members with types=specific
+        # and no filtered_types
+        (state_dict, is_all) = yield self.store._get_some_state_from_cache(
+            group, [(EventTypes.Member, e5.state_key)], filtered_types=None
+        )
+
+        self.assertEqual(is_all, True)
+        self.assertDictEqual({
+            (e5.type, e5.state_key): e5.event_id,
+        }, state_dict)
+
+        #######################################################
+        # deliberately remove e2 (room name) from the _state_group_cache
+
+        (is_all, known_absent, state_dict_ids) = self.store._state_group_cache.get(group)
+
+        self.assertEqual(is_all, True)
+        self.assertEqual(known_absent, set())
+        self.assertDictEqual(state_dict_ids, {
+            (e1.type, e1.state_key): e1.event_id,
+            (e2.type, e2.state_key): e2.event_id,
+            (e3.type, e3.state_key): e3.event_id,
+            # e4 is overwritten by e5
+            (e5.type, e5.state_key): e5.event_id,
+        })
+
+        state_dict_ids.pop((e2.type, e2.state_key))
+        self.store._state_group_cache.invalidate(group)
+        self.store._state_group_cache.update(
+            sequence=self.store._state_group_cache.sequence,
+            key=group,
+            value=state_dict_ids,
+            # list fetched keys so it knows it's partial
+            fetched_keys=(
+                (e1.type, e1.state_key),
+                (e3.type, e3.state_key),
+                (e5.type, e5.state_key),
+            )
+        )
+
+        (is_all, known_absent, state_dict_ids) = self.store._state_group_cache.get(group)
+
+        self.assertEqual(is_all, False)
+        self.assertEqual(known_absent, set([
+            (e1.type, e1.state_key),
+            (e3.type, e3.state_key),
+            (e5.type, e5.state_key),
+        ]))
+        self.assertDictEqual(state_dict_ids, {
+            (e1.type, e1.state_key): e1.event_id,
+            (e3.type, e3.state_key): e3.event_id,
+            (e5.type, e5.state_key): e5.event_id,
+        })
+
+        ###################################################
+        # test that things work with a partial cache
+
+        # test that _get_some_state_from_cache correctly filters out members with types=[]
+        room_id = self.room.to_string()
+        (state_dict, is_all) = yield self.store._get_some_state_from_cache(
+            group, [], filtered_types=[EventTypes.Member]
+        )
+
+        self.assertEqual(is_all, False)
+        self.assertDictEqual({
+            (e1.type, e1.state_key): e1.event_id,
+        }, state_dict)
+
+        # test that _get_some_state_from_cache correctly filters out members with types=wildcard
+        (state_dict, is_all) = yield self.store._get_some_state_from_cache(
+            group, [(EventTypes.Member, None)], filtered_types=[EventTypes.Member]
+        )
+
+        self.assertEqual(is_all, False)
+        self.assertDictEqual({
+            (e1.type, e1.state_key): e1.event_id,
+            (e3.type, e3.state_key): e3.event_id,
+            # e4 is overwritten by e5
+            (e5.type, e5.state_key): e5.event_id,
+        }, state_dict)
+
+        # test that _get_some_state_from_cache correctly filters out members with types=specific
+        (state_dict, is_all) = yield self.store._get_some_state_from_cache(
+            group, [(EventTypes.Member, e5.state_key)], filtered_types=[EventTypes.Member]
+        )
+
+        self.assertEqual(is_all, False)
+        self.assertDictEqual({
+            (e1.type, e1.state_key): e1.event_id,
+            (e5.type, e5.state_key): e5.event_id,
+        }, state_dict)
+
+        # test that _get_some_state_from_cache correctly filters out members with types=specific
+        # and no filtered_types
+        (state_dict, is_all) = yield self.store._get_some_state_from_cache(
+            group, [(EventTypes.Member, e5.state_key)], filtered_types=None
+        )
+
+        self.assertEqual(is_all, True)
+        self.assertDictEqual({
+            (e5.type, e5.state_key): e5.event_id,
+        }, state_dict)

From 0620d27f4d4a7f5c228c519b489efc8bc0c0a7d0 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Wed, 25 Jul 2018 17:21:17 +0100
Subject: [PATCH 157/830] flake8

---
 tests/storage/test_state.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/tests/storage/test_state.py b/tests/storage/test_state.py
index b7797c45e8fc..7a76d67b8c8b 100644
--- a/tests/storage/test_state.py
+++ b/tests/storage/test_state.py
@@ -170,15 +170,15 @@ def test_get_state_for_event(self):
             (e2.type, e2.state_key): e2,
         }, state)
 
-        ##################################
-        # _get_some_state_from_cache tests
-        ##################################
+        #######################################################
+        # _get_some_state_from_cache tests against a full cache
+        #######################################################
 
         room_id = self.room.to_string()
         group_ids = yield self.store.get_state_groups_ids(room_id, [e5.event_id])
         group = group_ids.keys()[0]
 
-        # test that _get_some_state_from_cache correctly filters out members with types=[]
+        # test _get_some_state_from_cache correctly filters out members with types=[]
         (state_dict, is_all) = yield self.store._get_some_state_from_cache(
             group, [], filtered_types=[EventTypes.Member]
         )
@@ -189,7 +189,7 @@ def test_get_state_for_event(self):
             (e2.type, e2.state_key): e2.event_id,
         }, state_dict)
 
-        # test that _get_some_state_from_cache correctly filters out members with types=wildcard
+        # test _get_some_state_from_cache correctly filters in members with wildcard types
         (state_dict, is_all) = yield self.store._get_some_state_from_cache(
             group, [(EventTypes.Member, None)], filtered_types=[EventTypes.Member]
         )
@@ -203,7 +203,7 @@ def test_get_state_for_event(self):
             (e5.type, e5.state_key): e5.event_id,
         }, state_dict)
 
-        # test that _get_some_state_from_cache correctly filters out members with types=specific
+        # test _get_some_state_from_cache correctly filters in members with specific types
         (state_dict, is_all) = yield self.store._get_some_state_from_cache(
             group, [(EventTypes.Member, e5.state_key)], filtered_types=[EventTypes.Member]
         )
@@ -215,7 +215,7 @@ def test_get_state_for_event(self):
             (e5.type, e5.state_key): e5.event_id,
         }, state_dict)
 
-        # test that _get_some_state_from_cache correctly filters out members with types=specific
+        # test _get_some_state_from_cache correctly filters in members with specific types
         # and no filtered_types
         (state_dict, is_all) = yield self.store._get_some_state_from_cache(
             group, [(EventTypes.Member, e5.state_key)], filtered_types=None
@@ -269,10 +269,10 @@ def test_get_state_for_event(self):
             (e5.type, e5.state_key): e5.event_id,
         })
 
-        ###################################################
+        ############################################
         # test that things work with a partial cache
 
-        # test that _get_some_state_from_cache correctly filters out members with types=[]
+        # test _get_some_state_from_cache correctly filters out members with types=[]
         room_id = self.room.to_string()
         (state_dict, is_all) = yield self.store._get_some_state_from_cache(
             group, [], filtered_types=[EventTypes.Member]
@@ -283,7 +283,7 @@ def test_get_state_for_event(self):
             (e1.type, e1.state_key): e1.event_id,
         }, state_dict)
 
-        # test that _get_some_state_from_cache correctly filters out members with types=wildcard
+        # test _get_some_state_from_cache correctly filters in members wildcard types
         (state_dict, is_all) = yield self.store._get_some_state_from_cache(
             group, [(EventTypes.Member, None)], filtered_types=[EventTypes.Member]
         )
@@ -296,7 +296,7 @@ def test_get_state_for_event(self):
             (e5.type, e5.state_key): e5.event_id,
         }, state_dict)
 
-        # test that _get_some_state_from_cache correctly filters out members with types=specific
+        # test _get_some_state_from_cache correctly filters in members with specific types
         (state_dict, is_all) = yield self.store._get_some_state_from_cache(
             group, [(EventTypes.Member, e5.state_key)], filtered_types=[EventTypes.Member]
         )
@@ -307,7 +307,7 @@ def test_get_state_for_event(self):
             (e5.type, e5.state_key): e5.event_id,
         }, state_dict)
 
-        # test that _get_some_state_from_cache correctly filters out members with types=specific
+        # test _get_some_state_from_cache correctly filters in members with specific types
         # and no filtered_types
         (state_dict, is_all) = yield self.store._get_some_state_from_cache(
             group, [(EventTypes.Member, e5.state_key)], filtered_types=None

From d8e65ed7e111243c08c0b87c9a49e7537c355074 Mon Sep 17 00:00:00 2001
From: Travis Ralston <travpc@gmail.com>
Date: Wed, 25 Jul 2018 15:44:41 -0600
Subject: [PATCH 158/830] Fix a minor documentation typo in on_make_leave

---
 synapse/handlers/federation.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 145c1a21d4c8..49068c06d994 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -1279,7 +1279,7 @@ def _sign_event(self, event):
     @log_function
     def on_make_leave_request(self, room_id, user_id):
         """ We've received a /make_leave/ request, so we create a partial
-        join event for the room and return that. We do *not* persist or
+        leave event for the room and return that. We do *not* persist or
         process it until the other server has signed it and sent it back.
         """
         builder = self.event_builder_factory.new({

From 6185650f9cb0b9db13d90b8447c441ecccfe1701 Mon Sep 17 00:00:00 2001
From: Travis Ralston <travpc@gmail.com>
Date: Wed, 25 Jul 2018 15:46:56 -0600
Subject: [PATCH 159/830] Create 3609.misc

---
 changelog.d/3609.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3609.misc

diff --git a/changelog.d/3609.misc b/changelog.d/3609.misc
new file mode 100644
index 000000000000..5b9566d0761f
--- /dev/null
+++ b/changelog.d/3609.misc
@@ -0,0 +1 @@
+Fix a documentation typo in on_make_leave_request

From bc7944e6d2ea0076badd0eba414e1ba7020eb1e6 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Wed, 25 Jul 2018 23:36:31 +0100
Subject: [PATCH 160/830] switch missing_types to be a bool

---
 synapse/storage/state.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index e38427bf9d42..b27b3ae14458 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -511,8 +511,8 @@ def _get_some_state_from_cache(self, group, types, filtered_types=None):
 
         type_to_key = {}
 
-        # tracks which of the requested types are missing from our cache
-        missing_types = set()
+        # tracks whether any of ourrequested types are missing from the cache
+        missing_types = False
 
         for typ, state_key in types:
             key = (typ, state_key)
@@ -526,13 +526,13 @@ def _get_some_state_from_cache(self, group, types, filtered_types=None):
                 # when the cache was populated it might have been done with a
                 # restricted set of state_keys, so the wildcard will not work
                 # and the cache may be incomplete.
-                missing_types.add(key)
+                missing_types = True
             else:
                 if type_to_key.get(typ, object()) is not None:
                     type_to_key.setdefault(typ, set()).add(state_key)
 
                 if key not in state_dict_ids and key not in known_absent:
-                    missing_types.add(key)
+                    missing_types = True
 
         sentinel = object()
 

From 03751a64203b169cbf33b636b6d940ca6d414c31 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 26 Jul 2018 11:44:26 +0100
Subject: [PATCH 161/830] Fix some looping_call calls which were broken in
 #3604

It turns out that looping_call does check the deferred returned by its
callback, and (at least in the case of client_ips), we were relying on this,
and I broke it in #3604.

Update run_as_background_process to return the deferred, and make sure we
return it to clock.looping_call.
---
 changelog.d/3610.feature                      |  1 +
 synapse/app/homeserver.py                     |  4 ++--
 synapse/groups/attestations.py                |  2 +-
 synapse/handlers/profile.py                   |  2 +-
 synapse/metrics/background_process_metrics.py | 10 ++++++++--
 synapse/rest/media/v1/media_repository.py     |  2 +-
 synapse/rest/media/v1/preview_url_resource.py |  2 +-
 synapse/storage/client_ips.py                 |  2 +-
 synapse/storage/devices.py                    |  2 +-
 synapse/storage/event_federation.py           |  2 +-
 synapse/storage/event_push_actions.py         |  4 ++--
 synapse/storage/transactions.py               |  4 +++-
 synapse/util/caches/expiringcache.py          |  2 +-
 13 files changed, 24 insertions(+), 15 deletions(-)
 create mode 100644 changelog.d/3610.feature

diff --git a/changelog.d/3610.feature b/changelog.d/3610.feature
new file mode 100644
index 000000000000..77a294cb9f42
--- /dev/null
+++ b/changelog.d/3610.feature
@@ -0,0 +1 @@
+Add metrics to track resource usage by background processes
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index b7e7718290af..57b815d777d7 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -429,7 +429,7 @@ def profiled(*args, **kargs):
     stats_process = []
 
     def start_phone_stats_home():
-        run_as_background_process("phone_stats_home", phone_stats_home)
+        return run_as_background_process("phone_stats_home", phone_stats_home)
 
     @defer.inlineCallbacks
     def phone_stats_home():
@@ -502,7 +502,7 @@ def performance_stats_init():
             )
 
     def generate_user_daily_visit_stats():
-        run_as_background_process(
+        return run_as_background_process(
             "generate_user_daily_visits",
             hs.get_datastore().generate_user_daily_visits,
         )
diff --git a/synapse/groups/attestations.py b/synapse/groups/attestations.py
index 4216af0a2743..b04f4234ca6e 100644
--- a/synapse/groups/attestations.py
+++ b/synapse/groups/attestations.py
@@ -153,7 +153,7 @@ def on_renew_attestation(self, group_id, user_id, content):
         defer.returnValue({})
 
     def _start_renew_attestations(self):
-        run_as_background_process("renew_attestations", self._renew_attestations)
+        return run_as_background_process("renew_attestations", self._renew_attestations)
 
     @defer.inlineCallbacks
     def _renew_attestations(self):
diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py
index 43692b83a817..cb5c6d587e0f 100644
--- a/synapse/handlers/profile.py
+++ b/synapse/handlers/profile.py
@@ -256,7 +256,7 @@ def _update_join_states(self, requester, target_user):
                 )
 
     def _start_update_remote_profile_cache(self):
-        run_as_background_process(
+        return run_as_background_process(
             "Update remote profile", self._update_remote_profile_cache,
         )
 
diff --git a/synapse/metrics/background_process_metrics.py b/synapse/metrics/background_process_metrics.py
index 9d820e44a6bd..ce678d5f75d9 100644
--- a/synapse/metrics/background_process_metrics.py
+++ b/synapse/metrics/background_process_metrics.py
@@ -151,13 +151,19 @@ def run_as_background_process(desc, func, *args, **kwargs):
     This should be used to wrap processes which are fired off to run in the
     background, instead of being associated with a particular request.
 
+    It returns a Deferred which completes when the function completes, but it doesn't
+    follow the synapse logcontext rules, which makes it appropriate for passing to
+    clock.looping_call and friends (or for firing-and-forgetting in the middle of a
+    normal synapse inlineCallbacks function).
+
     Args:
         desc (str): a description for this background process type
         func: a function, which may return a Deferred
         args: positional args for func
         kwargs: keyword args for func
 
-    Returns: None
+    Returns: Deferred which returns the result of func, but note that it does not
+        follow the synapse logcontext rules.
     """
     @defer.inlineCallbacks
     def run():
@@ -176,4 +182,4 @@ def run():
                 _background_processes[desc].remove(proc)
 
     with PreserveLoggingContext():
-        run()
+        return run()
diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py
index 5b13378caaf4..174ad20123cd 100644
--- a/synapse/rest/media/v1/media_repository.py
+++ b/synapse/rest/media/v1/media_repository.py
@@ -106,7 +106,7 @@ def __init__(self, hs):
         )
 
     def _start_update_recently_accessed(self):
-        run_as_background_process(
+        return run_as_background_process(
             "update_recently_accessed_media", self._update_recently_accessed,
         )
 
diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py
index 4efd5339a455..27aa0def2f02 100644
--- a/synapse/rest/media/v1/preview_url_resource.py
+++ b/synapse/rest/media/v1/preview_url_resource.py
@@ -373,7 +373,7 @@ def _download_url(self, url, user):
         })
 
     def _start_expire_url_cache_data(self):
-        run_as_background_process(
+        return run_as_background_process(
             "expire_url_cache_data", self._expire_url_cache_data,
         )
 
diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py
index 77ae10da3d7f..b8cefd43d6df 100644
--- a/synapse/storage/client_ips.py
+++ b/synapse/storage/client_ips.py
@@ -102,7 +102,7 @@ def update():
                 to_update,
             )
 
-        run_as_background_process(
+        return run_as_background_process(
             "update_client_ips", update,
         )
 
diff --git a/synapse/storage/devices.py b/synapse/storage/devices.py
index 52dccb150744..c0943ecf912a 100644
--- a/synapse/storage/devices.py
+++ b/synapse/storage/devices.py
@@ -712,7 +712,7 @@ def _prune_txn(txn):
 
             logger.info("Pruned %d device list outbound pokes", txn.rowcount)
 
-        run_as_background_process(
+        return run_as_background_process(
             "prune_old_outbound_device_pokes",
             self.runInteraction,
             "_prune_old_outbound_device_pokes",
diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py
index 65f2d19e208b..f269ec6fb34f 100644
--- a/synapse/storage/event_federation.py
+++ b/synapse/storage/event_federation.py
@@ -549,7 +549,7 @@ def _delete_old_forward_extrem_cache_txn(txn):
                 sql,
                 (self.stream_ordering_month_ago, self.stream_ordering_month_ago,)
             )
-        run_as_background_process(
+        return run_as_background_process(
             "delete_old_forward_extrem_cache",
             self.runInteraction,
             "_delete_old_forward_extrem_cache",
diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py
index 4f44b0ad47da..6840320641da 100644
--- a/synapse/storage/event_push_actions.py
+++ b/synapse/storage/event_push_actions.py
@@ -460,7 +460,7 @@ def remove_push_actions_from_staging(self, event_id):
             )
 
     def _find_stream_orderings_for_times(self):
-        run_as_background_process(
+        return run_as_background_process(
             "event_push_action_stream_orderings",
             self.runInteraction,
             "_find_stream_orderings_for_times",
@@ -790,7 +790,7 @@ def _remove_old_push_actions_before_txn(self, txn, room_id, user_id,
         """, (room_id, user_id, stream_ordering))
 
     def _start_rotate_notifs(self):
-        run_as_background_process("rotate_notifs", self._rotate_notifs)
+        return run_as_background_process("rotate_notifs", self._rotate_notifs)
 
     @defer.inlineCallbacks
     def _rotate_notifs(self):
diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py
index b4b479d94ced..428e7fa36e37 100644
--- a/synapse/storage/transactions.py
+++ b/synapse/storage/transactions.py
@@ -273,7 +273,9 @@ def _get_destinations_needing_retry(self, txn):
         return self.cursor_to_dict(txn)
 
     def _start_cleanup_transactions(self):
-        run_as_background_process("cleanup_transactions", self._cleanup_transactions)
+        return run_as_background_process(
+            "cleanup_transactions", self._cleanup_transactions,
+        )
 
     def _cleanup_transactions(self):
         now = self._clock.time_msec()
diff --git a/synapse/util/caches/expiringcache.py b/synapse/util/caches/expiringcache.py
index 465adc54a8e6..ce85b2ae1179 100644
--- a/synapse/util/caches/expiringcache.py
+++ b/synapse/util/caches/expiringcache.py
@@ -64,7 +64,7 @@ def start(self):
             return
 
         def f():
-            run_as_background_process(
+            return run_as_background_process(
                 "prune_cache_%s" % self._cache_name,
                 self._prune_cache,
             )

From 21e878ebb60afd27269fb5b4d6df3d0d8c570a7f Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 26 Jul 2018 12:48:51 +0100
Subject: [PATCH 162/830] Make EventStore inherit from EventFederationStore

(since it uses methods therein)

Turns out that we had a bunch of things which were incorrectly importing
EventWorkerStore from events.py rather than events_worker.py, which broke once
I removed the import into events.py.
---
 synapse/storage/__init__.py         | 2 +-
 synapse/storage/appservice.py       | 2 +-
 synapse/storage/event_federation.py | 2 +-
 synapse/storage/events.py           | 6 ++++--
 synapse/storage/roommember.py       | 2 +-
 synapse/storage/stream.py           | 2 +-
 6 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index ba88a5497975..3bd63cd195fa 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -66,6 +66,7 @@ class DataStore(RoomMemberStore, RoomStore,
                 PresenceStore, TransactionStore,
                 DirectoryStore, KeyStore, StateStore, SignatureStore,
                 ApplicationServiceStore,
+                EventsStore,
                 EventFederationStore,
                 MediaRepositoryStore,
                 RejectionsStore,
@@ -73,7 +74,6 @@ class DataStore(RoomMemberStore, RoomStore,
                 PusherStore,
                 PushRuleStore,
                 ApplicationServiceTransactionStore,
-                EventsStore,
                 ReceiptsStore,
                 EndToEndKeyStore,
                 SearchStore,
diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py
index 9f12b360bcbe..31248d5e068c 100644
--- a/synapse/storage/appservice.py
+++ b/synapse/storage/appservice.py
@@ -22,7 +22,7 @@
 
 from synapse.appservice import AppServiceTransaction
 from synapse.config.appservice import load_appservices
-from synapse.storage.events import EventsWorkerStore
+from synapse.storage.events_worker import EventsWorkerStore
 
 from ._base import SQLBaseStore
 
diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py
index 65f2d19e208b..14500ee59ced 100644
--- a/synapse/storage/event_federation.py
+++ b/synapse/storage/event_federation.py
@@ -25,7 +25,7 @@
 from synapse.api.errors import StoreError
 from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.storage._base import SQLBaseStore
-from synapse.storage.events import EventsWorkerStore
+from synapse.storage.events_worker import EventsWorkerStore
 from synapse.storage.signatures import SignatureWorkerStore
 from synapse.util.caches.descriptors import cached
 
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 200f5ec95f15..dbbfe04880fe 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -34,7 +34,7 @@
 from synapse.events import EventBase  # noqa: F401
 from synapse.events.snapshot import EventContext  # noqa: F401
 from synapse.metrics.background_process_metrics import run_as_background_process
-from synapse.storage.events_worker import EventsWorkerStore
+from synapse.storage.event_federation import EventFederationStore
 from synapse.types import RoomStreamToken, get_domain_from_id
 from synapse.util.async import ObservableDeferred
 from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
@@ -193,7 +193,9 @@ def f(self, *args, **kwargs):
     return f
 
 
-class EventsStore(EventsWorkerStore):
+# inherits from EventFederationStore so that we can call _update_backward_extremities
+# and _handle_mult_prev_events (though arguably those could both be moved in here)
+class EventsStore(EventFederationStore):
     EVENT_ORIGIN_SERVER_TS_NAME = "event_origin_server_ts"
     EVENT_FIELDS_SENDER_URL_UPDATE_NAME = "event_fields_sender_url"
 
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index 027bf8c85ee2..10dce21cea19 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -24,7 +24,7 @@
 from twisted.internet import defer
 
 from synapse.api.constants import EventTypes, Membership
-from synapse.storage.events import EventsWorkerStore
+from synapse.storage.events_worker import EventsWorkerStore
 from synapse.types import get_domain_from_id
 from synapse.util.async import Linearizer
 from synapse.util.caches import intern_string
diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py
index 66856342f0f9..9d85dbb1d6d3 100644
--- a/synapse/storage/stream.py
+++ b/synapse/storage/stream.py
@@ -43,7 +43,7 @@
 
 from synapse.storage._base import SQLBaseStore
 from synapse.storage.engines import PostgresEngine
-from synapse.storage.events import EventsWorkerStore
+from synapse.storage.events_worker import EventsWorkerStore
 from synapse.types import RoomStreamToken
 from synapse.util.caches.stream_change_cache import StreamChangeCache
 from synapse.util.logcontext import make_deferred_yieldable, run_in_background

From a15ed522675c06b4a451c7c9a8a1d2865e86fae5 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 26 Jul 2018 12:53:18 +0100
Subject: [PATCH 163/830] changelog

---
 changelog.d/3612.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3612.misc

diff --git a/changelog.d/3612.misc b/changelog.d/3612.misc
new file mode 100644
index 000000000000..f90d2f2ff518
--- /dev/null
+++ b/changelog.d/3612.misc
@@ -0,0 +1 @@
+Make EventStore inherit from EventFederationStore

From 1b4d73fa520a301ec35ba417d14a4549a44f33e0 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 26 Jul 2018 12:53:51 +0100
Subject: [PATCH 164/830] comment on event_edges

---
 synapse/storage/schema/full_schemas/16/event_edges.sql | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/synapse/storage/schema/full_schemas/16/event_edges.sql b/synapse/storage/schema/full_schemas/16/event_edges.sql
index 52eec883576b..6b5a5a88fae1 100644
--- a/synapse/storage/schema/full_schemas/16/event_edges.sql
+++ b/synapse/storage/schema/full_schemas/16/event_edges.sql
@@ -37,7 +37,8 @@ CREATE TABLE IF NOT EXISTS event_edges(
     event_id TEXT NOT NULL,
     prev_event_id TEXT NOT NULL,
     room_id TEXT NOT NULL,
-    is_state BOOL NOT NULL,
+    is_state BOOL NOT NULL,  -- true if this is a prev_state edge rather than a regular
+                             -- event dag edge.
     UNIQUE (event_id, prev_event_id, room_id, is_state)
 );
 

From bd4b25f4d07e07a2da0382cfc59a5a262883c0fc Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 26 Jul 2018 13:19:08 +0100
Subject: [PATCH 165/830] Remove some redundant joins on event_edges.room_id

We've long passed the point where it's possible to have the same event_id in
different tables, so these join conditions are redundant: we can just join on
event_id.

event_edges is of non-trivial size, and the room_id column is wasteful, so
let's stop reading from it. In future, we can stop writing to it, and then drop
it.
---
 synapse/storage/event_federation.py | 13 ++++++-------
 synapse/storage/events.py           |  1 -
 2 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py
index 65f2d19e208b..801581dcf7ad 100644
--- a/synapse/storage/event_federation.py
+++ b/synapse/storage/event_federation.py
@@ -114,9 +114,9 @@ def get_oldest_events_with_depth_in_room_txn(self, txn, room_id):
         sql = (
             "SELECT b.event_id, MAX(e.depth) FROM events as e"
             " INNER JOIN event_edges as g"
-            " ON g.event_id = e.event_id AND g.room_id = e.room_id"
+            " ON g.event_id = e.event_id"
             " INNER JOIN event_backward_extremities as b"
-            " ON g.prev_event_id = b.event_id AND g.room_id = b.room_id"
+            " ON g.prev_event_id = b.event_id"
             " WHERE b.room_id = ? AND g.is_state is ?"
             " GROUP BY b.event_id"
         )
@@ -330,8 +330,7 @@ def _get_backfill_events(self, txn, room_id, event_list, limit):
             "SELECT depth, prev_event_id FROM event_edges"
             " INNER JOIN events"
             " ON prev_event_id = events.event_id"
-            " AND event_edges.room_id = events.room_id"
-            " WHERE event_edges.room_id = ? AND event_edges.event_id = ?"
+            " WHERE event_edges.event_id = ?"
             " AND event_edges.is_state = ?"
             " LIMIT ?"
         )
@@ -365,7 +364,7 @@ def _get_backfill_events(self, txn, room_id, event_list, limit):
 
             txn.execute(
                 query,
-                (room_id, event_id, False, limit - len(event_results))
+                (event_id, False, limit - len(event_results))
             )
 
             for row in txn:
@@ -402,7 +401,7 @@ def _get_missing_events(self, txn, room_id, earliest_events, latest_events,
 
         query = (
             "SELECT prev_event_id FROM event_edges "
-            "WHERE room_id = ? AND event_id = ? AND is_state = ? "
+            "WHERE event_id = ? AND is_state = ? "
             "LIMIT ?"
         )
 
@@ -411,7 +410,7 @@ def _get_missing_events(self, txn, room_id, earliest_events, latest_events,
             for event_id in front:
                 txn.execute(
                     query,
-                    (room_id, event_id, False, limit - len(event_results))
+                    (event_id, False, limit - len(event_results))
                 )
 
                 for e_id, in txn:
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 200f5ec95f15..cb10fdedc002 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -520,7 +520,6 @@ def _calculate_new_extremeties(self, room_id, event_contexts, latest_event_ids):
             iterable=list(new_latest_event_ids),
             retcols=["prev_event_id"],
             keyvalues={
-                "room_id": room_id,
                 "is_state": False,
             },
             desc="_calculate_new_extremeties",

From cf78eaebad818df8a6bbe039a28689860523d2f9 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 26 Jul 2018 13:22:40 +0100
Subject: [PATCH 166/830] changelog

---
 changelog.d/3613.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3613.misc

diff --git a/changelog.d/3613.misc b/changelog.d/3613.misc
new file mode 100644
index 000000000000..d9378f6b49a3
--- /dev/null
+++ b/changelog.d/3613.misc
@@ -0,0 +1 @@
+Remove some redundant joins on event_edges.room_id

From 5c1d301fd9e669b6704d54caeb1e8a3a223ba053 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 26 Jul 2018 14:12:00 +0100
Subject: [PATCH 167/830] Stop populating events.content

This field is no longer read from, so we should stop populating it. Once we're
happy that this doesn't break everything, and a rollback is unlikely, we can
think about dropping the column.
---
 synapse/storage/events.py                     |  1 -
 .../delta/50/make_event_content_nullable.py   | 93 +++++++++++++++++++
 2 files changed, 93 insertions(+), 1 deletion(-)
 create mode 100644 synapse/storage/schema/delta/50/make_event_content_nullable.py

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 200f5ec95f15..94515cd153b9 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -1189,7 +1189,6 @@ def event_dict(event):
                     "type": event.type,
                     "processed": True,
                     "outlier": event.internal_metadata.is_outlier(),
-                    "content": encode_json(event.content).decode("UTF-8"),
                     "origin_server_ts": int(event.origin_server_ts),
                     "received_ts": self._clock.time_msec(),
                     "sender": event.sender,
diff --git a/synapse/storage/schema/delta/50/make_event_content_nullable.py b/synapse/storage/schema/delta/50/make_event_content_nullable.py
new file mode 100644
index 000000000000..fa4a289514dd
--- /dev/null
+++ b/synapse/storage/schema/delta/50/make_event_content_nullable.py
@@ -0,0 +1,93 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+We want to stop populating 'event.content', so we need to make it nullable.
+
+If this has to be rolled back, then the following should populate the missing data:
+
+Postgres:
+
+    UPDATE events SET content=(ej.json::json)->'content' FROM event_json ej
+    WHERE ej.event_id = events.event_id AND
+        stream_ordering < (
+            SELECT stream_ordering FROM events WHERE content IS NOT NULL
+            ORDER BY stream_ordering LIMIT 1
+        );
+
+    UPDATE events SET content=(ej.json::json)->'content' FROM event_json ej
+    WHERE ej.event_id = events.event_id AND
+        stream_ordering > (
+            SELECT stream_ordering FROM events WHERE content IS NOT NULL
+            ORDER BY stream_ordering DESC LIMIT 1
+        );
+
+SQLite:
+
+    UPDATE events SET content=(
+        SELECT json_extract(json,'$.content') FROM event_json ej
+        WHERE ej.event_id = events.event_id
+    )
+    WHERE
+        stream_ordering < (
+            SELECT stream_ordering FROM events WHERE content IS NOT NULL
+            ORDER BY stream_ordering LIMIT 1
+        )
+        OR stream_ordering > (
+            SELECT stream_ordering FROM events WHERE content IS NOT NULL
+            ORDER BY stream_ordering DESC LIMIT 1
+        );
+
+"""
+
+import logging
+
+from synapse.storage.engines import PostgresEngine
+
+logger = logging.getLogger(__name__)
+
+
+def run_create(cur, database_engine, *args, **kwargs):
+    if isinstance(database_engine, PostgresEngine):
+        cur.execute("""
+            ALTER TABLE events ALTER COLUMN content DROP NOT NULL;
+        """)
+        return
+
+    # sqlite is an arse about this. ref: https://www.sqlite.org/lang_altertable.html
+    cur.execute("PRAGMA schema_version")
+    (oldver,) = cur.fetchone()
+
+    cur.execute("SELECT sql FROM sqlite_master WHERE tbl_name='events' AND type='table'")
+    (oldsql,) = cur.fetchone()
+    sql = oldsql.replace("content TEXT NOT NULL", "content TEXT")
+    if sql == oldsql:
+        raise Exception("Couldn't find null constraint to drop in %s" % oldsql)
+
+    logger.info("Replacing definition of 'events' with: %s", sql)
+
+    cur.execute("PRAGMA writable_schema=ON")
+
+    cur.execute(
+        "UPDATE sqlite_master SET sql=? WHERE tbl_name='events' AND type='table'",
+        (sql, ),
+    )
+
+    cur.execute("PRAGMA schema_version=%i" % (oldver+1,))
+    cur.execute("PRAGMA writable_schema=OFF")
+
+
+def run_upgrade(*args, **kwargs):
+    pass

From 51d7df19158de0f21e659625bf43716ff0a700bc Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 26 Jul 2018 14:54:04 +0100
Subject: [PATCH 168/830] Create the column nullable

There's no real point in ever making the column non-nullable, and doing so
breaks the sytests.
---
 .../delta/50/make_event_content_nullable.py       | 15 +++++++--------
 synapse/storage/schema/full_schemas/16/im.sql     |  7 ++++++-
 2 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/synapse/storage/schema/delta/50/make_event_content_nullable.py b/synapse/storage/schema/delta/50/make_event_content_nullable.py
index fa4a289514dd..7d27342e3994 100644
--- a/synapse/storage/schema/delta/50/make_event_content_nullable.py
+++ b/synapse/storage/schema/delta/50/make_event_content_nullable.py
@@ -60,6 +60,10 @@
 
 
 def run_create(cur, database_engine, *args, **kwargs):
+    pass
+
+
+def run_upgrade(cur, database_engine, *args, **kwargs):
     if isinstance(database_engine, PostgresEngine):
         cur.execute("""
             ALTER TABLE events ALTER COLUMN content DROP NOT NULL;
@@ -67,27 +71,22 @@ def run_create(cur, database_engine, *args, **kwargs):
         return
 
     # sqlite is an arse about this. ref: https://www.sqlite.org/lang_altertable.html
-    cur.execute("PRAGMA schema_version")
-    (oldver,) = cur.fetchone()
 
     cur.execute("SELECT sql FROM sqlite_master WHERE tbl_name='events' AND type='table'")
     (oldsql,) = cur.fetchone()
+
     sql = oldsql.replace("content TEXT NOT NULL", "content TEXT")
     if sql == oldsql:
         raise Exception("Couldn't find null constraint to drop in %s" % oldsql)
 
     logger.info("Replacing definition of 'events' with: %s", sql)
 
+    cur.execute("PRAGMA schema_version")
+    (oldver,) = cur.fetchone()
     cur.execute("PRAGMA writable_schema=ON")
-
     cur.execute(
         "UPDATE sqlite_master SET sql=? WHERE tbl_name='events' AND type='table'",
         (sql, ),
     )
-
     cur.execute("PRAGMA schema_version=%i" % (oldver+1,))
     cur.execute("PRAGMA writable_schema=OFF")
-
-
-def run_upgrade(*args, **kwargs):
-    pass
diff --git a/synapse/storage/schema/full_schemas/16/im.sql b/synapse/storage/schema/full_schemas/16/im.sql
index ba5346806e9f..5f5cb8d01d93 100644
--- a/synapse/storage/schema/full_schemas/16/im.sql
+++ b/synapse/storage/schema/full_schemas/16/im.sql
@@ -19,7 +19,12 @@ CREATE TABLE IF NOT EXISTS events(
     event_id TEXT NOT NULL,
     type TEXT NOT NULL,
     room_id TEXT NOT NULL,
-    content TEXT NOT NULL,
+
+    -- 'content' used to be created NULLable, but as of delta 50 we drop that constraint.
+    -- the hack we use to drop the constraint doesn't work for an in-memory sqlite
+    -- database, which breaks the sytests. Hence, we no longer make it nullable.
+    content TEXT,
+
     unrecognized_keys TEXT,
     processed BOOL NOT NULL,
     outlier BOOL NOT NULL,

From 85531a06a27b0168b84d79fcc5102d2c204ba330 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 26 Jul 2018 14:55:29 +0100
Subject: [PATCH 169/830] changelog

---
 changelog.d/3614.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3614.misc

diff --git a/changelog.d/3614.misc b/changelog.d/3614.misc
new file mode 100644
index 000000000000..356f28471eaa
--- /dev/null
+++ b/changelog.d/3614.misc
@@ -0,0 +1 @@
+Stop populating events.content

From 7d32f0d745da6cbc2ec4bb28318488dfcab7b930 Mon Sep 17 00:00:00 2001
From: Travis Ralston <travpc@gmail.com>
Date: Thu, 26 Jul 2018 14:41:59 -0600
Subject: [PATCH 170/830] Update the send_leave path to be an event_id

It's still not used, however the parameter is an event ID not a transaction ID.
---
 synapse/federation/transport/server.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index c9beca27c2d6..8574898f0c25 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -404,10 +404,10 @@ def on_GET(self, origin, content, query, context, user_id):
 
 
 class FederationSendLeaveServlet(BaseFederationServlet):
-    PATH = "/send_leave/(?P<room_id>[^/]*)/(?P<txid>[^/]*)"
+    PATH = "/send_leave/(?P<room_id>[^/]*)/(?P<event_id>[^/]*)"
 
     @defer.inlineCallbacks
-    def on_PUT(self, origin, content, query, room_id, txid):
+    def on_PUT(self, origin, content, query, room_id, event_id):
         content = yield self.handler.on_send_leave_request(origin, content)
         defer.returnValue((200, content))
 

From 49254d43a62e5ace114d8f41c3959063391a1b26 Mon Sep 17 00:00:00 2001
From: Travis Ralston <travpc@gmail.com>
Date: Thu, 26 Jul 2018 14:45:48 -0600
Subject: [PATCH 171/830] Create 3616.misc

---
 changelog.d/3616.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3616.misc

diff --git a/changelog.d/3616.misc b/changelog.d/3616.misc
new file mode 100644
index 000000000000..04629faa367b
--- /dev/null
+++ b/changelog.d/3616.misc
@@ -0,0 +1 @@
+Update the /send_leave path registration to use event_id rather than a transaction ID.

From a75231b507e025eaaa4f06d8932c04fa4e942d48 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@arasphere.net>
Date: Thu, 26 Jul 2018 22:51:30 +0100
Subject: [PATCH 172/830] Deduplicate redundant lazy-loaded members (#3331)

* attempt at deduplicating lazy-loaded members

as per the proposal; we can deduplicate redundant lazy-loaded members
which are sent in the same sync sequence. we do this heuristically
rather than requiring the client to somehow tell us which members it
has chosen to cache, by instead caching the last N members sent to
a client, and not sending them again.  For now we hardcode N to 100.
Each cache for a given (user,device) tuple is in turn cached for up to
X minutes (to avoid the caches building up).  For now we hardcode X to 30.

* add include_redundant_members filter option & make it work

* remove stale todo

* add tests for _get_some_state_from_cache

* incorporate review
---
 changelog.d/3331.feature |  1 +
 synapse/api/filtering.py |  9 +++++
 synapse/handlers/sync.py | 87 ++++++++++++++++++++++++++++------------
 3 files changed, 72 insertions(+), 25 deletions(-)
 create mode 100644 changelog.d/3331.feature

diff --git a/changelog.d/3331.feature b/changelog.d/3331.feature
new file mode 100644
index 000000000000..e574b9bcc3af
--- /dev/null
+++ b/changelog.d/3331.feature
@@ -0,0 +1 @@
+add support for the include_redundant_members filter param as per MSC1227
diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py
index 7e767b9bf5af..186831e1181f 100644
--- a/synapse/api/filtering.py
+++ b/synapse/api/filtering.py
@@ -117,6 +117,9 @@
         "lazy_load_members": {
             "type": "boolean"
         },
+        "include_redundant_members": {
+            "type": "boolean"
+        },
     }
 }
 
@@ -267,6 +270,9 @@ def ephemeral_limit(self):
     def lazy_load_members(self):
         return self._room_state_filter.lazy_load_members()
 
+    def include_redundant_members(self):
+        return self._room_state_filter.include_redundant_members()
+
     def filter_presence(self, events):
         return self._presence_filter.filter(events)
 
@@ -426,6 +432,9 @@ def limit(self):
     def lazy_load_members(self):
         return self.filter_json.get("lazy_load_members", False)
 
+    def include_redundant_members(self):
+        return self.filter_json.get("include_redundant_members", False)
+
 
 def _matches_wildcard(actual_value, filter_value):
     if filter_value.endswith("*"):
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 4ced3144c8eb..dff1f67dcb45 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -26,6 +26,8 @@
 from synapse.push.clientformat import format_push_rules_for_user
 from synapse.types import RoomStreamToken
 from synapse.util.async import concurrently_execute
+from synapse.util.caches.expiringcache import ExpiringCache
+from synapse.util.caches.lrucache import LruCache
 from synapse.util.caches.response_cache import ResponseCache
 from synapse.util.logcontext import LoggingContext
 from synapse.util.metrics import Measure, measure_func
@@ -33,6 +35,14 @@
 
 logger = logging.getLogger(__name__)
 
+# Store the cache that tracks which lazy-loaded members have been sent to a given
+# client for no more than 30 minutes.
+LAZY_LOADED_MEMBERS_CACHE_MAX_AGE = 30 * 60 * 1000
+
+# Remember the last 100 members we sent to a client for the purposes of
+# avoiding redundantly sending the same lazy-loaded members to the client
+LAZY_LOADED_MEMBERS_CACHE_MAX_SIZE = 100
+
 
 SyncConfig = collections.namedtuple("SyncConfig", [
     "user",
@@ -182,6 +192,12 @@ def __init__(self, hs):
         self.response_cache = ResponseCache(hs, "sync")
         self.state = hs.get_state_handler()
 
+        # ExpiringCache((User, Device)) -> LruCache(state_key => event_id)
+        self.lazy_loaded_members_cache = ExpiringCache(
+            "lazy_loaded_members_cache", self.clock,
+            max_len=0, expiry_ms=LAZY_LOADED_MEMBERS_CACHE_MAX_AGE,
+        )
+
     def wait_for_sync_for_user(self, sync_config, since_token=None, timeout=0,
                                full_state=False):
         """Get the sync for a client if we have new data for it now. Otherwise
@@ -505,9 +521,13 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
         with Measure(self.clock, "compute_state_delta"):
 
             types = None
-            lazy_load_members = sync_config.filter_collection.lazy_load_members()
             filtered_types = None
 
+            lazy_load_members = sync_config.filter_collection.lazy_load_members()
+            include_redundant_members = (
+                sync_config.filter_collection.include_redundant_members()
+            )
+
             if lazy_load_members:
                 # We only request state for the members needed to display the
                 # timeline:
@@ -523,6 +543,11 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                 # only apply the filtering to room members
                 filtered_types = [EventTypes.Member]
 
+            timeline_state = {
+                (event.type, event.state_key): event.event_id
+                for event in batch.events if event.is_state()
+            }
+
             if full_state:
                 if batch:
                     current_state_ids = yield self.store.get_state_ids_for_event(
@@ -543,11 +568,6 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
 
                     state_ids = current_state_ids
 
-                timeline_state = {
-                    (event.type, event.state_key): event.event_id
-                    for event in batch.events if event.is_state()
-                }
-
                 state_ids = _calculate_state(
                     timeline_contains=timeline_state,
                     timeline_start=state_ids,
@@ -571,21 +591,6 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                     filtered_types=filtered_types,
                 )
 
-                timeline_state = {
-                    (event.type, event.state_key): event.event_id
-                    for event in batch.events if event.is_state()
-                }
-
-                # TODO: optionally filter out redundant membership events at this
-                # point, to stop repeatedly sending members in every /sync as if
-                # the client isn't tracking them.
-                # When implemented, this should filter using event_ids (not mxids).
-                # In practice, limited syncs are
-                # relatively rare so it's not a total disaster to send redundant
-                # members down at this point. Redundant members are ones which
-                # repeatedly get sent down /sync because we don't know if the client
-                # is caching them or not.
-
                 state_ids = _calculate_state(
                     timeline_contains=timeline_state,
                     timeline_start=state_at_timeline_start,
@@ -596,16 +601,48 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
             else:
                 state_ids = {}
                 if lazy_load_members:
-                    # TODO: filter out redundant members based on their mxids (not their
-                    # event_ids) at this point. We know we can do it based on mxid as this
-                    # is an non-gappy incremental sync.
-
                     if types:
                         state_ids = yield self.store.get_state_ids_for_event(
                             batch.events[0].event_id, types=types,
                             filtered_types=filtered_types,
                         )
 
+            if lazy_load_members and not include_redundant_members:
+                cache_key = (sync_config.user.to_string(), sync_config.device_id)
+                cache = self.lazy_loaded_members_cache.get(cache_key)
+                if cache is None:
+                    logger.debug("creating LruCache for %r", cache_key)
+                    cache = LruCache(LAZY_LOADED_MEMBERS_CACHE_MAX_SIZE)
+                    self.lazy_loaded_members_cache[cache_key] = cache
+                else:
+                    logger.debug("found LruCache for %r", cache_key)
+
+                # if it's a new sync sequence, then assume the client has had
+                # amnesia and doesn't want any recent lazy-loaded members
+                # de-duplicated.
+                if since_token is None:
+                    logger.debug("clearing LruCache for %r", cache_key)
+                    cache.clear()
+                else:
+                    # only send members which aren't in our LruCache (either
+                    # because they're new to this client or have been pushed out
+                    # of the cache)
+                    logger.debug("filtering state from %r...", state_ids)
+                    state_ids = {
+                        t: event_id
+                        for t, event_id in state_ids.iteritems()
+                        if cache.get(t[1]) != event_id
+                    }
+                    logger.debug("...to %r", state_ids)
+
+                # add any member IDs we are about to send into our LruCache
+                for t, event_id in itertools.chain(
+                    state_ids.items(),
+                    timeline_state.items(),
+                ):
+                    if t[0] == EventTypes.Member:
+                        cache.set(t[1], event_id)
+
         state = {}
         if state_ids:
             state = yield self.store.get_events(list(state_ids.values()))

From 8e3f75b39abb846687ac905d8e31d7f41372e5d7 Mon Sep 17 00:00:00 2001
From: Jeroen <vo.jeroen@gmail.com>
Date: Fri, 27 Jul 2018 12:17:31 +0200
Subject: [PATCH 173/830] fix accidental removal of hs

---
 synapse/crypto/keyring.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py
index 762e4b8014e7..30e2742102d6 100644
--- a/synapse/crypto/keyring.py
+++ b/synapse/crypto/keyring.py
@@ -512,7 +512,7 @@ def get_server_verify_key_v2_direct(self, server_name, key_ids):
                 continue
 
             (response, tls_certificate) = yield fetch_server_key(
-                server_name, self.tls_client_options_factory,
+                server_name, self.hs.tls_client_options_factory,
                 path=(b"/_matrix/key/v2/server/%s" % (
                     urllib.quote(requested_key_id),
                 )).encode("ascii"),

From f102c05856623fdea45309d58c759d1d784d19fd Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Sun, 10 Jun 2018 22:38:50 +0100
Subject: [PATCH 174/830] Rewrite cache list decorator

Because it was complicated and annoyed me. I suspect this will be more
efficient too.
---
 changelog.d/3384.misc                 |   1 +
 synapse/util/caches/descriptors.py    | 131 +++++++++++++-------------
 tests/util/caches/test_descriptors.py |  61 ++++++++++++
 3 files changed, 126 insertions(+), 67 deletions(-)
 create mode 100644 changelog.d/3384.misc

diff --git a/changelog.d/3384.misc b/changelog.d/3384.misc
new file mode 100644
index 000000000000..5d56c876d90b
--- /dev/null
+++ b/changelog.d/3384.misc
@@ -0,0 +1 @@
+Rewrite cache list decorator
diff --git a/synapse/util/caches/descriptors.py b/synapse/util/caches/descriptors.py
index f8a07df6b8f1..1a8c99d387ca 100644
--- a/synapse/util/caches/descriptors.py
+++ b/synapse/util/caches/descriptors.py
@@ -473,105 +473,101 @@ def __get__(self, obj, objtype=None):
 
         @functools.wraps(self.orig)
         def wrapped(*args, **kwargs):
-            # If we're passed a cache_context then we'll want to call its invalidate()
-            # whenever we are invalidated
+            # If we're passed a cache_context then we'll want to call its
+            # invalidate() whenever we are invalidated
             invalidate_callback = kwargs.pop("on_invalidate", None)
 
             arg_dict = inspect.getcallargs(self.orig, obj, *args, **kwargs)
             keyargs = [arg_dict[arg_nm] for arg_nm in self.arg_names]
             list_args = arg_dict[self.list_name]
 
-            # cached is a dict arg -> deferred, where deferred results in a
-            # 2-tuple (`arg`, `result`)
             results = {}
-            cached_defers = {}
-            missing = []
+
+            def update_results_dict(res, arg):
+                results[arg] = res
+
+            # list of deferreds to wait for
+            cached_defers = []
+
+            missing = set()
 
             # If the cache takes a single arg then that is used as the key,
             # otherwise a tuple is used.
             if num_args == 1:
-                def cache_get(arg):
-                    return cache.get(arg, callback=invalidate_callback)
+                def arg_to_cache_key(arg):
+                    return arg
             else:
-                key = list(keyargs)
+                keylist = list(keyargs)
 
-                def cache_get(arg):
-                    key[self.list_pos] = arg
-                    return cache.get(tuple(key), callback=invalidate_callback)
+                def arg_to_cache_key(arg):
+                    keylist[self.list_pos] = arg
+                    return tuple(keylist)
 
             for arg in list_args:
                 try:
-                    res = cache_get(arg)
-
+                    res = cache.get(arg_to_cache_key(arg),
+                                    callback=invalidate_callback)
                     if not isinstance(res, ObservableDeferred):
                         results[arg] = res
                     elif not res.has_succeeded():
                         res = res.observe()
-                        res.addCallback(lambda r, arg: (arg, r), arg)
-                        cached_defers[arg] = res
+                        res.addCallback(update_results_dict, arg)
+                        cached_defers.append(res)
                     else:
                         results[arg] = res.get_result()
                 except KeyError:
-                    missing.append(arg)
+                    missing.add(arg)
 
             if missing:
+                # we need an observable deferred for each entry in the list,
+                # which we put in the cache. Each deferred resolves with the
+                # relevant result for that key.
+                deferreds_map = {}
+                for arg in missing:
+                    deferred = defer.Deferred()
+                    deferreds_map[arg] = deferred
+                    key = arg_to_cache_key(arg)
+                    observable = ObservableDeferred(deferred)
+                    cache.set(key, observable)
+
+                def complete_all(res):
+                    # the wrapped function has completed. It returns a
+                    # a dict. We can now resolve the observable deferreds in
+                    # the cache and update our own result map.
+                    for e in missing:
+                        val = res.get(e, None)
+                        deferreds_map[e].callback(val)
+                        results[e] = val
+
+                def errback(f):
+                    # the wrapped function has failed. Invalidate any cache
+                    # entries we're supposed to be populating, and fail
+                    # their deferreds.
+                    for e in missing:
+                        key = arg_to_cache_key(e)
+                        cache.invalidate(key)
+                        deferreds_map[e].errback(f)
+
+                    # return the failure, to propagate to our caller.
+                    return f
+
                 args_to_call = dict(arg_dict)
-                args_to_call[self.list_name] = missing
+                args_to_call[self.list_name] = list(missing)
 
-                ret_d = defer.maybeDeferred(
+                cached_defers.append(defer.maybeDeferred(
                     logcontext.preserve_fn(self.function_to_call),
                     **args_to_call
-                )
-
-                ret_d = ObservableDeferred(ret_d)
-
-                # We need to create deferreds for each arg in the list so that
-                # we can insert the new deferred into the cache.
-                for arg in missing:
-                    observer = ret_d.observe()
-                    observer.addCallback(lambda r, arg: r.get(arg, None), arg)
-
-                    observer = ObservableDeferred(observer)
-
-                    if num_args == 1:
-                        cache.set(
-                            arg, observer,
-                            callback=invalidate_callback
-                        )
-
-                        def invalidate(f, key):
-                            cache.invalidate(key)
-                            return f
-                        observer.addErrback(invalidate, arg)
-                    else:
-                        key = list(keyargs)
-                        key[self.list_pos] = arg
-                        cache.set(
-                            tuple(key), observer,
-                            callback=invalidate_callback
-                        )
-
-                        def invalidate(f, key):
-                            cache.invalidate(key)
-                            return f
-                        observer.addErrback(invalidate, tuple(key))
-
-                    res = observer.observe()
-                    res.addCallback(lambda r, arg: (arg, r), arg)
-
-                    cached_defers[arg] = res
+                ).addCallbacks(complete_all, errback))
 
             if cached_defers:
-                def update_results_dict(res):
-                    results.update(res)
-                    return results
-
-                return logcontext.make_deferred_yieldable(defer.gatherResults(
-                    list(cached_defers.values()),
+                d = defer.gatherResults(
+                    cached_defers,
                     consumeErrors=True,
-                ).addCallback(update_results_dict).addErrback(
+                ).addCallbacks(
+                    lambda _: results,
                     unwrapFirstError
-                ))
+                )
+                return logcontext.make_deferred_yieldable(d)
             else:
                 return results
 
@@ -625,7 +621,8 @@ def cachedList(cached_method_name, list_name, num_args=None, inlineCallbacks=Fal
     cache.
 
     Args:
-        cache (Cache): The underlying cache to use.
+        cached_method_name (str): The name of the single-item lookup method.
+            This is only used to find the cache to use.
         list_name (str): The name of the argument that is the list to use to
             do batch lookups in the cache.
         num_args (int): Number of arguments to use as the key in the cache
diff --git a/tests/util/caches/test_descriptors.py b/tests/util/caches/test_descriptors.py
index 8176a7dabd23..1ac967b63e6c 100644
--- a/tests/util/caches/test_descriptors.py
+++ b/tests/util/caches/test_descriptors.py
@@ -273,3 +273,64 @@ def fn(self, arg1, arg2=2, arg3=3):
         r = yield obj.fn(2, 3)
         self.assertEqual(r, 'chips')
         obj.mock.assert_not_called()
+
+
+@unittest.DEBUG
+class CachedListDescriptorTestCase(unittest.TestCase):
+    @defer.inlineCallbacks
+    def test_cache(self):
+        class Cls(object):
+            def __init__(self):
+                self.mock = mock.Mock()
+
+            @descriptors.cached()
+            def fn(self, arg1, arg2):
+                pass
+
+            @descriptors.cachedList("fn", "args1", inlineCallbacks=True)
+            def list_fn(self, args1, arg2):
+                assert (
+                    logcontext.LoggingContext.current_context().request == "c1"
+                )
+                # we want this to behave like an asynchronous function
+                yield run_on_reactor()
+                assert (
+                    logcontext.LoggingContext.current_context().request == "c1"
+                )
+                defer.returnValue(self.mock(args1, arg2))
+
+        with logcontext.LoggingContext() as c1:
+            c1.request = "c1"
+            obj = Cls()
+            obj.mock.return_value = {10: 'fish', 20: 'chips'}
+            d1 = obj.list_fn([10, 20], 2)
+            self.assertEqual(
+                logcontext.LoggingContext.current_context(),
+                logcontext.LoggingContext.sentinel,
+            )
+            r = yield d1
+            self.assertEqual(
+                logcontext.LoggingContext.current_context(),
+                c1
+            )
+            obj.mock.assert_called_once_with([10, 20], 2)
+            self.assertEqual(r, {10: 'fish', 20: 'chips'})
+            obj.mock.reset_mock()
+
+            # a call with different params should call the mock again
+            obj.mock.return_value = {30: 'peas'}
+            r = yield obj.list_fn([20, 30], 2)
+            obj.mock.assert_called_once_with([30], 2)
+            self.assertEqual(r, {20: 'chips', 30: 'peas'})
+            obj.mock.reset_mock()
+
+            # all the values should now be cached
+            r = yield obj.fn(10, 2)
+            self.assertEqual(r, 'fish')
+            r = yield obj.fn(20, 2)
+            self.assertEqual(r, 'chips')
+            r = yield obj.fn(30, 2)
+            self.assertEqual(r, 'peas')
+            r = yield obj.list_fn([10, 20, 30], 2)
+            obj.mock.assert_not_called()
+            self.assertEqual(r, {10: 'fish', 20: 'chips', 30: 'peas'})

From ad7cd95a7842bc2eafb5bd69467978e60b4ac0d8 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 27 Jul 2018 15:02:57 +0100
Subject: [PATCH 175/830] Newsfile

---
 changelog.d/3621.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3621.misc

diff --git a/changelog.d/3621.misc b/changelog.d/3621.misc
new file mode 100644
index 000000000000..83e5fc8aa1ac
--- /dev/null
+++ b/changelog.d/3621.misc
@@ -0,0 +1 @@
+Refactor FederationHandler to move DB writes into separate functions

From e9b2d047f68b74e231609ce40978f4452ac9e22f Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@arasphere.net>
Date: Fri, 27 Jul 2018 15:12:50 +0100
Subject: [PATCH 176/830] make /context lazyload & filter aware (#3567)

make /context lazyload & filter aware.
---
 changelog.d/3567.feature       |  1 +
 synapse/handlers/room.py       | 24 +++++++++++++++++++++---
 synapse/handlers/search.py     |  2 +-
 synapse/rest/client/v1/room.py |  9 +++++++++
 synapse/storage/stream.py      | 14 +++++++++++---
 5 files changed, 43 insertions(+), 7 deletions(-)
 create mode 100644 changelog.d/3567.feature

diff --git a/changelog.d/3567.feature b/changelog.d/3567.feature
new file mode 100644
index 000000000000..c74c1f57a90a
--- /dev/null
+++ b/changelog.d/3567.feature
@@ -0,0 +1 @@
+make the /context API filter & lazy-load aware as per MSC1227
diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index 003b848c0012..7b7804d9b2cc 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -15,6 +15,7 @@
 # limitations under the License.
 
 """Contains functions for performing events on rooms."""
+import itertools
 import logging
 import math
 import string
@@ -401,7 +402,7 @@ def __init__(self, hs):
         self.store = hs.get_datastore()
 
     @defer.inlineCallbacks
-    def get_event_context(self, user, room_id, event_id, limit):
+    def get_event_context(self, user, room_id, event_id, limit, event_filter):
         """Retrieves events, pagination tokens and state around a given event
         in a room.
 
@@ -411,6 +412,8 @@ def get_event_context(self, user, room_id, event_id, limit):
             event_id (str)
             limit (int): The maximum number of events to return in total
                 (excluding state).
+            event_filter (Filter|None): the filter to apply to the events returned
+                (excluding the target event_id)
 
         Returns:
             dict, or None if the event isn't found
@@ -443,7 +446,7 @@ def filter_evts(events):
             )
 
         results = yield self.store.get_events_around(
-            room_id, event_id, before_limit, after_limit
+            room_id, event_id, before_limit, after_limit, event_filter
         )
 
         results["events_before"] = yield filter_evts(results["events_before"])
@@ -455,8 +458,23 @@ def filter_evts(events):
         else:
             last_event_id = event_id
 
+        types = None
+        filtered_types = None
+        if event_filter and event_filter.lazy_load_members():
+            members = set(ev.sender for ev in itertools.chain(
+                results["events_before"],
+                (results["event"],),
+                results["events_after"],
+            ))
+            filtered_types = [EventTypes.Member]
+            types = [(EventTypes.Member, member) for member in members]
+
+        # XXX: why do we return the state as of the last event rather than the
+        # first? Shouldn't we be consistent with /sync?
+        # https://github.com/matrix-org/matrix-doc/issues/687
+
         state = yield self.store.get_state_for_events(
-            [last_event_id], None
+            [last_event_id], types, filtered_types=filtered_types,
         )
         results["state"] = list(state[last_event_id].values())
 
diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py
index 69ae9731d585..c464adbd0b9c 100644
--- a/synapse/handlers/search.py
+++ b/synapse/handlers/search.py
@@ -287,7 +287,7 @@ def search(self, user, content, batch=None):
             contexts = {}
             for event in allowed_events:
                 res = yield self.store.get_events_around(
-                    event.room_id, event.event_id, before_limit, after_limit
+                    event.room_id, event.event_id, before_limit, after_limit,
                 )
 
                 logger.info(
diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py
index b7bd878c905e..13c331550b35 100644
--- a/synapse/rest/client/v1/room.py
+++ b/synapse/rest/client/v1/room.py
@@ -531,11 +531,20 @@ def on_GET(self, request, room_id, event_id):
 
         limit = parse_integer(request, "limit", default=10)
 
+        # picking the API shape for symmetry with /messages
+        filter_bytes = parse_string(request, "filter")
+        if filter_bytes:
+            filter_json = urlparse.unquote(filter_bytes).decode("UTF-8")
+            event_filter = Filter(json.loads(filter_json))
+        else:
+            event_filter = None
+
         results = yield self.room_context_handler.get_event_context(
             requester.user,
             room_id,
             event_id,
             limit,
+            event_filter,
         )
 
         if not results:
diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py
index 66856342f0f9..25d0097b58e4 100644
--- a/synapse/storage/stream.py
+++ b/synapse/storage/stream.py
@@ -527,7 +527,9 @@ def _set_before_and_after(events, rows, topo_order=True):
             )
 
     @defer.inlineCallbacks
-    def get_events_around(self, room_id, event_id, before_limit, after_limit):
+    def get_events_around(
+        self, room_id, event_id, before_limit, after_limit, event_filter=None,
+    ):
         """Retrieve events and pagination tokens around a given event in a
         room.
 
@@ -536,6 +538,7 @@ def get_events_around(self, room_id, event_id, before_limit, after_limit):
             event_id (str)
             before_limit (int)
             after_limit (int)
+            event_filter (Filter|None)
 
         Returns:
             dict
@@ -543,7 +546,7 @@ def get_events_around(self, room_id, event_id, before_limit, after_limit):
 
         results = yield self.runInteraction(
             "get_events_around", self._get_events_around_txn,
-            room_id, event_id, before_limit, after_limit
+            room_id, event_id, before_limit, after_limit, event_filter,
         )
 
         events_before = yield self._get_events(
@@ -563,7 +566,9 @@ def get_events_around(self, room_id, event_id, before_limit, after_limit):
             "end": results["after"]["token"],
         })
 
-    def _get_events_around_txn(self, txn, room_id, event_id, before_limit, after_limit):
+    def _get_events_around_txn(
+        self, txn, room_id, event_id, before_limit, after_limit, event_filter,
+    ):
         """Retrieves event_ids and pagination tokens around a given event in a
         room.
 
@@ -572,6 +577,7 @@ def _get_events_around_txn(self, txn, room_id, event_id, before_limit, after_lim
             event_id (str)
             before_limit (int)
             after_limit (int)
+            event_filter (Filter|None)
 
         Returns:
             dict
@@ -601,11 +607,13 @@ def _get_events_around_txn(self, txn, room_id, event_id, before_limit, after_lim
 
         rows, start_token = self._paginate_room_events_txn(
             txn, room_id, before_token, direction='b', limit=before_limit,
+            event_filter=event_filter,
         )
         events_before = [r.event_id for r in rows]
 
         rows, end_token = self._paginate_room_events_txn(
             txn, room_id, after_token, direction='f', limit=after_limit,
+            event_filter=event_filter,
         )
         events_after = [r.event_id for r in rows]
 

From a8cbce0ced662f92ce7576dd44e5fefe98ffae62 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 27 Jul 2018 16:17:17 +0100
Subject: [PATCH 177/830] fix invalidation

---
 synapse/util/caches/descriptors.py    |  2 +-
 tests/util/caches/test_descriptors.py | 42 ++++++++++++++++++++++++++-
 2 files changed, 42 insertions(+), 2 deletions(-)

diff --git a/synapse/util/caches/descriptors.py b/synapse/util/caches/descriptors.py
index 1a8c99d387ca..861c24809c31 100644
--- a/synapse/util/caches/descriptors.py
+++ b/synapse/util/caches/descriptors.py
@@ -528,7 +528,7 @@ def arg_to_cache_key(arg):
                     deferreds_map[arg] = deferred
                     key = arg_to_cache_key(arg)
                     observable = ObservableDeferred(deferred)
-                    cache.set(key, observable)
+                    cache.set(key, observable, callback=invalidate_callback)
 
                 def complete_all(res):
                     # the wrapped function has completed. It returns a
diff --git a/tests/util/caches/test_descriptors.py b/tests/util/caches/test_descriptors.py
index 1ac967b63e6c..ca8a7c907fbb 100644
--- a/tests/util/caches/test_descriptors.py
+++ b/tests/util/caches/test_descriptors.py
@@ -275,7 +275,6 @@ def fn(self, arg1, arg2=2, arg3=3):
         obj.mock.assert_not_called()
 
 
-@unittest.DEBUG
 class CachedListDescriptorTestCase(unittest.TestCase):
     @defer.inlineCallbacks
     def test_cache(self):
@@ -334,3 +333,44 @@ def list_fn(self, args1, arg2):
             r = yield obj.list_fn([10, 20, 30], 2)
             obj.mock.assert_not_called()
             self.assertEqual(r, {10: 'fish', 20: 'chips', 30: 'peas'})
+
+    @defer.inlineCallbacks
+    def test_invalidate(self):
+        """Make sure that invalidation callbacks are called."""
+        class Cls(object):
+            def __init__(self):
+                self.mock = mock.Mock()
+
+            @descriptors.cached()
+            def fn(self, arg1, arg2):
+                pass
+
+            @descriptors.cachedList("fn", "args1", inlineCallbacks=True)
+            def list_fn(self, args1, arg2):
+                # we want this to behave like an asynchronous function
+                yield run_on_reactor()
+                defer.returnValue(self.mock(args1, arg2))
+
+        obj = Cls()
+        invalidate0 = mock.Mock()
+        invalidate1 = mock.Mock()
+
+        # cache miss
+        obj.mock.return_value = {10: 'fish', 20: 'chips'}
+        r1 = yield obj.list_fn([10, 20], 2, on_invalidate=invalidate0)
+        obj.mock.assert_called_once_with([10, 20], 2)
+        self.assertEqual(r1, {10: 'fish', 20: 'chips'})
+        obj.mock.reset_mock()
+
+        # cache hit
+        r2 = yield obj.list_fn([10, 20], 2, on_invalidate=invalidate1)
+        obj.mock.assert_not_called()
+        self.assertEqual(r2, {10: 'fish', 20: 'chips'})
+
+        invalidate0.assert_not_called()
+        invalidate1.assert_not_called()
+
+        # now if we invalidate the keys, both invalidations should get called
+        obj.fn.invalidate((10, 2))
+        invalidate0.assert_called_once()
+        invalidate1.assert_called_once()

From 2903e65affdcbd77ec794b0f998afb9e24f50215 Mon Sep 17 00:00:00 2001
From: Jeroen <vo.jeroen@gmail.com>
Date: Sun, 29 Jul 2018 19:47:08 +0200
Subject: [PATCH 178/830] fix isort

---
 synapse/crypto/context_factory.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py
index 57694e18b0a5..569377e65363 100644
--- a/synapse/crypto/context_factory.py
+++ b/synapse/crypto/context_factory.py
@@ -13,12 +13,13 @@
 # limitations under the License.
 import logging
 
+from zope.interface import implementer
+
 from OpenSSL import SSL, crypto
 from twisted.internet._idna import _idnaBytes
-from twisted.internet.ssl import ContextFactory, CertificateOptions
 from twisted.internet._sslverify import _defaultCurveName, _tolerateErrors
 from twisted.internet.interfaces import IOpenSSLClientConnectionCreator
-from zope.interface import implementer
+from twisted.internet.ssl import CertificateOptions, ContextFactory
 
 logger = logging.getLogger(__name__)
 

From 251e6c1210087069a6133140519de80a4ddf218a Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Mon, 30 Jul 2018 15:55:57 +0100
Subject: [PATCH 179/830] limit register and sign in on number of monthly users

---
 synapse/api/errors.py           |  1 +
 synapse/config/server.py        |  5 ++++
 synapse/handlers/auth.py        | 13 +++++++++
 synapse/handlers/register.py    | 18 ++++++++++--
 synapse/storage/__init__.py     | 34 +++++++++++++++++++++++
 tests/handlers/test_auth.py     | 49 ++++++++++++++++++++++++++++++++-
 tests/handlers/test_register.py | 49 +++++++++++++++++++++++++++++++++
 7 files changed, 166 insertions(+), 3 deletions(-)

diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index 6074df292f55..14f55402807f 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -55,6 +55,7 @@ class Codes(object):
     SERVER_NOT_TRUSTED = "M_SERVER_NOT_TRUSTED"
     CONSENT_NOT_GIVEN = "M_CONSENT_NOT_GIVEN"
     CANNOT_LEAVE_SERVER_NOTICE_ROOM = "M_CANNOT_LEAVE_SERVER_NOTICE_ROOM"
+    MAU_LIMIT_EXCEEDED = "M_MAU_LIMIT_EXCEEDED"
 
 
 class CodeMessageException(RuntimeError):
diff --git a/synapse/config/server.py b/synapse/config/server.py
index 18102656b041..8b335bff3f30 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -67,6 +67,11 @@ def read_config(self, config):
             "block_non_admin_invites", False,
         )
 
+        # Options to control access by tracking MAU
+        self.limit_usage_by_mau = config.get("limit_usage_by_mau", False)
+        self.max_mau_value = config.get(
+                "max_mau_value", 0,
+        )
         # FIXME: federation_domain_whitelist needs sytests
         self.federation_domain_whitelist = None
         federation_domain_whitelist = config.get(
diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py
index 402e44cdef97..f3734f11bdae 100644
--- a/synapse/handlers/auth.py
+++ b/synapse/handlers/auth.py
@@ -519,6 +519,7 @@ def get_access_token_for_user_id(self, user_id, device_id=None):
         """
         logger.info("Logging in user %s on device %s", user_id, device_id)
         access_token = yield self.issue_access_token(user_id, device_id)
+        self._check_mau_limits()
 
         # the device *should* have been registered before we got here; however,
         # it's possible we raced against a DELETE operation. The thing we
@@ -729,6 +730,7 @@ def issue_access_token(self, user_id, device_id=None):
         defer.returnValue(access_token)
 
     def validate_short_term_login_token_and_get_user_id(self, login_token):
+        self._check_mau_limits()
         auth_api = self.hs.get_auth()
         try:
             macaroon = pymacaroons.Macaroon.deserialize(login_token)
@@ -892,6 +894,17 @@ def _do_validate_hash():
         else:
             return defer.succeed(False)
 
+    def _check_mau_limits(self):
+        """
+        Ensure that if mau blocking is enabled that invalid users cannot
+        log in.
+        """
+        if self.hs.config.limit_usage_by_mau is True:
+            current_mau = self.store.count_monthly_users()
+            if current_mau >= self.hs.config.max_mau_value:
+                raise AuthError(
+                        403, "MAU Limit Exceeded", errcode=Codes.MAU_LIMIT_EXCEEDED
+                )
 
 @attr.s
 class MacaroonGenerator(object):
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index 7caff0cbc8d0..f46b8355c0d0 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -45,7 +45,7 @@ def __init__(self, hs):
             hs (synapse.server.HomeServer):
         """
         super(RegistrationHandler, self).__init__(hs)
-
+        self.hs = hs
         self.auth = hs.get_auth()
         self._auth_handler = hs.get_auth_handler()
         self.profile_handler = hs.get_profile_handler()
@@ -144,6 +144,7 @@ def register(
         Raises:
             RegistrationError if there was a problem registering.
         """
+        self._check_mau_limits()
         password_hash = None
         if password:
             password_hash = yield self.auth_handler().hash(password)
@@ -288,6 +289,7 @@ def register_saml2(self, localpart):
                 400,
                 "User ID can only contain characters a-z, 0-9, or '=_-./'",
             )
+        self._check_mau_limits()
         user = UserID(localpart, self.hs.hostname)
         user_id = user.to_string()
 
@@ -437,7 +439,7 @@ def get_or_create_user(self, requester, localpart, displayname,
         """
         if localpart is None:
             raise SynapseError(400, "Request must include user id")
-
+        self._check_mau_limits()
         need_register = True
 
         try:
@@ -531,3 +533,15 @@ def _join_user_to_room(self, requester, room_identifier):
             remote_room_hosts=remote_room_hosts,
             action="join",
         )
+
+    def _check_mau_limits(self):
+        """
+        Do not accept registrations if monthly active user limits exceeded
+         and limiting is enabled
+        """
+        if self.hs.config.limit_usage_by_mau is True:
+            current_mau = self.store.count_monthly_users()
+            if current_mau >= self.hs.config.max_mau_value:
+                raise RegistrationError(
+                    403, "MAU Limit Exceeded", Codes.MAU_LIMIT_EXCEEDED
+                )
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index ba88a5497975..6a75bf0e529a 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -19,6 +19,7 @@
 import time
 
 from dateutil import tz
+from prometheus_client import Gauge
 
 from synapse.api.constants import PresenceState
 from synapse.storage.devices import DeviceStore
@@ -60,6 +61,13 @@
 
 logger = logging.getLogger(__name__)
 
+# Gauges to expose monthly active user control metrics
+current_mau_gauge = Gauge("synapse_admin_current_mau", "Current MAU")
+max_mau_value_gauge = Gauge("synapse_admin_max_mau_value", "MAU Limit")
+limit_usage_by_mau_gauge = Gauge(
+    "synapse_admin_limit_usage_by_mau", "MAU Limiting enabled"
+)
+
 
 class DataStore(RoomMemberStore, RoomStore,
                 RegistrationStore, StreamStore, ProfileStore,
@@ -266,6 +274,32 @@ def _count_users(txn):
 
         return self.runInteraction("count_users", _count_users)
 
+    def count_monthly_users(self):
+        """
+        Counts the number of users who used this homeserver in the last 30 days
+        This method should be refactored with count_daily_users - the only
+        reason not to is waiting on definition of mau
+        returns:
+            int: count of current monthly active users
+        """
+        def _count_monthly_users(txn):
+            thirty_days_ago = int(self._clock.time_msec()) - (1000 * 60 * 60 * 24 * 30)
+            sql = """
+                SELECT COUNT(*) FROM user_ips
+                WHERE last_seen > ?
+            """
+            txn.execute(sql, (thirty_days_ago,))
+            count, = txn.fetchone()
+
+            self._current_mau = count
+            current_mau_gauge.set(self._current_mau)
+            max_mau_value_gauge.set(self.hs.config.max_mau_value)
+            limit_usage_by_mau_gauge.set(self.hs.config.limit_usage_by_mau)
+            logger.info("calling mau stats")
+            return count
+        return self.runInteraction("count_monthly_users", _count_monthly_users)
+
+
     def count_r30_users(self):
         """
         Counts the number of 30 day retained users, defined as:-
diff --git a/tests/handlers/test_auth.py b/tests/handlers/test_auth.py
index 2e5e8e4decb9..57f78a6becf6 100644
--- a/tests/handlers/test_auth.py
+++ b/tests/handlers/test_auth.py
@@ -12,15 +12,17 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+from mock import Mock
 import pymacaroons
 
 from twisted.internet import defer
 
 import synapse
+from synapse.api.errors import AuthError
 import synapse.api.errors
 from synapse.handlers.auth import AuthHandler
 
+
 from tests import unittest
 from tests.utils import setup_test_homeserver
 
@@ -37,6 +39,10 @@ def setUp(self):
         self.hs.handlers = AuthHandlers(self.hs)
         self.auth_handler = self.hs.handlers.auth_handler
         self.macaroon_generator = self.hs.get_macaroon_generator()
+        # MAU tests
+        self.hs.config.max_mau_value = 50
+        self.small_number_of_users = 1
+        self.large_number_of_users = 100
 
     def test_token_is_a_macaroon(self):
         token = self.macaroon_generator.generate_access_token("some_user")
@@ -113,3 +119,44 @@ def test_short_term_login_token_cannot_replace_user_id(self):
             self.auth_handler.validate_short_term_login_token_and_get_user_id(
                 macaroon.serialize()
             )
+
+    @defer.inlineCallbacks
+    def test_mau_limits_disabled(self):
+        self.hs.config.limit_usage_by_mau = False
+        # Ensure does not throw exception
+        yield self.auth_handler.get_access_token_for_user_id('user_a')
+
+        self.auth_handler.validate_short_term_login_token_and_get_user_id(
+            self._get_macaroon().serialize()
+        )
+
+    @defer.inlineCallbacks
+    def test_mau_limits_exceeded(self):
+        self.hs.config.limit_usage_by_mau = True
+        self.hs.get_datastore().count_monthly_users = Mock(
+            return_value=self.large_number_of_users
+        )
+        with self.assertRaises(AuthError):
+            yield self.auth_handler.get_access_token_for_user_id('user_a')
+        with self.assertRaises(AuthError):
+            self.auth_handler.validate_short_term_login_token_and_get_user_id(
+                self._get_macaroon().serialize()
+            )
+
+    @defer.inlineCallbacks
+    def test_mau_limits_not_exceeded(self):
+        self.hs.config.limit_usage_by_mau = True
+        self.hs.get_datastore().count_monthly_users = Mock(
+            return_value=self.small_number_of_users
+        )
+        # Ensure does not raise exception
+        yield self.auth_handler.get_access_token_for_user_id('user_a')
+        self.auth_handler.validate_short_term_login_token_and_get_user_id(
+            self._get_macaroon().serialize()
+        )
+
+    def _get_macaroon(self):
+        token = self.macaroon_generator.generate_short_term_login_token(
+            "user_a", 5000
+        )
+        return pymacaroons.Macaroon.deserialize(token)
diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py
index 025fa1be81c6..a5a8e7c95402 100644
--- a/tests/handlers/test_register.py
+++ b/tests/handlers/test_register.py
@@ -17,6 +17,7 @@
 
 from twisted.internet import defer
 
+from synapse.api.errors import RegistrationError
 from synapse.handlers.register import RegistrationHandler
 from synapse.types import UserID, create_requester
 
@@ -77,3 +78,51 @@ def test_if_user_exists(self):
             requester, local_part, display_name)
         self.assertEquals(result_user_id, user_id)
         self.assertEquals(result_token, 'secret')
+
+    @defer.inlineCallbacks
+    def test_cannot_register_when_mau_limits_exceeded(self):
+        local_part = "someone"
+        display_name = "someone"
+        requester = create_requester("@as:test")
+        store = self.hs.get_datastore()
+        self.hs.config.limit_usage_by_mau = False
+        self.hs.config.max_mau_value = 50
+        lots_of_users = 100
+        small_number_users = 1
+
+        store.count_monthly_users = Mock(return_value=lots_of_users)
+
+        # Ensure does not throw exception
+        yield self.handler.get_or_create_user(requester, 'a', display_name)
+
+        self.hs.config.limit_usage_by_mau = True
+
+        with self.assertRaises(RegistrationError):
+            yield self.handler.get_or_create_user(requester, 'b', display_name)
+
+        store.count_monthly_users = Mock(return_value=small_number_users)
+
+        self._macaroon_mock_generator("another_secret")
+
+        # Ensure does not throw exception
+        yield self.handler.get_or_create_user("@neil:matrix.org", 'c', "Neil")
+
+        self._macaroon_mock_generator("another another secret")
+        store.count_monthly_users = Mock(return_value=lots_of_users)
+        with self.assertRaises(RegistrationError):
+            yield self.handler.register(localpart=local_part)
+
+        self._macaroon_mock_generator("another another secret")
+        store.count_monthly_users = Mock(return_value=lots_of_users)
+        with self.assertRaises(RegistrationError):
+            yield self.handler.register_saml2(local_part)
+
+    def _macaroon_mock_generator(self, secret):
+        """
+        Reset macaroon generator in the case where the test creates multiple users
+        """
+        macaroon_generator = Mock(
+            generate_access_token=Mock(return_value=secret))
+        self.hs.get_macaroon_generator = Mock(return_value=macaroon_generator)
+        self.hs.handlers = RegistrationHandlers(self.hs)
+        self.handler = self.hs.get_handlers().registration_handler

From 9b13817e067d370f3ddb860baef884bf460449b5 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Mon, 30 Jul 2018 22:07:07 +0100
Subject: [PATCH 180/830] factor out metrics from __init__ to app/homeserver

---
 synapse/app/homeserver.py   | 20 +++++++++++++++++++-
 synapse/storage/__init__.py | 37 +++++++++++++------------------------
 2 files changed, 32 insertions(+), 25 deletions(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 57b815d777d7..a1512ccea400 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -18,6 +18,7 @@
 import os
 import sys
 
+from prometheus_client import Gauge
 from six import iteritems
 
 from twisted.application import service
@@ -299,7 +300,12 @@ def run_startup_checks(self, db_conn, database_engine):
         except IncorrectDatabaseSetup as e:
             quit_with_error(e.message)
 
-
+# Gauges to expose monthly active user control metrics
+current_mau_gauge = Gauge("synapse_admin_current_mau", "Current MAU")
+max_mau_value_gauge = Gauge("synapse_admin_max_mau_value", "MAU Limit")
+limit_usage_by_mau_gauge = Gauge(
+    "synapse_admin_limit_usage_by_mau", "MAU Limiting enabled"
+)
 def setup(config_options):
     """
     Args:
@@ -512,6 +518,18 @@ def generate_user_daily_visit_stats():
     # table will decrease
     clock.looping_call(generate_user_daily_visit_stats, 5 * 60 * 1000)
 
+    def generate_monthly_active_users():
+        count = 0
+        if hs.config.limit_usage_by_mau:
+            count = hs.get_datastore().count_monthly_users()
+        logger.info("NJ count is %d" % (count,))
+        current_mau_gauge.set(float(count))
+        max_mau_value_gauge.set(float(hs.config.max_mau_value))
+        limit_usage_by_mau_gauge.set(float(hs.config.limit_usage_by_mau))
+
+    generate_monthly_active_users()
+    clock.looping_call(generate_monthly_active_users, 5 * 60 * 1000)
+
     if hs.config.report_stats:
         logger.info("Scheduling stats reporting for 3 hour intervals")
         clock.looping_call(start_phone_stats_home, 3 * 60 * 60 * 1000)
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 6a75bf0e529a..7b8215bf0856 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -19,7 +19,6 @@
 import time
 
 from dateutil import tz
-from prometheus_client import Gauge
 
 from synapse.api.constants import PresenceState
 from synapse.storage.devices import DeviceStore
@@ -61,14 +60,6 @@
 
 logger = logging.getLogger(__name__)
 
-# Gauges to expose monthly active user control metrics
-current_mau_gauge = Gauge("synapse_admin_current_mau", "Current MAU")
-max_mau_value_gauge = Gauge("synapse_admin_max_mau_value", "MAU Limit")
-limit_usage_by_mau_gauge = Gauge(
-    "synapse_admin_limit_usage_by_mau", "MAU Limiting enabled"
-)
-
-
 class DataStore(RoomMemberStore, RoomStore,
                 RegistrationStore, StreamStore, ProfileStore,
                 PresenceStore, TransactionStore,
@@ -102,6 +93,7 @@ def __init__(self, db_conn, hs):
         self._clock = hs.get_clock()
         self.database_engine = hs.database_engine
 
+        self.db_conn = db_conn
         self._stream_id_gen = StreamIdGenerator(
             db_conn, "events", "stream_ordering",
             extra_tables=[("local_invites", "stream_id")]
@@ -282,22 +274,19 @@ def count_monthly_users(self):
         returns:
             int: count of current monthly active users
         """
-        def _count_monthly_users(txn):
-            thirty_days_ago = int(self._clock.time_msec()) - (1000 * 60 * 60 * 24 * 30)
-            sql = """
-                SELECT COUNT(*) FROM user_ips
-                WHERE last_seen > ?
-            """
-            txn.execute(sql, (thirty_days_ago,))
-            count, = txn.fetchone()
 
-            self._current_mau = count
-            current_mau_gauge.set(self._current_mau)
-            max_mau_value_gauge.set(self.hs.config.max_mau_value)
-            limit_usage_by_mau_gauge.set(self.hs.config.limit_usage_by_mau)
-            logger.info("calling mau stats")
-            return count
-        return self.runInteraction("count_monthly_users", _count_monthly_users)
+        thirty_days_ago = int(self._clock.time_msec()) - (1000 * 60 * 60 * 24 * 30)
+        sql = """
+            SELECT COALESCE(count(*), 0) FROM (
+                SELECT user_id FROM user_ips
+                WHERE last_seen > ?
+                GROUP BY user_id
+            ) u
+        """
+        txn = self.db_conn.cursor()
+        txn.execute(sql, (thirty_days_ago,))
+        count, = txn.fetchone()
+        return count
 
 
     def count_r30_users(self):

From cefac79c105515fb2e5b05fbbc0fdb9b5b977f30 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Mon, 30 Jul 2018 22:08:09 +0100
Subject: [PATCH 181/830] monthly_active_tests

---
 tests/storage/test__init__.py | 48 +++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)
 create mode 100644 tests/storage/test__init__.py

diff --git a/tests/storage/test__init__.py b/tests/storage/test__init__.py
new file mode 100644
index 000000000000..b763d395cc5d
--- /dev/null
+++ b/tests/storage/test__init__.py
@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import sys
+
+from twisted.internet import defer
+
+import tests.unittest
+import tests.utils
+
+
+class InitTestCase(tests.unittest.TestCase):
+    def __init__(self, *args, **kwargs):
+        super(InitTestCase, self).__init__(*args, **kwargs)
+        self.store = None  # type: synapse.storage.DataStore
+
+    @defer.inlineCallbacks
+    def setUp(self):
+        hs = yield tests.utils.setup_test_homeserver()
+        hs.config.max_mau_value = 50
+        hs.config.limit_usage_by_mau = True
+        self.store = hs.get_datastore()
+
+    @defer.inlineCallbacks
+    def test_count_monthly_users(self):
+        count = yield self.store.count_monthly_users()
+        self.assertEqual(0, count)
+        yield self.store.insert_client_ip(
+            "@user:server1", "access_token", "ip", "user_agent", "device_id"
+        )
+
+        yield self.store.insert_client_ip(
+            "@user:server2", "access_token", "ip", "user_agent", "device_id"
+        )
+        count = self.store.count_monthly_users()
+
+        self.assertEqual(2, count)

From fef7e58ac63d58f4a13f0914f51fc5956981af6a Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Mon, 30 Jul 2018 22:29:44 +0100
Subject: [PATCH 182/830] actually close conn

---
 synapse/storage/__init__.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 7b8215bf0856..044e988e9279 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -283,10 +283,14 @@ def count_monthly_users(self):
                 GROUP BY user_id
             ) u
         """
-        txn = self.db_conn.cursor()
-        txn.execute(sql, (thirty_days_ago,))
-        count, = txn.fetchone()
-        return count
+        try:
+            txn = self.db_conn.cursor()
+            txn.execute(sql, (thirty_days_ago,))
+            count, = txn.fetchone()
+            return count
+        finally:
+            txn.close()
+
 
 
     def count_r30_users(self):

From 21276ff84601745093c2416bbf3db1f0ae56155d Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Mon, 30 Jul 2018 22:42:12 +0100
Subject: [PATCH 183/830] remove errant logging

---
 synapse/app/homeserver.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index a1512ccea400..96c45b7209eb 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -522,7 +522,6 @@ def generate_monthly_active_users():
         count = 0
         if hs.config.limit_usage_by_mau:
             count = hs.get_datastore().count_monthly_users()
-        logger.info("NJ count is %d" % (count,))
         current_mau_gauge.set(float(count))
         max_mau_value_gauge.set(float(hs.config.max_mau_value))
         limit_usage_by_mau_gauge.set(float(hs.config.limit_usage_by_mau))

From 254e8267e20ccde19f89eccdfb67b7aaecb32eec Mon Sep 17 00:00:00 2001
From: Krombel <krombel@krombel.de>
Date: Mon, 30 Jul 2018 19:38:38 +0200
Subject: [PATCH 184/830] Only import secrets when available

secrets got introduced in python 3.6 so this class is not available
in 3.5 and before.

This now checks for the current running version and only tries using
secrets if the version is 3.6 or above

Signed-Off-By: Matthias Kesler <krombel@krombel.de>
---
 changelog.d/3626.bugfix | 1 +
 synapse/secrets.py      | 7 +++----
 2 files changed, 4 insertions(+), 4 deletions(-)
 create mode 100644 changelog.d/3626.bugfix

diff --git a/changelog.d/3626.bugfix b/changelog.d/3626.bugfix
new file mode 100644
index 000000000000..9a4b87898615
--- /dev/null
+++ b/changelog.d/3626.bugfix
@@ -0,0 +1 @@
+Only import secrets when available (fix for py < 3.6)
diff --git a/synapse/secrets.py b/synapse/secrets.py
index f397daaa5edb..f05e9ea53595 100644
--- a/synapse/secrets.py
+++ b/synapse/secrets.py
@@ -20,17 +20,16 @@
 used in Python 3.6, and the API emulated in Python 2.7.
 """
 
-import six
+import sys
 
-if six.PY3:
+# secrets is available since python 3.6
+if sys.version_info[0:2] >= (3, 6):
     import secrets
 
     def Secrets():
         return secrets
 
-
 else:
-
     import os
     import binascii
 

From e908b8683248328dd92479fc81be350336b9c8f4 Mon Sep 17 00:00:00 2001
From: Travis Ralston <travpc@gmail.com>
Date: Mon, 30 Jul 2018 16:24:02 -0600
Subject: [PATCH 185/830] Remove pdu_failures from transactions

The field is never read from, and all the opportunities given to populate it are not utilized. It should be very safe to remove this.
---
 changelog.d/3628.misc                   |  1 +
 synapse/federation/federation_server.py |  4 --
 synapse/federation/send_queue.py        | 63 +------------------------
 synapse/federation/transaction_queue.py | 32 ++-----------
 synapse/federation/transport/server.py  |  3 +-
 synapse/federation/units.py             |  1 -
 tests/handlers/test_typing.py           |  1 -
 7 files changed, 8 insertions(+), 97 deletions(-)
 create mode 100644 changelog.d/3628.misc

diff --git a/changelog.d/3628.misc b/changelog.d/3628.misc
new file mode 100644
index 000000000000..1aebefbe186e
--- /dev/null
+++ b/changelog.d/3628.misc
@@ -0,0 +1 @@
+Remove unused field "pdu_failures" from transactions.
diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index e501251b6e96..657935d1acb7 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -207,10 +207,6 @@ def process_pdus_for_room(room_id):
                     edu.content
                 )
 
-        pdu_failures = getattr(transaction, "pdu_failures", [])
-        for fail in pdu_failures:
-            logger.info("Got failure %r", fail)
-
         response = {
             "pdus": pdu_results,
         }
diff --git a/synapse/federation/send_queue.py b/synapse/federation/send_queue.py
index 5157c3860d89..0bb468385d0c 100644
--- a/synapse/federation/send_queue.py
+++ b/synapse/federation/send_queue.py
@@ -62,8 +62,6 @@ def __init__(self, hs):
 
         self.edus = SortedDict()  # stream position -> Edu
 
-        self.failures = SortedDict()  # stream position -> (destination, Failure)
-
         self.device_messages = SortedDict()  # stream position -> destination
 
         self.pos = 1
@@ -79,7 +77,7 @@ def register(name, queue):
 
         for queue_name in [
             "presence_map", "presence_changed", "keyed_edu", "keyed_edu_changed",
-            "edus", "failures", "device_messages", "pos_time",
+            "edus", "device_messages", "pos_time",
         ]:
             register(queue_name, getattr(self, queue_name))
 
@@ -149,12 +147,6 @@ def _clear_queue_before_pos(self, position_to_delete):
             for key in keys[:i]:
                 del self.edus[key]
 
-            # Delete things out of failure map
-            keys = self.failures.keys()
-            i = self.failures.bisect_left(position_to_delete)
-            for key in keys[:i]:
-                del self.failures[key]
-
             # Delete things out of device map
             keys = self.device_messages.keys()
             i = self.device_messages.bisect_left(position_to_delete)
@@ -204,13 +196,6 @@ def send_presence(self, states):
 
         self.notifier.on_new_replication_data()
 
-    def send_failure(self, failure, destination):
-        """As per TransactionQueue"""
-        pos = self._next_pos()
-
-        self.failures[pos] = (destination, str(failure))
-        self.notifier.on_new_replication_data()
-
     def send_device_messages(self, destination):
         """As per TransactionQueue"""
         pos = self._next_pos()
@@ -285,17 +270,6 @@ def get_replication_rows(self, from_token, to_token, limit, federation_ack=None)
         for (pos, edu) in edus:
             rows.append((pos, EduRow(edu)))
 
-        # Fetch changed failures
-        i = self.failures.bisect_right(from_token)
-        j = self.failures.bisect_right(to_token) + 1
-        failures = self.failures.items()[i:j]
-
-        for (pos, (destination, failure)) in failures:
-            rows.append((pos, FailureRow(
-                destination=destination,
-                failure=failure,
-            )))
-
         # Fetch changed device messages
         i = self.device_messages.bisect_right(from_token)
         j = self.device_messages.bisect_right(to_token) + 1
@@ -417,34 +391,6 @@ def add_to_buffer(self, buff):
         buff.edus.setdefault(self.edu.destination, []).append(self.edu)
 
 
-class FailureRow(BaseFederationRow, namedtuple("FailureRow", (
-    "destination",  # str
-    "failure",
-))):
-    """Streams failures to a remote server. Failures are issued when there was
-    something wrong with a transaction the remote sent us, e.g. it included
-    an event that was invalid.
-    """
-
-    TypeId = "f"
-
-    @staticmethod
-    def from_data(data):
-        return FailureRow(
-            destination=data["destination"],
-            failure=data["failure"],
-        )
-
-    def to_data(self):
-        return {
-            "destination": self.destination,
-            "failure": self.failure,
-        }
-
-    def add_to_buffer(self, buff):
-        buff.failures.setdefault(self.destination, []).append(self.failure)
-
-
 class DeviceRow(BaseFederationRow, namedtuple("DeviceRow", (
     "destination",  # str
 ))):
@@ -471,7 +417,6 @@ def add_to_buffer(self, buff):
         PresenceRow,
         KeyedEduRow,
         EduRow,
-        FailureRow,
         DeviceRow,
     )
 }
@@ -481,7 +426,6 @@ def add_to_buffer(self, buff):
     "presence",  # list(UserPresenceState)
     "keyed_edus",  # dict of destination -> { key -> Edu }
     "edus",  # dict of destination -> [Edu]
-    "failures",  # dict of destination -> [failures]
     "device_destinations",  # set of destinations
 ))
 
@@ -503,7 +447,6 @@ def process_rows_for_federation(transaction_queue, rows):
         presence=[],
         keyed_edus={},
         edus={},
-        failures={},
         device_destinations=set(),
     )
 
@@ -532,9 +475,5 @@ def process_rows_for_federation(transaction_queue, rows):
                 edu.destination, edu.edu_type, edu.content, key=None,
             )
 
-    for destination, failure_list in iteritems(buff.failures):
-        for failure in failure_list:
-            transaction_queue.send_failure(destination, failure)
-
     for destination in buff.device_destinations:
         transaction_queue.send_device_messages(destination)
diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py
index 6996d6b695b9..78f9d40a3ad1 100644
--- a/synapse/federation/transaction_queue.py
+++ b/synapse/federation/transaction_queue.py
@@ -116,9 +116,6 @@ def __init__(self, hs):
             ),
         )
 
-        # destination -> list of tuple(failure, deferred)
-        self.pending_failures_by_dest = {}
-
         # destination -> stream_id of last successfully sent to-device message.
         # NB: may be a long or an int.
         self.last_device_stream_id_by_dest = {}
@@ -382,19 +379,6 @@ def send_edu(self, destination, edu_type, content, key=None):
 
         self._attempt_new_transaction(destination)
 
-    def send_failure(self, failure, destination):
-        if destination == self.server_name or destination == "localhost":
-            return
-
-        if not self.can_send_to(destination):
-            return
-
-        self.pending_failures_by_dest.setdefault(
-            destination, []
-        ).append(failure)
-
-        self._attempt_new_transaction(destination)
-
     def send_device_messages(self, destination):
         if destination == self.server_name or destination == "localhost":
             return
@@ -469,7 +453,6 @@ def _transaction_transmission_loop(self, destination):
                 pending_pdus = self.pending_pdus_by_dest.pop(destination, [])
                 pending_edus = self.pending_edus_by_dest.pop(destination, [])
                 pending_presence = self.pending_presence_by_dest.pop(destination, {})
-                pending_failures = self.pending_failures_by_dest.pop(destination, [])
 
                 pending_edus.extend(
                     self.pending_edus_keyed_by_dest.pop(destination, {}).values()
@@ -497,7 +480,7 @@ def _transaction_transmission_loop(self, destination):
                     logger.debug("TX [%s] len(pending_pdus_by_dest[dest]) = %d",
                                  destination, len(pending_pdus))
 
-                if not pending_pdus and not pending_edus and not pending_failures:
+                if not pending_pdus and not pending_edus:
                     logger.debug("TX [%s] Nothing to send", destination)
                     self.last_device_stream_id_by_dest[destination] = (
                         device_stream_id
@@ -507,7 +490,7 @@ def _transaction_transmission_loop(self, destination):
                 # END CRITICAL SECTION
 
                 success = yield self._send_new_transaction(
-                    destination, pending_pdus, pending_edus, pending_failures,
+                    destination, pending_pdus, pending_edus,
                 )
                 if success:
                     sent_transactions_counter.inc()
@@ -584,14 +567,12 @@ def _get_new_device_messages(self, destination):
 
     @measure_func("_send_new_transaction")
     @defer.inlineCallbacks
-    def _send_new_transaction(self, destination, pending_pdus, pending_edus,
-                              pending_failures):
+    def _send_new_transaction(self, destination, pending_pdus, pending_edus):
 
         # Sort based on the order field
         pending_pdus.sort(key=lambda t: t[1])
         pdus = [x[0] for x in pending_pdus]
         edus = pending_edus
-        failures = [x.get_dict() for x in pending_failures]
 
         success = True
 
@@ -601,11 +582,10 @@ def _send_new_transaction(self, destination, pending_pdus, pending_edus,
 
         logger.debug(
             "TX [%s] {%s} Attempting new transaction"
-            " (pdus: %d, edus: %d, failures: %d)",
+            " (pdus: %d, edus: %d)",
             destination, txn_id,
             len(pdus),
             len(edus),
-            len(failures)
         )
 
         logger.debug("TX [%s] Persisting transaction...", destination)
@@ -617,7 +597,6 @@ def _send_new_transaction(self, destination, pending_pdus, pending_edus,
             destination=destination,
             pdus=pdus,
             edus=edus,
-            pdu_failures=failures,
         )
 
         self._next_txn_id += 1
@@ -627,12 +606,11 @@ def _send_new_transaction(self, destination, pending_pdus, pending_edus,
         logger.debug("TX [%s] Persisted transaction", destination)
         logger.info(
             "TX [%s] {%s} Sending transaction [%s],"
-            " (PDUs: %d, EDUs: %d, failures: %d)",
+            " (PDUs: %d, EDUs: %d)",
             destination, txn_id,
             transaction.transaction_id,
             len(pdus),
             len(edus),
-            len(failures),
         )
 
         # Actually send the transaction
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 8574898f0c25..3b5ea9515a5b 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -283,11 +283,10 @@ def on_PUT(self, origin, content, query, transaction_id):
             )
 
             logger.info(
-                "Received txn %s from %s. (PDUs: %d, EDUs: %d, failures: %d)",
+                "Received txn %s from %s. (PDUs: %d, EDUs: %d)",
                 transaction_id, origin,
                 len(transaction_data.get("pdus", [])),
                 len(transaction_data.get("edus", [])),
-                len(transaction_data.get("failures", [])),
             )
 
             # We should ideally be getting this from the security layer.
diff --git a/synapse/federation/units.py b/synapse/federation/units.py
index bb1b3b13f7de..c5ab14314efc 100644
--- a/synapse/federation/units.py
+++ b/synapse/federation/units.py
@@ -73,7 +73,6 @@ class Transaction(JsonEncodedObject):
         "previous_ids",
         "pdus",
         "edus",
-        "pdu_failures",
     ]
 
     internal_keys = [
diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py
index b08856f763f4..2c263af1a3a0 100644
--- a/tests/handlers/test_typing.py
+++ b/tests/handlers/test_typing.py
@@ -44,7 +44,6 @@ def _expect_edu(destination, edu_type, content, origin="test"):
                 "content": content,
             }
         ],
-        "pdu_failures": [],
     }
 
 

From 6aab397adabeae5c45b75be28691ea1286b90869 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 31 Jul 2018 09:45:35 +0100
Subject: [PATCH 186/830] synapse grafana dashboard

---
 contrib/grafana/synapse.json | 4961 ++++++++++++++++++++++++++++++++++
 1 file changed, 4961 insertions(+)
 create mode 100644 contrib/grafana/synapse.json

diff --git a/contrib/grafana/synapse.json b/contrib/grafana/synapse.json
new file mode 100644
index 000000000000..94a1de58f4e3
--- /dev/null
+++ b/contrib/grafana/synapse.json
@@ -0,0 +1,4961 @@
+{
+  "__inputs": [
+    {
+      "name": "DS_PROMETHEUS",
+      "label": "Prometheus",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "prometheus",
+      "pluginName": "Prometheus"
+    }
+  ],
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "5.2.0"
+    },
+    {
+      "type": "panel",
+      "id": "graph",
+      "name": "Graph",
+      "version": "5.0.0"
+    },
+    {
+      "type": "panel",
+      "id": "heatmap",
+      "name": "Heatmap",
+      "version": "5.0.0"
+    },
+    {
+      "type": "datasource",
+      "id": "prometheus",
+      "name": "Prometheus",
+      "version": "5.0.0"
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "$datasource",
+        "enable": false,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "limit": 100,
+        "name": "Annotations & Alerts",
+        "showIn": 0,
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "id": null,
+  "iteration": 1533026624326,
+  "links": [
+    {
+      "asDropdown": true,
+      "icon": "external link",
+      "keepTime": true,
+      "tags": [
+        "matrix"
+      ],
+      "title": "Dashboards",
+      "type": "dashboards"
+    }
+  ],
+  "panels": [
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 73,
+      "panels": [],
+      "title": "Overview",
+      "type": "row"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "fill": 1,
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 1
+      },
+      "id": 75,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "process_cpu_seconds:rate2m{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "{{job}}-{{index}} ",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "CPU usage",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "decimals": null,
+          "format": "percentunit",
+          "label": null,
+          "logBase": 1,
+          "max": "1",
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "cards": {
+        "cardPadding": 0,
+        "cardRound": null
+      },
+      "color": {
+        "cardColor": "#b4ff00",
+        "colorScale": "sqrt",
+        "colorScheme": "interpolateSpectral",
+        "exponent": 0.5,
+        "mode": "spectrum"
+      },
+      "dataFormat": "tsbuckets",
+      "datasource": "${DS_PROMETHEUS}",
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 12,
+        "y": 1
+      },
+      "heatmap": {},
+      "highlightCards": true,
+      "id": 85,
+      "legend": {
+        "show": false
+      },
+      "links": [],
+      "targets": [
+        {
+          "expr": "sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\"}[$bucket_size])) by (le)",
+          "format": "heatmap",
+          "intervalFactor": 1,
+          "legendFormat": "{{le}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Event Send Time",
+      "tooltip": {
+        "show": true,
+        "showHistogram": false
+      },
+      "type": "heatmap",
+      "xAxis": {
+        "show": true
+      },
+      "xBucketNumber": null,
+      "xBucketSize": null,
+      "yAxis": {
+        "decimals": null,
+        "format": "s",
+        "logBase": 2,
+        "max": null,
+        "min": null,
+        "show": true,
+        "splitFactor": null
+      },
+      "yBucketBound": "auto",
+      "yBucketNumber": null,
+      "yBucketSize": null
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "$datasource",
+      "editable": true,
+      "error": false,
+      "fill": 1,
+      "grid": {},
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 10
+      },
+      "id": 33,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": false,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 2,
+      "links": [],
+      "nullPointMode": "null",
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(synapse_storage_events_persisted_events{instance=\"$instance\"}[$bucket_size])) without (job,index)",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "",
+          "refId": "A",
+          "step": 20,
+          "target": ""
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Events Persisted",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "cumulative"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "hertz",
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 17
+      },
+      "id": 54,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "editable": true,
+          "error": false,
+          "fill": 0,
+          "grid": {},
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 18
+          },
+          "id": 34,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": true,
+          "targets": [
+            {
+              "expr": "process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}} {{index}}",
+              "refId": "A",
+              "step": 20,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Memory",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "bytes",
+              "logBase": 1,
+              "max": null,
+              "min": "0",
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "fill": 1,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 18
+          },
+          "id": 37,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [
+            {
+              "alias": "/max$/",
+              "color": "#890F02",
+              "fill": 0,
+              "legend": false
+            }
+          ],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "process_open_fds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}}",
+              "refId": "A",
+              "step": 20
+            },
+            {
+              "expr": "process_max_fds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+              "format": "time_series",
+              "hide": true,
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}} max",
+              "refId": "B",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Open FDs",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "none",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "fill": 1,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 25
+          },
+          "id": 48,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_storage_schedule_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_schedule_time_count[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Avg time waiting for db conn",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "s",
+              "label": "",
+              "logBase": 1,
+              "max": null,
+              "min": "0",
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": false
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "fill": 1,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 25
+          },
+          "id": 49,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [
+            {
+              "alias": "/^up/",
+              "legend": false,
+              "yaxis": 2
+            }
+          ],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "scrape_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Prometheus scrape time",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "s",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": "0",
+              "show": true
+            },
+            {
+              "decimals": 0,
+              "format": "none",
+              "label": "",
+              "logBase": 1,
+              "max": "0",
+              "min": "-1",
+              "show": false
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "fill": 1,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 32
+          },
+          "id": 50,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(python_twisted_reactor_tick_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(python_twisted_reactor_tick_time_count[$bucket_size])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Avg reactor tick time",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "s",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": false
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {},
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 32
+          },
+          "id": 5,
+          "legend": {
+            "alignAsTable": false,
+            "avg": false,
+            "current": false,
+            "hideEmpty": false,
+            "hideZero": false,
+            "max": false,
+            "min": false,
+            "rightSide": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [
+            {
+              "alias": "/user/"
+            },
+            {
+              "alias": "/system/"
+            }
+          ],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(process_cpu_system_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} system ",
+              "metric": "",
+              "refId": "B",
+              "step": 20
+            },
+            {
+              "expr": "rate(process_cpu_user_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} user",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [
+            {
+              "colorMode": "custom",
+              "line": true,
+              "lineColor": "rgba(216, 200, 27, 0.27)",
+              "op": "gt",
+              "value": 0.5
+            },
+            {
+              "colorMode": "custom",
+              "line": true,
+              "lineColor": "rgba(234, 112, 112, 0.22)",
+              "op": "gt",
+              "value": 0.8
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "CPU",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "percentunit",
+              "label": "",
+              "logBase": 1,
+              "max": "1.2",
+              "min": 0,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "${DS_PROMETHEUS}",
+          "fill": 0,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 39
+          },
+          "id": 53,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "min_over_time(up{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Up",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        }
+      ],
+      "repeat": null,
+      "title": "Process info",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 18
+      },
+      "id": 56,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "decimals": 1,
+          "fill": 1,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 49
+          },
+          "id": 40,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_storage_events_persisted_by_source_type{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{type}}",
+              "refId": "D"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Events/s Local vs Remote",
+          "tooltip": {
+            "shared": true,
+            "sort": 2,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "label": "",
+              "logBase": 1,
+              "max": null,
+              "min": "0",
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "decimals": 1,
+          "fill": 1,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 49
+          },
+          "id": 46,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_storage_events_persisted_by_event_type{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+              "format": "time_series",
+              "instant": false,
+              "intervalFactor": 2,
+              "legendFormat": "{{type}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Events/s by Type",
+          "tooltip": {
+            "shared": false,
+            "sort": 2,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": "0",
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {
+            "irc-freenode (local)": "#EAB839"
+          },
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "decimals": 1,
+          "fill": 1,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 56
+          },
+          "id": 44,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "hideEmpty": true,
+            "hideZero": true,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_storage_events_persisted_by_origin{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{origin_entity}} ({{origin_type}})",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Events/s by Origin",
+          "tooltip": {
+            "shared": false,
+            "sort": 2,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": "0",
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "decimals": 1,
+          "fill": 1,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 56
+          },
+          "id": 45,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "hideEmpty": true,
+            "hideZero": true,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "sum(rate(synapse_storage_events_persisted_events_sep{job=~\"$job\",index=~\"$index\", type=\"m.room.member\",instance=\"$instance\"}[$bucket_size])) by (origin_type, origin_entity)",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{origin_entity}} ({{origin_type}})",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Memberships/s by Origin",
+          "tooltip": {
+            "shared": true,
+            "sort": 2,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": "0",
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        }
+      ],
+      "repeat": null,
+      "title": "Event persist rates",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 19
+      },
+      "id": 57,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "decimals": null,
+          "editable": true,
+          "error": false,
+          "fill": 2,
+          "grid": {},
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 48
+          },
+          "id": 4,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "hideEmpty": false,
+            "hideZero": true,
+            "max": false,
+            "min": false,
+            "rightSide": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_http_server_requests_received{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [
+            {
+              "colorMode": "custom",
+              "fill": true,
+              "fillColor": "rgba(216, 200, 27, 0.27)",
+              "op": "gt",
+              "value": 100
+            },
+            {
+              "colorMode": "custom",
+              "fill": true,
+              "fillColor": "rgba(234, 112, 112, 0.22)",
+              "op": "gt",
+              "value": 250
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Request Count by arrival time",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {},
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 48
+          },
+          "id": 32,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_http_server_requests_received{instance=\"$instance\",job=~\"$job\",index=~\"$index\",method!=\"OPTIONS\"}[$bucket_size]) and topk(10,synapse_http_server_requests_received{instance=\"$instance\",job=~\"$job\",method!=\"OPTIONS\"})",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{method}} {{servlet}} {{job}}-{{index}}",
+              "refId": "A",
+              "step": 20,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Top 10 Request Counts",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "decimals": null,
+          "editable": true,
+          "error": false,
+          "fill": 2,
+          "grid": {},
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 56
+          },
+          "id": 23,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "hideEmpty": false,
+            "hideZero": true,
+            "max": false,
+            "min": false,
+            "rightSide": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_http_server_response_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_response_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [
+            {
+              "colorMode": "custom",
+              "fill": true,
+              "fillColor": "rgba(216, 200, 27, 0.27)",
+              "op": "gt",
+              "value": 100,
+              "yaxis": "left"
+            },
+            {
+              "colorMode": "custom",
+              "fill": true,
+              "fillColor": "rgba(234, 112, 112, 0.22)",
+              "op": "gt",
+              "value": 250,
+              "yaxis": "left"
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Total CPU Usage by Endpoint",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "percentunit",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "decimals": null,
+          "editable": true,
+          "error": false,
+          "fill": 2,
+          "grid": {},
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 56
+          },
+          "id": 52,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "hideEmpty": false,
+            "hideZero": true,
+            "max": false,
+            "min": false,
+            "rightSide": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "(rate(synapse_http_server_response_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_response_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) / rate(synapse_http_server_response_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [
+            {
+              "colorMode": "custom",
+              "fill": true,
+              "fillColor": "rgba(216, 200, 27, 0.27)",
+              "op": "gt",
+              "value": 100
+            },
+            {
+              "colorMode": "custom",
+              "fill": true,
+              "fillColor": "rgba(234, 112, 112, 0.22)",
+              "op": "gt",
+              "value": 250
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Average CPU Usage by Endpoint",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "s",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {},
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 64
+          },
+          "id": 7,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "hideEmpty": true,
+            "hideZero": true,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_http_server_response_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "DB Usage by endpoint",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "percentunit",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "decimals": null,
+          "editable": true,
+          "error": false,
+          "fill": 2,
+          "grid": {},
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 64
+          },
+          "id": 47,
+          "legend": {
+            "alignAsTable": true,
+            "avg": true,
+            "current": false,
+            "hideEmpty": false,
+            "hideZero": true,
+            "max": true,
+            "min": false,
+            "rightSide": false,
+            "show": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_http_server_response_time_seconds_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\",tag!=\"incremental_sync\"}[$bucket_size])/rate(synapse_http_server_response_time_seconds_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\",tag!=\"incremental_sync\"}[$bucket_size])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Non-sync avg response time",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "s",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": false
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "${DS_PROMETHEUS}",
+          "fill": 1,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 0,
+            "y": 72
+          },
+          "id": 103,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "topk(10,synapse_http_server_in_flight_requests_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"})",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Requests in flight",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        }
+      ],
+      "repeat": null,
+      "title": "Requests",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 20
+      },
+      "id": 97,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "${DS_PROMETHEUS}",
+          "fill": 1,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 0,
+            "y": 23
+          },
+          "id": 99,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_background_process_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_background_process_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} {{name}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "CPU usage by background jobs",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "percentunit",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "${DS_PROMETHEUS}",
+          "fill": 1,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 12,
+            "y": 23
+          },
+          "id": 101,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_background_process_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} {{name}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "DB usage by background jobs",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "percentunit",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        }
+      ],
+      "title": "Background jobs",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 21
+      },
+      "id": 81,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "${DS_PROMETHEUS}",
+          "fill": 1,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 0,
+            "y": 25
+          },
+          "id": 79,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_federation_client_sent_transactions{instance=\"$instance\", job=~\"$job\", index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "txn rate",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Outgoing federation transaction rate",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "${DS_PROMETHEUS}",
+          "fill": 1,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 12,
+            "y": 25
+          },
+          "id": 83,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_federation_server_received_pdus{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "pdus",
+              "refId": "A"
+            },
+            {
+              "expr": "rate(synapse_federation_server_received_edus{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "edus",
+              "refId": "B"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Incoming PDU/EDU rate",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        }
+      ],
+      "title": "Federation",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 22
+      },
+      "id": 60,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "fill": 1,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 23
+          },
+          "id": 51,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_push_httppusher_http_pushes_processed{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "processed {{job}}",
+              "refId": "A",
+              "step": 20
+            },
+            {
+              "expr": "rate(synapse_push_httppusher_http_pushes_failed{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "failed {{job}}",
+              "refId": "B",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "HTTP Push rate",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        }
+      ],
+      "repeat": null,
+      "title": "Pushes",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 23
+      },
+      "id": 58,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "editable": true,
+          "error": false,
+          "fill": 0,
+          "grid": {},
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 25
+          },
+          "id": 10,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "hideEmpty": true,
+            "hideZero": true,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "topk(10, rate(synapse_storage_transaction_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}} {{desc}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Top DB transactions by txn rate",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "logBase": 1,
+              "max": null,
+              "min": 0,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {},
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 25
+          },
+          "id": 11,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "hideEmpty": true,
+            "hideZero": true,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": true,
+          "targets": [
+            {
+              "expr": "topk(5, rate(synapse_storage_transaction_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+              "format": "time_series",
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} {{desc}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Top DB transactions by total txn time",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "percentunit",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        }
+      ],
+      "repeat": null,
+      "title": "Database",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 24
+      },
+      "id": 59,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {},
+          "gridPos": {
+            "h": 13,
+            "w": 12,
+            "x": 0,
+            "y": 17
+          },
+          "id": 12,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_util_metrics_block_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\",block_name!=\"wrapped_request_handler\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds[$bucket_size])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}} {{block_name}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Total CPU Usage by Block",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "percentunit",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {},
+          "gridPos": {
+            "h": 13,
+            "w": 12,
+            "x": 12,
+            "y": 17
+          },
+          "id": 26,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "(rate(synapse_util_metrics_block_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds[$bucket_size])) / rate(synapse_util_metrics_block_count[$bucket_size])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}} {{block_name}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Average CPU Time per Block",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "ms",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {},
+          "gridPos": {
+            "h": 13,
+            "w": 12,
+            "x": 0,
+            "y": 30
+          },
+          "id": 13,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\",block_name!=\"wrapped_request_handler\"}[$bucket_size])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}} {{block_name}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Total DB Usage by Block",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "percentunit",
+              "logBase": 1,
+              "max": null,
+              "min": 0,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {},
+          "gridPos": {
+            "h": 13,
+            "w": 12,
+            "x": 12,
+            "y": 30
+          },
+          "id": 27,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}} {{block_name}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Average Database Time per Block",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "ms",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {},
+          "gridPos": {
+            "h": 13,
+            "w": 12,
+            "x": 0,
+            "y": 43
+          },
+          "id": 28,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": false,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}} {{block_name}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Average Transactions per Block",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "none",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {},
+          "gridPos": {
+            "h": 13,
+            "w": 12,
+            "x": 12,
+            "y": 43
+          },
+          "id": 25,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": false,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_util_metrics_block_time_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_count[$bucket_size])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}} {{block_name}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Average Wallclock Time per Block",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "ms",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        }
+      ],
+      "repeat": null,
+      "title": "Per-block metrics",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 25
+      },
+      "id": 61,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "decimals": 2,
+          "editable": true,
+          "error": false,
+          "fill": 0,
+          "grid": {},
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 55
+          },
+          "id": 1,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "hideEmpty": true,
+            "hideZero": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_util_caches_cache:hits{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])/rate(synapse_util_caches_cache:total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{name}} {{job}}-{{index}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Cache Hit Ratio",
+          "tooltip": {
+            "msResolution": true,
+            "shared": false,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "percentunit",
+              "label": "",
+              "logBase": 1,
+              "max": "1",
+              "min": 0,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": false
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {},
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 55
+          },
+          "id": 8,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "hideZero": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "connected",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "synapse_util_caches_cache:size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+              "format": "time_series",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{name}} {{job}}-{{index}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Cache Size",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "logBase": 1,
+              "max": null,
+              "min": 0,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {},
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 65
+          },
+          "id": 38,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "hideZero": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "connected",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_util_caches_cache:total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{name}} {{job}}-{{index}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Cache request rate",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "rps",
+              "logBase": 1,
+              "max": null,
+              "min": 0,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "fill": 1,
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 65
+          },
+          "id": 39,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "topk(10, rate(synapse_util_caches_cache:total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]) - rate(synapse_util_caches_cache:hits{job=\"$job\",instance=\"$instance\"}[$bucket_size]))",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{name}} {{job}}-{{index}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Top 10 cache misses",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "rps",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "${DS_PROMETHEUS}",
+          "fill": 1,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 0,
+            "y": 75
+          },
+          "id": 65,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_util_caches_cache:evicted_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{name}} {{job}}-{{index}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Cache eviction rate",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "hertz",
+              "label": "entries / second",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        }
+      ],
+      "repeat": null,
+      "title": "Caches",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 26
+      },
+      "id": 62,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "${DS_PROMETHEUS}",
+          "fill": 1,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 0,
+            "y": 90
+          },
+          "id": 91,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": true,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(python_gc_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[10m])",
+              "format": "time_series",
+              "instant": false,
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} gen {{gen}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Total GC time by bucket (10m smoothing)",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "percentunit",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": "0",
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "decimals": 3,
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {},
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 12,
+            "y": 90
+          },
+          "id": 21,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "null as zero",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(python_gc_time_sum{instance=\"$instance\",job=~\"$job\"}[$bucket_size])/rate(python_gc_time_count[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}} {{index}} gen {{gen}} ",
+              "refId": "A",
+              "step": 20,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Average GC Time Per Collection",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "s",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "${DS_PROMETHEUS}",
+          "fill": 1,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 0,
+            "y": 99
+          },
+          "id": 89,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "hideEmpty": true,
+            "hideZero": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "python_gc_counts{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} gen {{gen}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Currently allocated objects",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "${DS_PROMETHEUS}",
+          "fill": 1,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 12,
+            "y": 99
+          },
+          "id": 93,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(python_gc_unreachable_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(python_gc_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} gen {{gen}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Object counts per collection",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "${DS_PROMETHEUS}",
+          "fill": 1,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 0,
+            "y": 108
+          },
+          "id": 95,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(python_gc_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} gen {{gen}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "GC frequency",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "cards": {
+            "cardPadding": 0,
+            "cardRound": null
+          },
+          "color": {
+            "cardColor": "#b4ff00",
+            "colorScale": "sqrt",
+            "colorScheme": "interpolateSpectral",
+            "exponent": 0.5,
+            "max": null,
+            "min": 0,
+            "mode": "spectrum"
+          },
+          "dataFormat": "tsbuckets",
+          "datasource": "${DS_PROMETHEUS}",
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 12,
+            "y": 108
+          },
+          "heatmap": {},
+          "highlightCards": true,
+          "id": 87,
+          "legend": {
+            "show": true
+          },
+          "links": [],
+          "targets": [
+            {
+              "expr": "sum(rate(python_gc_time_bucket[$bucket_size])) by (le)",
+              "format": "heatmap",
+              "intervalFactor": 1,
+              "legendFormat": "{{le}}",
+              "refId": "A"
+            }
+          ],
+          "title": "GC durations",
+          "tooltip": {
+            "show": true,
+            "showHistogram": false
+          },
+          "type": "heatmap",
+          "xAxis": {
+            "show": true
+          },
+          "xBucketNumber": null,
+          "xBucketSize": null,
+          "yAxis": {
+            "decimals": null,
+            "format": "s",
+            "logBase": 1,
+            "max": null,
+            "min": null,
+            "show": true,
+            "splitFactor": null
+          },
+          "yBucketBound": "auto",
+          "yBucketNumber": null,
+          "yBucketSize": null
+        }
+      ],
+      "repeat": null,
+      "title": "GC",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 27
+      },
+      "id": 63,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "${DS_PROMETHEUS}",
+          "fill": 1,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 19
+          },
+          "id": 2,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_replication_tcp_resource_user_sync{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "user started/stopped syncing",
+              "refId": "A",
+              "step": 20
+            },
+            {
+              "expr": "rate(synapse_replication_tcp_resource_federation_ack{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "federation ack",
+              "refId": "B",
+              "step": 20
+            },
+            {
+              "expr": "rate(synapse_replication_tcp_resource_remove_pusher{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "remove pusher",
+              "refId": "C",
+              "step": 20
+            },
+            {
+              "expr": "rate(synapse_replication_tcp_resource_invalidate_cache{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "invalidate cache",
+              "refId": "D",
+              "step": 20
+            },
+            {
+              "expr": "rate(synapse_replication_tcp_resource_user_ip_cache{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "user ip cache",
+              "refId": "E",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Rate of events on replication master",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ]
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "${DS_PROMETHEUS}",
+          "fill": 1,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 19
+          },
+          "id": 41,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_replication_tcp_resource_stream_updates{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{stream_name}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Outgoing stream updates",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ]
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "${DS_PROMETHEUS}",
+          "fill": 1,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 26
+          },
+          "id": 42,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "sum (rate(synapse_replication_tcp_protocol_inbound_commands{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (name, conn_id)",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}} {{command}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Rate of incoming commands",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ]
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "${DS_PROMETHEUS}",
+          "fill": 1,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 26
+          },
+          "id": 43,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "sum (rate(synapse_replication_tcp_protocol_outbound_commands{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (name, conn_id)",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}} {{command}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Rate of outgoing commands",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ]
+        }
+      ],
+      "repeat": null,
+      "title": "Replication",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 28
+      },
+      "id": 69,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "${DS_PROMETHEUS}",
+          "fill": 1,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 0,
+            "y": 11
+          },
+          "id": 67,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": " synapse_event_persisted_position{instance=\"$instance\"} - ignoring(index, job, name) group_right(instance) synapse_event_processing_positions{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Event processing lag",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": "events",
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ]
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "${DS_PROMETHEUS}",
+          "fill": 1,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 12,
+            "y": 11
+          },
+          "id": 71,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "time()*1000-synapse_event_processing_last_ts{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} {{name}}",
+              "refId": "B"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Age of last processed event",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "ms",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ]
+        }
+      ],
+      "title": "Event processing loop positions",
+      "type": "row"
+    }
+  ],
+  "refresh": "1m",
+  "schemaVersion": 16,
+  "style": "dark",
+  "tags": [
+    "matrix"
+  ],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "text": "Prometheus",
+          "value": "Prometheus"
+        },
+        "hide": 0,
+        "label": null,
+        "name": "datasource",
+        "options": [],
+        "query": "prometheus",
+        "refresh": 1,
+        "regex": "",
+        "type": "datasource"
+      },
+      {
+        "allFormat": "glob",
+        "auto": true,
+        "auto_count": 100,
+        "auto_min": "30s",
+        "current": {
+          "text": "auto",
+          "value": "$__auto_interval_bucket_size"
+        },
+        "datasource": null,
+        "hide": 0,
+        "includeAll": false,
+        "label": "Bucket Size",
+        "multi": false,
+        "multiFormat": "glob",
+        "name": "bucket_size",
+        "options": [
+          {
+            "selected": true,
+            "text": "auto",
+            "value": "$__auto_interval_bucket_size"
+          },
+          {
+            "selected": false,
+            "text": "30s",
+            "value": "30s"
+          },
+          {
+            "selected": false,
+            "text": "1m",
+            "value": "1m"
+          },
+          {
+            "selected": false,
+            "text": "2m",
+            "value": "2m"
+          },
+          {
+            "selected": false,
+            "text": "5m",
+            "value": "5m"
+          }
+        ],
+        "query": "30s,1m,2m,5m",
+        "refresh": 2,
+        "type": "interval"
+      },
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "$datasource",
+        "hide": 0,
+        "includeAll": false,
+        "label": null,
+        "multi": false,
+        "name": "instance",
+        "options": [],
+        "query": "label_values(process_cpu_user_seconds_total{job=~\"synapse.*\"}, instance)",
+        "refresh": 2,
+        "regex": "",
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allFormat": "regex wildcard",
+        "allValue": "",
+        "current": {},
+        "datasource": "$datasource",
+        "hide": 0,
+        "hideLabel": false,
+        "includeAll": true,
+        "label": "Job",
+        "multi": true,
+        "multiFormat": "regex values",
+        "name": "job",
+        "options": [],
+        "query": "label_values(process_cpu_user_seconds_total{job=~\"synapse.*\"}, job)",
+        "refresh": 2,
+        "refresh_on_load": false,
+        "regex": "",
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allFormat": "regex wildcard",
+        "allValue": ".*",
+        "current": {},
+        "datasource": "$datasource",
+        "hide": 0,
+        "hideLabel": false,
+        "includeAll": true,
+        "label": "",
+        "multi": true,
+        "multiFormat": "regex values",
+        "name": "index",
+        "options": [],
+        "query": "label_values(process_cpu_user_seconds_total{job=~\"synapse.*\"}, index)",
+        "refresh": 2,
+        "refresh_on_load": false,
+        "regex": "",
+        "sort": 3,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {
+    "now": true,
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "",
+  "title": "Synapse",
+  "uid": "000000012",
+  "version": 125
+}
\ No newline at end of file

From 9c14c2b5610956f4daa4c472929ccaa2b6b6ae22 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Paul=20T=C3=B6tterman?= <ptman@users.noreply.github.com>
Date: Tue, 31 Jul 2018 12:48:37 +0300
Subject: [PATCH 187/830] Add some documentation for using the dashboard

---
 contrib/grafana/README.md | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 contrib/grafana/README.md

diff --git a/contrib/grafana/README.md b/contrib/grafana/README.md
new file mode 100644
index 000000000000..6a6cc0bed45d
--- /dev/null
+++ b/contrib/grafana/README.md
@@ -0,0 +1,6 @@
+# Using the Synapse Grafana dashboard
+
+0. Set up Prometheus and Grafana. Out of scope for this readme. Useful documentation about using Grafana with Prometheus: http://docs.grafana.org/features/datasources/prometheus/
+1. Have your Prometheus scrape your Synapse. https://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.rst
+2. Import dashboard into Grafana. Download `synapse.json`. Import it to Grafana and select the correct Prometheus datasource. http://docs.grafana.org/reference/export_import/
+3. Set up additional recording rules

From 7d05406a07994c1b59f4d9224415d336fcf41686 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Tue, 31 Jul 2018 12:03:23 +0100
Subject: [PATCH 188/830] fix user_ips counting

---
 tests/storage/test__init__.py | 42 +++++++++++++++++++++++++++--------
 1 file changed, 33 insertions(+), 9 deletions(-)

diff --git a/tests/storage/test__init__.py b/tests/storage/test__init__.py
index b763d395cc5d..c9ae34987131 100644
--- a/tests/storage/test__init__.py
+++ b/tests/storage/test__init__.py
@@ -28,21 +28,45 @@ def __init__(self, *args, **kwargs):
     @defer.inlineCallbacks
     def setUp(self):
         hs = yield tests.utils.setup_test_homeserver()
+
         hs.config.max_mau_value = 50
         hs.config.limit_usage_by_mau = True
         self.store = hs.get_datastore()
+        self.clock = hs.get_clock()
 
-    @defer.inlineCallbacks
     def test_count_monthly_users(self):
-        count = yield self.store.count_monthly_users()
+        count = self.store.count_monthly_users()
         self.assertEqual(0, count)
-        yield self.store.insert_client_ip(
-            "@user:server1", "access_token", "ip", "user_agent", "device_id"
-        )
 
-        yield self.store.insert_client_ip(
-            "@user:server2", "access_token", "ip", "user_agent", "device_id"
-        )
-        count = self.store.count_monthly_users()
+        self._insert_user_ips("@user:server1")
+        self._insert_user_ips("@user:server2")
 
+        count = self.store.count_monthly_users()
         self.assertEqual(2, count)
+
+    def _insert_user_ips(self, user):
+        """
+        Helper function to populate user_ips without using batch insertion infra
+        args:
+            user (str):  specify username i.e. @user:server.com
+        """
+        try:
+            txn = self.store.db_conn.cursor()
+            self.store.database_engine.lock_table(txn, "user_ips")
+            self.store._simple_upsert_txn(
+                txn,
+                table="user_ips",
+                keyvalues={
+                    "user_id": user,
+                    "access_token": "access_token",
+                    "ip": "ip",
+                    "user_agent": "user_agent",
+                    "device_id": "device_id",
+                },
+                values={
+                    "last_seen": self.clock.time_msec(),
+                },
+                lock=False,
+            )
+        finally:
+            txn.close()

From 0bc9b9e397b413ca104a3ddb6db181e7a0fb0917 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 31 Jul 2018 13:11:04 +0100
Subject: [PATCH 189/830] reinstate explicit include of EventsWorkerStore

---
 synapse/storage/events.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index dbbfe04880fe..a32a30649576 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -34,7 +34,9 @@
 from synapse.events import EventBase  # noqa: F401
 from synapse.events.snapshot import EventContext  # noqa: F401
 from synapse.metrics.background_process_metrics import run_as_background_process
+from synapse.storage.background_updates import BackgroundUpdateStore
 from synapse.storage.event_federation import EventFederationStore
+from synapse.storage.events_worker import EventsWorkerStore
 from synapse.types import RoomStreamToken, get_domain_from_id
 from synapse.util.async import ObservableDeferred
 from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
@@ -195,7 +197,7 @@ def f(self, *args, **kwargs):
 
 # inherits from EventFederationStore so that we can call _update_backward_extremities
 # and _handle_mult_prev_events (though arguably those could both be moved in here)
-class EventsStore(EventFederationStore):
+class EventsStore(EventFederationStore, EventsWorkerStore, BackgroundUpdateStore):
     EVENT_ORIGIN_SERVER_TS_NAME = "event_origin_server_ts"
     EVENT_FIELDS_SENDER_URL_UPDATE_NAME = "event_fields_sender_url"
 

From df2235e7fab44a5155134a336a4c27424398c1be Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Tue, 31 Jul 2018 13:16:20 +0100
Subject: [PATCH 190/830] coding style

---
 synapse/app/homeserver.py                                   | 6 +++++-
 synapse/config/server.py                                    | 2 +-
 synapse/handlers/auth.py                                    | 3 ++-
 synapse/storage/__init__.py                                 | 3 +--
 .../storage/schema/delta/50/make_event_content_nullable.py  | 2 +-
 tests/handlers/test_auth.py                                 | 4 ++--
 tests/storage/test__init__.py                               | 1 -
 7 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 96c45b7209eb..82979e7d1b0a 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -18,9 +18,10 @@
 import os
 import sys
 
-from prometheus_client import Gauge
 from six import iteritems
 
+from prometheus_client import Gauge
+
 from twisted.application import service
 from twisted.internet import defer, reactor
 from twisted.web.resource import EncodingResourceWrapper, NoResource
@@ -300,12 +301,15 @@ def run_startup_checks(self, db_conn, database_engine):
         except IncorrectDatabaseSetup as e:
             quit_with_error(e.message)
 
+
 # Gauges to expose monthly active user control metrics
 current_mau_gauge = Gauge("synapse_admin_current_mau", "Current MAU")
 max_mau_value_gauge = Gauge("synapse_admin_max_mau_value", "MAU Limit")
 limit_usage_by_mau_gauge = Gauge(
     "synapse_admin_limit_usage_by_mau", "MAU Limiting enabled"
 )
+
+
 def setup(config_options):
     """
     Args:
diff --git a/synapse/config/server.py b/synapse/config/server.py
index 8b335bff3f30..9af42a93ad09 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -70,7 +70,7 @@ def read_config(self, config):
         # Options to control access by tracking MAU
         self.limit_usage_by_mau = config.get("limit_usage_by_mau", False)
         self.max_mau_value = config.get(
-                "max_mau_value", 0,
+            "max_mau_value", 0,
         )
         # FIXME: federation_domain_whitelist needs sytests
         self.federation_domain_whitelist = None
diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py
index f3734f11bdae..28f1c1afbb55 100644
--- a/synapse/handlers/auth.py
+++ b/synapse/handlers/auth.py
@@ -903,9 +903,10 @@ def _check_mau_limits(self):
             current_mau = self.store.count_monthly_users()
             if current_mau >= self.hs.config.max_mau_value:
                 raise AuthError(
-                        403, "MAU Limit Exceeded", errcode=Codes.MAU_LIMIT_EXCEEDED
+                    403, "MAU Limit Exceeded", errcode=Codes.MAU_LIMIT_EXCEEDED
                 )
 
+
 @attr.s
 class MacaroonGenerator(object):
 
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 044e988e9279..4747118ed716 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -60,6 +60,7 @@
 
 logger = logging.getLogger(__name__)
 
+
 class DataStore(RoomMemberStore, RoomStore,
                 RegistrationStore, StreamStore, ProfileStore,
                 PresenceStore, TransactionStore,
@@ -291,8 +292,6 @@ def count_monthly_users(self):
         finally:
             txn.close()
 
-
-
     def count_r30_users(self):
         """
         Counts the number of 30 day retained users, defined as:-
diff --git a/synapse/storage/schema/delta/50/make_event_content_nullable.py b/synapse/storage/schema/delta/50/make_event_content_nullable.py
index 7d27342e3994..6dd467b6c533 100644
--- a/synapse/storage/schema/delta/50/make_event_content_nullable.py
+++ b/synapse/storage/schema/delta/50/make_event_content_nullable.py
@@ -88,5 +88,5 @@ def run_upgrade(cur, database_engine, *args, **kwargs):
         "UPDATE sqlite_master SET sql=? WHERE tbl_name='events' AND type='table'",
         (sql, ),
     )
-    cur.execute("PRAGMA schema_version=%i" % (oldver+1,))
+    cur.execute("PRAGMA schema_version=%i" % (oldver + 1,))
     cur.execute("PRAGMA writable_schema=OFF")
diff --git a/tests/handlers/test_auth.py b/tests/handlers/test_auth.py
index 57f78a6becf6..e01f14a10aa1 100644
--- a/tests/handlers/test_auth.py
+++ b/tests/handlers/test_auth.py
@@ -13,16 +13,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 from mock import Mock
+
 import pymacaroons
 
 from twisted.internet import defer
 
 import synapse
-from synapse.api.errors import AuthError
 import synapse.api.errors
+from synapse.api.errors import AuthError
 from synapse.handlers.auth import AuthHandler
 
-
 from tests import unittest
 from tests.utils import setup_test_homeserver
 
diff --git a/tests/storage/test__init__.py b/tests/storage/test__init__.py
index c9ae34987131..fe6eeeaf1076 100644
--- a/tests/storage/test__init__.py
+++ b/tests/storage/test__init__.py
@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import sys
 
 from twisted.internet import defer
 

From 5bb39b1e0c086bbc65bb93f5b2b0c4c3fb82accf Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Tue, 31 Jul 2018 13:22:14 +0100
Subject: [PATCH 191/830] mau limts

---
 changelog.d/3630.feature | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3630.feature

diff --git a/changelog.d/3630.feature b/changelog.d/3630.feature
new file mode 100644
index 000000000000..20398da9e0aa
--- /dev/null
+++ b/changelog.d/3630.feature
@@ -0,0 +1 @@
+Blocks registration and sign in if max mau value exceeded

From d81602b75afc2c39314da1f9a21be436134e5361 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 31 Jul 2018 13:52:49 +0100
Subject: [PATCH 192/830] Add helper base class for generating new replication
 endpoints

This will hopefully reduce the boiler plate required to implement new
internal HTTP requests.
---
 synapse/replication/http/_base.py | 208 ++++++++++++++++++++++++++++++
 1 file changed, 208 insertions(+)
 create mode 100644 synapse/replication/http/_base.py

diff --git a/synapse/replication/http/_base.py b/synapse/replication/http/_base.py
new file mode 100644
index 000000000000..24f00d95fce8
--- /dev/null
+++ b/synapse/replication/http/_base.py
@@ -0,0 +1,208 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import abc
+import logging
+import re
+
+from six.moves import urllib
+
+from twisted.internet import defer
+
+from synapse.api.errors import (
+    CodeMessageException,
+    MatrixCodeMessageException,
+    SynapseError,
+)
+from synapse.util.caches.response_cache import ResponseCache
+from synapse.util.stringutils import random_string
+
+logger = logging.getLogger(__name__)
+
+
+class ReplicationEndpoint(object):
+    """Helper base class for defining new replication HTTP endpoints.
+
+    This creates an endpoint under `/_synapse/replication/:NAME/:PATH_ARGS..`
+    (with an `/:txn_id` prefix for cached requests.), where NAME is a name,
+    PATH_ARGS are a tuple of parameters to be encoded in the URL.
+
+    For example, if `NAME` is "send_event" and `PATH_ARGS` is `("event_id",)`,
+    with `CACHE` set to true then this generates an endpoint:
+
+        /_synapse/replication/send_event/:event_id/:txn_id
+
+    For POST requests the payload is serialized to json and sent as the body,
+    while for GET requests the payload is added as query parameters. See
+    `_serialize_payload` for details.
+
+    Incoming requests are handled by overriding `_handle_request`. Servers
+    must call `register` to register the path with the HTTP server.
+
+    Requests can be sent by calling the client returned by `make_client`.
+
+    Attributes:
+        NAME (str): A name for the endpoint, added to the path as well as used
+            in logging and metrics.
+        PATH_ARGS (tuple[str]): A list of parameters to be added to the path.
+            Adding parameters to the path (rather than payload) can make it
+            easier to follow along in the log files.
+        POST (bool): True to use POST request with JSON body, or false to use
+            GET requests with query params.
+        CACHE (bool): Whether server should cache the result of the request/
+            If true then transparently adds a txn_id to all requests, and
+            `_handle_request` must return a Deferred.
+        RETRY_ON_TIMEOUT(bool): Whether or not to retry the request when a 504
+            is received.
+    """
+
+    __metaclass__ = abc.ABCMeta
+
+    NAME = abc.abstractproperty()
+    PATH_ARGS = abc.abstractproperty()
+
+    POST = True
+    CACHE = True
+    RETRY_ON_TIMEOUT = True
+
+    def __init__(self, hs):
+        if self.CACHE:
+            self.response_cache = ResponseCache(
+                hs, "repl." + self.NAME,
+                timeout_ms=30 * 60 * 1000,
+            )
+
+    @abc.abstractmethod
+    def _serialize_payload(**kwargs):
+        """Static method that is called when creating a request.
+
+        Concrete implementations should have explicit parameters (rather than
+        kwargs) so that an appropriate exception is raised if the client is
+        called with unexpected parameters. All PATH_ARGS must appear in
+        argument list.
+
+        Returns:
+            Deferred[dict]|dict: If POST request then dictionary must be JSON
+            serialisable, otherwise must be appropriate for adding as query
+            args.
+        """
+        return {}
+
+    @abc.abstractmethod
+    def _handle_request(self, request, **kwargs):
+        """Handle incoming request.
+
+        This is called with the request object and PATH_ARGS.
+
+        Returns:
+            Deferred[dict]: A JSON serialisable dict to be used as response
+            body of request.
+        """
+        pass
+
+    @classmethod
+    def make_client(cls, hs):
+        """Create a client that makes requests.
+
+        Returns a callable that accepts the same parameters as `_serialize_payload`.
+        """
+        clock = hs.get_clock()
+        host = hs.config.worker_replication_host
+        port = hs.config.worker_replication_http_port
+
+        client = hs.get_simple_http_client()
+
+        @defer.inlineCallbacks
+        def send_request(**kwargs):
+            data = yield cls._serialize_payload(**kwargs)
+
+            url_args = [urllib.parse.quote(kwargs[name]) for name in cls.PATH_ARGS]
+
+            if cls.CACHE:
+                txn_id = random_string(10)
+                url_args.append(txn_id)
+
+            if cls.POST:
+                request_func = client.post_json_get_json
+            else:
+                request_func = client.get_json
+
+            uri = "http://%s:%s/_synapse/replication/%s/%s" % (
+                host, port, cls.NAME, "/".join(url_args)
+            )
+
+            try:
+                # We keep retrying the same request for timeouts. This is so that we
+                # have a good idea that the request has either succeeded or failed on
+                # the master, and so whether we should clean up or not.
+                while True:
+                    try:
+                        result = yield request_func(uri, data)
+                        break
+                    except CodeMessageException as e:
+                        if e.code != 504 or not cls.RETRY_ON_TIMEOUT:
+                            raise
+
+                    logger.warn("send_federation_events_to_master request timed out")
+
+                    # If we timed out we probably don't need to worry about backing
+                    # off too much, but lets just wait a little anyway.
+                    yield clock.sleep(1)
+            except MatrixCodeMessageException as e:
+                # We convert to SynapseError as we know that it was a SynapseError
+                # on the master process that we should send to the client. (And
+                # importantly, not stack traces everywhere)
+                raise SynapseError(e.code, e.msg, e.errcode)
+
+            defer.returnValue(result)
+
+        return send_request
+
+    def register(self, http_server):
+        """Called by the server to register this as a handler to the
+        appropriate path.
+        """
+
+        url_args = list(self.PATH_ARGS)
+        method = "GET"
+        handler = self._handle_request
+        if self.POST:
+            method = "POST"
+
+        if self.CACHE:
+            handler = self._cached_handler
+            url_args.append("txn_id")
+
+        args = "/".join("(?P<%s>[^/]+)" % (arg,) for arg in url_args)
+        pattern = re.compile("^/_synapse/replication/%s/%s$" % (
+            self.NAME,
+            args
+        ))
+
+        http_server.register_paths(method, [pattern], handler)
+
+    def _cached_handler(self, request, txn_id, **kwargs):
+        """Wraps `_handle_request` the responses should be cached.
+        """
+        # We just use the txn_id here, but we probably also want to use the
+        # other PATH_ARGS as well.
+
+        assert self.CACHE
+
+        return self.response_cache.wrap(
+            txn_id,
+            self._handle_request,
+            request, **kwargs
+        )

From 729b672823132f413800a10f5fa8cac1f9b99008 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 31 Jul 2018 13:53:54 +0100
Subject: [PATCH 193/830] Use new helper base class for
 ReplicationSendEventRestServlet

---
 synapse/handlers/message.py            |  11 +--
 synapse/replication/http/send_event.py | 115 ++++++++-----------------
 2 files changed, 40 insertions(+), 86 deletions(-)

diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 39d77247786c..bcb093ba3ea7 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -30,7 +30,7 @@
 from synapse.crypto.event_signing import add_hashes_and_signatures
 from synapse.events.utils import serialize_event
 from synapse.events.validator import EventValidator
-from synapse.replication.http.send_event import send_event_to_master
+from synapse.replication.http.send_event import ReplicationSendEventRestServlet
 from synapse.types import RoomAlias, UserID
 from synapse.util.async import Linearizer
 from synapse.util.frozenutils import frozendict_json_encoder
@@ -171,7 +171,7 @@ def __init__(self, hs):
         self.notifier = hs.get_notifier()
         self.config = hs.config
 
-        self.http_client = hs.get_simple_http_client()
+        self.send_event_to_master = ReplicationSendEventRestServlet.make_client(hs)
 
         # This is only used to get at ratelimit function, and maybe_kick_guest_users
         self.base_handler = BaseHandler(hs)
@@ -559,12 +559,9 @@ def handle_new_client_event(
         try:
             # If we're a worker we need to hit out to the master.
             if self.config.worker_app:
-                yield send_event_to_master(
-                    clock=self.hs.get_clock(),
+                yield self.send_event_to_master(
+                    event_id=event.event_id,
                     store=self.store,
-                    client=self.http_client,
-                    host=self.config.worker_replication_host,
-                    port=self.config.worker_replication_http_port,
                     requester=requester,
                     event=event,
                     context=context,
diff --git a/synapse/replication/http/send_event.py b/synapse/replication/http/send_event.py
index 5227bc333d19..50810d94cb64 100644
--- a/synapse/replication/http/send_event.py
+++ b/synapse/replication/http/send_event.py
@@ -14,90 +14,26 @@
 # limitations under the License.
 
 import logging
-import re
 
 from twisted.internet import defer
 
-from synapse.api.errors import (
-    CodeMessageException,
-    MatrixCodeMessageException,
-    SynapseError,
-)
 from synapse.events import FrozenEvent
 from synapse.events.snapshot import EventContext
-from synapse.http.servlet import RestServlet, parse_json_object_from_request
+from synapse.http.servlet import parse_json_object_from_request
+from synapse.replication.http._base import ReplicationEndpoint
 from synapse.types import Requester, UserID
-from synapse.util.caches.response_cache import ResponseCache
 from synapse.util.metrics import Measure
 
 logger = logging.getLogger(__name__)
 
 
-@defer.inlineCallbacks
-def send_event_to_master(clock, store, client, host, port, requester, event, context,
-                         ratelimit, extra_users):
-    """Send event to be handled on the master
-
-    Args:
-        clock (synapse.util.Clock)
-        store (DataStore)
-        client (SimpleHttpClient)
-        host (str): host of master
-        port (int): port on master listening for HTTP replication
-        requester (Requester)
-        event (FrozenEvent)
-        context (EventContext)
-        ratelimit (bool)
-        extra_users (list(UserID)): Any extra users to notify about event
-    """
-    uri = "http://%s:%s/_synapse/replication/send_event/%s" % (
-        host, port, event.event_id,
-    )
-
-    serialized_context = yield context.serialize(event, store)
-
-    payload = {
-        "event": event.get_pdu_json(),
-        "internal_metadata": event.internal_metadata.get_dict(),
-        "rejected_reason": event.rejected_reason,
-        "context": serialized_context,
-        "requester": requester.serialize(),
-        "ratelimit": ratelimit,
-        "extra_users": [u.to_string() for u in extra_users],
-    }
-
-    try:
-        # We keep retrying the same request for timeouts. This is so that we
-        # have a good idea that the request has either succeeded or failed on
-        # the master, and so whether we should clean up or not.
-        while True:
-            try:
-                result = yield client.put_json(uri, payload)
-                break
-            except CodeMessageException as e:
-                if e.code != 504:
-                    raise
-
-            logger.warn("send_event request timed out")
-
-            # If we timed out we probably don't need to worry about backing
-            # off too much, but lets just wait a little anyway.
-            yield clock.sleep(1)
-    except MatrixCodeMessageException as e:
-        # We convert to SynapseError as we know that it was a SynapseError
-        # on the master process that we should send to the client. (And
-        # importantly, not stack traces everywhere)
-        raise SynapseError(e.code, e.msg, e.errcode)
-    defer.returnValue(result)
-
-
-class ReplicationSendEventRestServlet(RestServlet):
+class ReplicationSendEventRestServlet(ReplicationEndpoint):
     """Handles events newly created on workers, including persisting and
     notifying.
 
     The API looks like:
 
-        POST /_synapse/replication/send_event/:event_id
+        POST /_synapse/replication/send_event/:event_id/:txn_id
 
         {
             "event": { .. serialized event .. },
@@ -109,27 +45,48 @@ class ReplicationSendEventRestServlet(RestServlet):
             "extra_users": [],
         }
     """
-    PATTERNS = [re.compile("^/_synapse/replication/send_event/(?P<event_id>[^/]+)$")]
+    NAME = "send_event"
+    PATH_ARGS = ("event_id",)
+    POST = True
 
     def __init__(self, hs):
-        super(ReplicationSendEventRestServlet, self).__init__()
+        super(ReplicationSendEventRestServlet, self).__init__(hs)
 
         self.event_creation_handler = hs.get_event_creation_handler()
         self.store = hs.get_datastore()
         self.clock = hs.get_clock()
 
-        # The responses are tiny, so we may as well cache them for a while
-        self.response_cache = ResponseCache(hs, "send_event", timeout_ms=30 * 60 * 1000)
+    @staticmethod
+    @defer.inlineCallbacks
+    def _serialize_payload(event_id, store, event, context, requester,
+                           ratelimit, extra_users):
+        """
+        Args:
+            event_id (str)
+            store (DataStore)
+            requester (Requester)
+            event (FrozenEvent)
+            context (EventContext)
+            ratelimit (bool)
+            extra_users (list(UserID)): Any extra users to notify about event
+        """
+
+        serialized_context = yield context.serialize(event, store)
+
+        payload = {
+            "event": event.get_pdu_json(),
+            "internal_metadata": event.internal_metadata.get_dict(),
+            "rejected_reason": event.rejected_reason,
+            "context": serialized_context,
+            "requester": requester.serialize(),
+            "ratelimit": ratelimit,
+            "extra_users": [u.to_string() for u in extra_users],
+        }
 
-    def on_PUT(self, request, event_id):
-        return self.response_cache.wrap(
-            event_id,
-            self._handle_request,
-            request
-        )
+        defer.returnValue(payload)
 
     @defer.inlineCallbacks
-    def _handle_request(self, request):
+    def _handle_request(self, request, event_id):
         with Measure(self.clock, "repl_send_event_parse"):
             content = parse_json_object_from_request(request)
 

From 443da003bc46da8d6e46403cfa31ee6a4e4da230 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 31 Jul 2018 14:31:51 +0100
Subject: [PATCH 194/830] Use new helper base class for membership requests

---
 synapse/handlers/room_member_worker.py |  41 ++--
 synapse/replication/http/membership.py | 262 +++++++++----------------
 2 files changed, 108 insertions(+), 195 deletions(-)

diff --git a/synapse/handlers/room_member_worker.py b/synapse/handlers/room_member_worker.py
index 22d8b4b0d346..acc6eb8099fb 100644
--- a/synapse/handlers/room_member_worker.py
+++ b/synapse/handlers/room_member_worker.py
@@ -20,16 +20,24 @@
 from synapse.api.errors import SynapseError
 from synapse.handlers.room_member import RoomMemberHandler
 from synapse.replication.http.membership import (
-    get_or_register_3pid_guest,
-    notify_user_membership_change,
-    remote_join,
-    remote_reject_invite,
+    ReplicationRegister3PIDGuestRestServlet as Repl3PID,
+    ReplicationRemoteJoinRestServlet as ReplRemoteJoin,
+    ReplicationRemoteRejectInviteRestServlet as ReplRejectInvite,
+    ReplicationUserJoinedLeftRoomRestServlet as ReplJoinedLeft,
 )
 
 logger = logging.getLogger(__name__)
 
 
 class RoomMemberWorkerHandler(RoomMemberHandler):
+    def __init__(self, hs):
+        super(RoomMemberWorkerHandler, self).__init__(hs)
+
+        self._get_register_3pid_client = Repl3PID.make_client(hs)
+        self._remote_join_client = ReplRemoteJoin.make_client(hs)
+        self._remote_reject_client = ReplRejectInvite.make_client(hs)
+        self._notify_change_client = ReplJoinedLeft.make_client(hs)
+
     @defer.inlineCallbacks
     def _remote_join(self, requester, remote_room_hosts, room_id, user, content):
         """Implements RoomMemberHandler._remote_join
@@ -37,10 +45,7 @@ def _remote_join(self, requester, remote_room_hosts, room_id, user, content):
         if len(remote_room_hosts) == 0:
             raise SynapseError(404, "No known servers")
 
-        ret = yield remote_join(
-            self.simple_http_client,
-            host=self.config.worker_replication_host,
-            port=self.config.worker_replication_http_port,
+        ret = yield self._remote_join_client(
             requester=requester,
             remote_room_hosts=remote_room_hosts,
             room_id=room_id,
@@ -55,10 +60,7 @@ def _remote_join(self, requester, remote_room_hosts, room_id, user, content):
     def _remote_reject_invite(self, requester, remote_room_hosts, room_id, target):
         """Implements RoomMemberHandler._remote_reject_invite
         """
-        return remote_reject_invite(
-            self.simple_http_client,
-            host=self.config.worker_replication_host,
-            port=self.config.worker_replication_http_port,
+        return self._remote_reject_client(
             requester=requester,
             remote_room_hosts=remote_room_hosts,
             room_id=room_id,
@@ -68,10 +70,7 @@ def _remote_reject_invite(self, requester, remote_room_hosts, room_id, target):
     def _user_joined_room(self, target, room_id):
         """Implements RoomMemberHandler._user_joined_room
         """
-        return notify_user_membership_change(
-            self.simple_http_client,
-            host=self.config.worker_replication_host,
-            port=self.config.worker_replication_http_port,
+        return self._notify_change_client(
             user_id=target.to_string(),
             room_id=room_id,
             change="joined",
@@ -80,10 +79,7 @@ def _user_joined_room(self, target, room_id):
     def _user_left_room(self, target, room_id):
         """Implements RoomMemberHandler._user_left_room
         """
-        return notify_user_membership_change(
-            self.simple_http_client,
-            host=self.config.worker_replication_host,
-            port=self.config.worker_replication_http_port,
+        return self._notify_change_client(
             user_id=target.to_string(),
             room_id=room_id,
             change="left",
@@ -92,10 +88,7 @@ def _user_left_room(self, target, room_id):
     def get_or_register_3pid_guest(self, requester, medium, address, inviter_user_id):
         """Implements RoomMemberHandler.get_or_register_3pid_guest
         """
-        return get_or_register_3pid_guest(
-            self.simple_http_client,
-            host=self.config.worker_replication_host,
-            port=self.config.worker_replication_http_port,
+        return self._get_register_3pid_client(
             requester=requester,
             medium=medium,
             address=address,
diff --git a/synapse/replication/http/membership.py b/synapse/replication/http/membership.py
index 6bfc8a5b89d4..8ad83e8421c6 100644
--- a/synapse/replication/http/membership.py
+++ b/synapse/replication/http/membership.py
@@ -14,182 +14,53 @@
 # limitations under the License.
 
 import logging
-import re
 
 from twisted.internet import defer
 
-from synapse.api.errors import MatrixCodeMessageException, SynapseError
-from synapse.http.servlet import RestServlet, parse_json_object_from_request
+from synapse.http.servlet import parse_json_object_from_request
+from synapse.replication.http._base import ReplicationEndpoint
 from synapse.types import Requester, UserID
 from synapse.util.distributor import user_joined_room, user_left_room
 
 logger = logging.getLogger(__name__)
 
 
-@defer.inlineCallbacks
-def remote_join(client, host, port, requester, remote_room_hosts,
-                room_id, user_id, content):
-    """Ask the master to do a remote join for the given user to the given room
-
-    Args:
-        client (SimpleHttpClient)
-        host (str): host of master
-        port (int): port on master listening for HTTP replication
-        requester (Requester)
-        remote_room_hosts (list[str]): Servers to try and join via
-        room_id (str)
-        user_id (str)
-        content (dict): The event content to use for the join event
-
-    Returns:
-        Deferred
-    """
-    uri = "http://%s:%s/_synapse/replication/remote_join" % (host, port)
-
-    payload = {
-        "requester": requester.serialize(),
-        "remote_room_hosts": remote_room_hosts,
-        "room_id": room_id,
-        "user_id": user_id,
-        "content": content,
-    }
-
-    try:
-        result = yield client.post_json_get_json(uri, payload)
-    except MatrixCodeMessageException as e:
-        # We convert to SynapseError as we know that it was a SynapseError
-        # on the master process that we should send to the client. (And
-        # importantly, not stack traces everywhere)
-        raise SynapseError(e.code, e.msg, e.errcode)
-    defer.returnValue(result)
-
-
-@defer.inlineCallbacks
-def remote_reject_invite(client, host, port, requester, remote_room_hosts,
-                         room_id, user_id):
-    """Ask master to reject the invite for the user and room.
-
-    Args:
-        client (SimpleHttpClient)
-        host (str): host of master
-        port (int): port on master listening for HTTP replication
-        requester (Requester)
-        remote_room_hosts (list[str]): Servers to try and reject via
-        room_id (str)
-        user_id (str)
-
-    Returns:
-        Deferred
-    """
-    uri = "http://%s:%s/_synapse/replication/remote_reject_invite" % (host, port)
-
-    payload = {
-        "requester": requester.serialize(),
-        "remote_room_hosts": remote_room_hosts,
-        "room_id": room_id,
-        "user_id": user_id,
-    }
-
-    try:
-        result = yield client.post_json_get_json(uri, payload)
-    except MatrixCodeMessageException as e:
-        # We convert to SynapseError as we know that it was a SynapseError
-        # on the master process that we should send to the client. (And
-        # importantly, not stack traces everywhere)
-        raise SynapseError(e.code, e.msg, e.errcode)
-    defer.returnValue(result)
-
-
-@defer.inlineCallbacks
-def get_or_register_3pid_guest(client, host, port, requester,
-                               medium, address, inviter_user_id):
-    """Ask the master to get/create a guest account for given 3PID.
-
-    Args:
-        client (SimpleHttpClient)
-        host (str): host of master
-        port (int): port on master listening for HTTP replication
-        requester (Requester)
-        medium (str)
-        address (str)
-        inviter_user_id (str): The user ID who is trying to invite the
-            3PID
-
-    Returns:
-        Deferred[(str, str)]: A 2-tuple of `(user_id, access_token)` of the
-        3PID guest account.
-    """
-
-    uri = "http://%s:%s/_synapse/replication/get_or_register_3pid_guest" % (host, port)
-
-    payload = {
-        "requester": requester.serialize(),
-        "medium": medium,
-        "address": address,
-        "inviter_user_id": inviter_user_id,
-    }
-
-    try:
-        result = yield client.post_json_get_json(uri, payload)
-    except MatrixCodeMessageException as e:
-        # We convert to SynapseError as we know that it was a SynapseError
-        # on the master process that we should send to the client. (And
-        # importantly, not stack traces everywhere)
-        raise SynapseError(e.code, e.msg, e.errcode)
-    defer.returnValue(result)
-
-
-@defer.inlineCallbacks
-def notify_user_membership_change(client, host, port, user_id, room_id, change):
-    """Notify master that a user has joined or left the room
-
-    Args:
-        client (SimpleHttpClient)
-        host (str): host of master
-        port (int): port on master listening for HTTP replication.
-        user_id (str)
-        room_id (str)
-        change (str): Either "join" or "left"
-
-    Returns:
-        Deferred
+class ReplicationRemoteJoinRestServlet(ReplicationEndpoint):
+    """Does a remote join for the given user to the given room
     """
-    assert change in ("joined", "left")
-
-    uri = "http://%s:%s/_synapse/replication/user_%s_room" % (host, port, change)
-
-    payload = {
-        "user_id": user_id,
-        "room_id": room_id,
-    }
-
-    try:
-        result = yield client.post_json_get_json(uri, payload)
-    except MatrixCodeMessageException as e:
-        # We convert to SynapseError as we know that it was a SynapseError
-        # on the master process that we should send to the client. (And
-        # importantly, not stack traces everywhere)
-        raise SynapseError(e.code, e.msg, e.errcode)
-    defer.returnValue(result)
-
 
-class ReplicationRemoteJoinRestServlet(RestServlet):
-    PATTERNS = [re.compile("^/_synapse/replication/remote_join$")]
+    NAME = "remote_join"
+    PATH_ARGS = ("room_id", "user_id",)
 
     def __init__(self, hs):
-        super(ReplicationRemoteJoinRestServlet, self).__init__()
+        super(ReplicationRemoteJoinRestServlet, self).__init__(hs)
 
         self.federation_handler = hs.get_handlers().federation_handler
         self.store = hs.get_datastore()
         self.clock = hs.get_clock()
 
+    @staticmethod
+    def _serialize_payload(requester, room_id, user_id, remote_room_hosts,
+                           content):
+        """
+        Args:
+            requester(Requester)
+            room_id (str)
+            user_id (str)
+            remote_room_hosts (list[str]): Servers to try and join via
+            content(dict): The event content to use for the join event
+        """
+        return {
+            "requester": requester.serialize(),
+            "remote_room_hosts": remote_room_hosts,
+            "content": content,
+        }
+
     @defer.inlineCallbacks
-    def on_POST(self, request):
+    def _handle_request(self, request, room_id, user_id):
         content = parse_json_object_from_request(request)
 
         remote_room_hosts = content["remote_room_hosts"]
-        room_id = content["room_id"]
-        user_id = content["user_id"]
         event_content = content["content"]
 
         requester = Requester.deserialize(self.store, content["requester"])
@@ -212,23 +83,39 @@ def on_POST(self, request):
         defer.returnValue((200, {}))
 
 
-class ReplicationRemoteRejectInviteRestServlet(RestServlet):
-    PATTERNS = [re.compile("^/_synapse/replication/remote_reject_invite$")]
+class ReplicationRemoteRejectInviteRestServlet(ReplicationEndpoint):
+    """Rejects the invite for the user and room.
+    """
+
+    NAME = "remote_reject_invite"
+    PATH_ARGS = ("room_id", "user_id",)
 
     def __init__(self, hs):
-        super(ReplicationRemoteRejectInviteRestServlet, self).__init__()
+        super(ReplicationRemoteRejectInviteRestServlet, self).__init__(hs)
 
         self.federation_handler = hs.get_handlers().federation_handler
         self.store = hs.get_datastore()
         self.clock = hs.get_clock()
 
+    @staticmethod
+    def _serialize_payload(requester, room_id, user_id, remote_room_hosts):
+        """
+        Args:
+            requester(Requester)
+            room_id (str)
+            user_id (str)
+            remote_room_hosts (list[str]): Servers to try and reject via
+        """
+        return {
+            "requester": requester.serialize(),
+            "remote_room_hosts": remote_room_hosts,
+        }
+
     @defer.inlineCallbacks
-    def on_POST(self, request):
+    def _handle_request(self, request, room_id, user_id):
         content = parse_json_object_from_request(request)
 
         remote_room_hosts = content["remote_room_hosts"]
-        room_id = content["room_id"]
-        user_id = content["user_id"]
 
         requester = Requester.deserialize(self.store, content["requester"])
 
@@ -264,18 +151,39 @@ def on_POST(self, request):
         defer.returnValue((200, ret))
 
 
-class ReplicationRegister3PIDGuestRestServlet(RestServlet):
-    PATTERNS = [re.compile("^/_synapse/replication/get_or_register_3pid_guest$")]
+class ReplicationRegister3PIDGuestRestServlet(ReplicationEndpoint):
+    """Gets/creates a guest account for given 3PID.
+    """
+
+    NAME = "get_or_register_3pid_guest"
+    PATH_ARGS = ()
 
     def __init__(self, hs):
-        super(ReplicationRegister3PIDGuestRestServlet, self).__init__()
+        super(ReplicationRegister3PIDGuestRestServlet, self).__init__(hs)
 
         self.registeration_handler = hs.get_handlers().registration_handler
         self.store = hs.get_datastore()
         self.clock = hs.get_clock()
 
+    @staticmethod
+    def _serialize_payload(requester, medium, address, inviter_user_id):
+        """
+        Args:
+            requester(Requester)
+            medium (str)
+            address (str)
+            inviter_user_id (str): The user ID who is trying to invite the
+                3PID
+        """
+        return {
+            "requester": requester.serialize(),
+            "medium": medium,
+            "address": address,
+            "inviter_user_id": inviter_user_id,
+        }
+
     @defer.inlineCallbacks
-    def on_POST(self, request):
+    def _handle_request(self, request):
         content = parse_json_object_from_request(request)
 
         medium = content["medium"]
@@ -296,23 +204,35 @@ def on_POST(self, request):
         defer.returnValue((200, ret))
 
 
-class ReplicationUserJoinedLeftRoomRestServlet(RestServlet):
-    PATTERNS = [re.compile("^/_synapse/replication/user_(?P<change>joined|left)_room$")]
+class ReplicationUserJoinedLeftRoomRestServlet(ReplicationEndpoint):
+    """Notifies that a user has joined or left the room
+    """
+
+    NAME = "membership_change"
+    PATH_ARGS = ("room_id", "user_id", "change")
+    CACHE = False  # No point caching as should return instantly.
 
     def __init__(self, hs):
-        super(ReplicationUserJoinedLeftRoomRestServlet, self).__init__()
+        super(ReplicationUserJoinedLeftRoomRestServlet, self).__init__(hs)
 
         self.registeration_handler = hs.get_handlers().registration_handler
         self.store = hs.get_datastore()
         self.clock = hs.get_clock()
         self.distributor = hs.get_distributor()
 
-    def on_POST(self, request, change):
-        content = parse_json_object_from_request(request)
+    @staticmethod
+    def _serialize_payload(room_id, user_id, change):
+        """
+        Args:
+            room_id (str)
+            user_id (str)
+            change (str): Either "joined" or "left"
+        """
+        assert change in ("joined", "left",)
 
-        user_id = content["user_id"]
-        room_id = content["room_id"]
+        return {}
 
+    def _handle_request(self, request, room_id, user_id, change):
         logger.info("user membership change: %s in %s", user_id, room_id)
 
         user = UserID.from_string(user_id)

From 16bd63f32f97a25252bb438413b79f6bdf1e5c1e Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 31 Jul 2018 14:48:43 +0100
Subject: [PATCH 195/830] Newsfile

---
 changelog.d/3632.misc | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 changelog.d/3632.misc

diff --git a/changelog.d/3632.misc b/changelog.d/3632.misc
new file mode 100644
index 000000000000..e69de29bb2d1

From bdbdceeafa7755348486e1d0262a662f2529b884 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 31 Jul 2018 15:44:05 +0100
Subject: [PATCH 196/830] rename replication_layer to federation_client

I have HAD ENOUGH of trying to remember wtf a replication layer is in terms of
classes.
---
 synapse/handlers/federation.py | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 49068c06d994..91d8def08ba6 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -76,7 +76,7 @@ def __init__(self, hs):
         self.hs = hs
 
         self.store = hs.get_datastore()
-        self.replication_layer = hs.get_federation_client()
+        self.federation_client = hs.get_federation_client()
         self.state_handler = hs.get_state_handler()
         self.server_name = hs.hostname
         self.keyring = hs.get_keyring()
@@ -255,7 +255,7 @@ def on_receive_pdu(
                     # know about
                     for p in prevs - seen:
                         state, got_auth_chain = (
-                            yield self.replication_layer.get_state_for_room(
+                            yield self.federation_client.get_state_for_room(
                                 origin, pdu.room_id, p
                             )
                         )
@@ -338,7 +338,7 @@ def _get_missing_events_for_pdu(self, origin, pdu, prevs, min_depth):
         #
         # see https://github.com/matrix-org/synapse/pull/1744
 
-        missing_events = yield self.replication_layer.get_missing_events(
+        missing_events = yield self.federation_client.get_missing_events(
             origin,
             pdu.room_id,
             earliest_events_ids=list(latest),
@@ -522,7 +522,7 @@ def backfill(self, dest, room_id, limit, extremities):
         if dest == self.server_name:
             raise SynapseError(400, "Can't backfill from self.")
 
-        events = yield self.replication_layer.backfill(
+        events = yield self.federation_client.backfill(
             dest,
             room_id,
             limit=limit,
@@ -570,7 +570,7 @@ def backfill(self, dest, room_id, limit, extremities):
         state_events = {}
         events_to_state = {}
         for e_id in edges:
-            state, auth = yield self.replication_layer.get_state_for_room(
+            state, auth = yield self.federation_client.get_state_for_room(
                 destination=dest,
                 room_id=room_id,
                 event_id=e_id
@@ -612,7 +612,7 @@ def backfill(self, dest, room_id, limit, extremities):
                 results = yield logcontext.make_deferred_yieldable(defer.gatherResults(
                     [
                         logcontext.run_in_background(
-                            self.replication_layer.get_pdu,
+                            self.federation_client.get_pdu,
                             [dest],
                             event_id,
                             outlier=True,
@@ -893,7 +893,7 @@ def send_invite(self, target_host, event):
 
         Invites must be signed by the invitee's server before distribution.
         """
-        pdu = yield self.replication_layer.send_invite(
+        pdu = yield self.federation_client.send_invite(
             destination=target_host,
             room_id=event.room_id,
             event_id=event.event_id,
@@ -955,7 +955,7 @@ def do_invite_join(self, target_hosts, room_id, joinee, content):
                 target_hosts.insert(0, origin)
             except ValueError:
                 pass
-            ret = yield self.replication_layer.send_join(target_hosts, event)
+            ret = yield self.federation_client.send_join(target_hosts, event)
 
             origin = ret["origin"]
             state = ret["state"]
@@ -1211,7 +1211,7 @@ def do_remotely_reject_invite(self, target_hosts, room_id, user_id):
         except ValueError:
             pass
 
-        yield self.replication_layer.send_leave(
+        yield self.federation_client.send_leave(
             target_hosts,
             event
         )
@@ -1234,7 +1234,7 @@ def do_remotely_reject_invite(self, target_hosts, room_id, user_id):
     @defer.inlineCallbacks
     def _make_and_verify_event(self, target_hosts, room_id, user_id, membership,
                                content={},):
-        origin, pdu = yield self.replication_layer.make_membership_event(
+        origin, pdu = yield self.federation_client.make_membership_event(
             target_hosts,
             room_id,
             user_id,
@@ -1567,7 +1567,7 @@ def _persist_auth_tree(self, origin, auth_events, state, event):
                     missing_auth_events.add(e_id)
 
         for e_id in missing_auth_events:
-            m_ev = yield self.replication_layer.get_pdu(
+            m_ev = yield self.federation_client.get_pdu(
                 [origin],
                 e_id,
                 outlier=True,
@@ -1777,7 +1777,7 @@ def do_auth(self, origin, event, context, auth_events):
             logger.info("Missing auth: %s", missing_auth)
             # If we don't have all the auth events, we need to get them.
             try:
-                remote_auth_chain = yield self.replication_layer.get_event_auth(
+                remote_auth_chain = yield self.federation_client.get_event_auth(
                     origin, event.room_id, event.event_id
                 )
 
@@ -1893,7 +1893,7 @@ def do_auth(self, origin, event, context, auth_events):
 
                 try:
                     # 2. Get remote difference.
-                    result = yield self.replication_layer.query_auth(
+                    result = yield self.federation_client.query_auth(
                         origin,
                         event.room_id,
                         event.event_id,
@@ -2192,7 +2192,7 @@ def exchange_third_party_invite(
             yield member_handler.send_membership_event(None, event, context)
         else:
             destinations = set(x.split(":", 1)[-1] for x in (sender_user_id, room_id))
-            yield self.replication_layer.forward_third_party_invite(
+            yield self.federation_client.forward_third_party_invite(
                 destinations,
                 room_id,
                 event_dict,

From 6ef983ce5cc0a1cd7323ac82c8eed41d72ff3a99 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Tue, 31 Jul 2018 16:36:24 +0100
Subject: [PATCH 197/830] api into monthly_active_users table

---
 synapse/app/homeserver.py                     |  4 +
 synapse/storage/monthly_active_users.py       | 89 +++++++++++++++++++
 synapse/storage/prepare_database.py           |  2 +-
 .../delta/50/make_event_content_nullable.py   |  2 +-
 .../schema/delta/51/monthly_active_users.sql  | 23 +++++
 tests/storage/test_monthly_active_users.py    | 42 +++++++++
 6 files changed, 160 insertions(+), 2 deletions(-)
 create mode 100644 synapse/storage/monthly_active_users.py
 create mode 100644 synapse/storage/schema/delta/51/monthly_active_users.sql
 create mode 100644 tests/storage/test_monthly_active_users.py

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 57b815d777d7..79772fa61a8a 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -62,6 +62,7 @@
 from synapse.server import HomeServer
 from synapse.storage import are_all_users_on_domain
 from synapse.storage.engines import IncorrectDatabaseSetup, create_engine
+from synapse.storage.monthly_active_users import MonthlyActiveUsersStore
 from synapse.storage.prepare_database import UpgradeDatabaseException, prepare_database
 from synapse.util.caches import CACHE_SIZE_FACTOR
 from synapse.util.httpresourcetree import create_resource_tree
@@ -511,6 +512,9 @@ def generate_user_daily_visit_stats():
     # If you increase the loop period, the accuracy of user_daily_visits
     # table will decrease
     clock.looping_call(generate_user_daily_visit_stats, 5 * 60 * 1000)
+    clock.looping_call(
+        MonthlyActiveUsersStore(hs).reap_monthly_active_users, 1000 * 60 * 60
+    )
 
     if hs.config.report_stats:
         logger.info("Scheduling stats reporting for 3 hour intervals")
diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py
new file mode 100644
index 000000000000..373e828c0ae7
--- /dev/null
+++ b/synapse/storage/monthly_active_users.py
@@ -0,0 +1,89 @@
+from twisted.internet import defer
+
+from ._base import SQLBaseStore
+
+
+class MonthlyActiveUsersStore(SQLBaseStore):
+    def __init__(self, hs):
+        super(MonthlyActiveUsersStore, self).__init__(None, hs)
+        self._clock = hs.get_clock()
+
+    def reap_monthly_active_users(self):
+        """
+        Cleans out monthly active user table to ensure that no stale
+        entries exist.
+        Return:
+            defered, no return type
+        """
+        def _reap_users(txn):
+            thirty_days_ago = (
+                int(self._clock.time_msec()) - (1000 * 60 * 60 * 24 * 30)
+            )
+            sql = "DELETE FROM monthly_active_users WHERE timestamp < ?"
+            txn.execute(sql, (thirty_days_ago,))
+
+        return self.runInteraction("reap_monthly_active_users", _reap_users)
+
+    def get_monthly_active_count(self):
+        """
+            Generates current count of monthly active users.abs
+            return:
+                defered resolves to int
+        """
+        def _count_users(txn):
+            sql = """
+                SELECT COALESCE(count(*), 0) FROM (
+                    SELECT user_id FROM monthly_active_users
+                ) u
+            """
+            txn.execute(sql)
+            count, = txn.fetchone()
+            return count
+        return self.runInteraction("count_users", _count_users)
+
+    def upsert_monthly_active_user(self, user_id):
+        """
+            Updates or inserts monthly active user member
+            Arguments:
+                user_id (str): user to add/update
+        """
+        return self._simple_upsert(
+            desc="upsert_monthly_active_user",
+            table="monthly_active_users",
+            keyvalues={
+                "user_id": user_id,
+            },
+            values={
+                "timestamp": int(self._clock.time_msec()),
+            },
+            lock=False,
+        )
+
+    def clean_out_monthly_active_users(self):
+        pass
+
+    @defer.inlineCallbacks
+    def is_user_monthly_active(self, user_id):
+        """
+            Checks if a given user is part of the monthly active user group
+            Arguments:
+                user_id (str): user to add/update
+            Return:
+                bool : True if user part of group, False otherwise
+        """
+        user_present = yield self._simple_select_onecol(
+            table="monthly_active_users",
+            keyvalues={
+                "user_id": user_id,
+            },
+            retcol="user_id",
+            desc="is_user_monthly_active",
+        )
+        # jeff = self.cursor_to_dict(res)
+        result = False
+        if user_present:
+            result = True
+
+        defer.returnValue(
+            result
+        )
diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py
index b290f834b3d2..b364719312d6 100644
--- a/synapse/storage/prepare_database.py
+++ b/synapse/storage/prepare_database.py
@@ -25,7 +25,7 @@
 
 # Remember to update this number every time a change is made to database
 # schema files, so the users will be informed on server restarts.
-SCHEMA_VERSION = 50
+SCHEMA_VERSION = 51
 
 dir_path = os.path.abspath(os.path.dirname(__file__))
 
diff --git a/synapse/storage/schema/delta/50/make_event_content_nullable.py b/synapse/storage/schema/delta/50/make_event_content_nullable.py
index 7d27342e3994..6dd467b6c533 100644
--- a/synapse/storage/schema/delta/50/make_event_content_nullable.py
+++ b/synapse/storage/schema/delta/50/make_event_content_nullable.py
@@ -88,5 +88,5 @@ def run_upgrade(cur, database_engine, *args, **kwargs):
         "UPDATE sqlite_master SET sql=? WHERE tbl_name='events' AND type='table'",
         (sql, ),
     )
-    cur.execute("PRAGMA schema_version=%i" % (oldver+1,))
+    cur.execute("PRAGMA schema_version=%i" % (oldver + 1,))
     cur.execute("PRAGMA writable_schema=OFF")
diff --git a/synapse/storage/schema/delta/51/monthly_active_users.sql b/synapse/storage/schema/delta/51/monthly_active_users.sql
new file mode 100644
index 000000000000..b3c0e1a67621
--- /dev/null
+++ b/synapse/storage/schema/delta/51/monthly_active_users.sql
@@ -0,0 +1,23 @@
+/* Copyright 2018 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- a table of users who have requested that their details be erased
+CREATE TABLE monthly_active_users (
+    user_id TEXT NOT NULL,
+    timestamp BIGINT NOT NULL
+);
+
+CREATE UNIQUE INDEX monthly_active_users_users ON monthly_active_users(user_id);
+CREATE INDEX monthly_active_users_time_stamp ON monthly_active_users(timestamp);
diff --git a/tests/storage/test_monthly_active_users.py b/tests/storage/test_monthly_active_users.py
new file mode 100644
index 000000000000..9b1ffc6369ee
--- /dev/null
+++ b/tests/storage/test_monthly_active_users.py
@@ -0,0 +1,42 @@
+from twisted.internet import defer
+
+from synapse.storage.monthly_active_users import MonthlyActiveUsersStore
+
+import tests.unittest
+import tests.utils
+from tests.utils import setup_test_homeserver
+
+
+class MonthlyActiveUsersTestCase(tests.unittest.TestCase):
+    def __init__(self, *args, **kwargs):
+        super(MonthlyActiveUsersTestCase, self).__init__(*args, **kwargs)
+        self.mau = None
+
+    @defer.inlineCallbacks
+    def setUp(self):
+        hs = yield setup_test_homeserver()
+        self.mau = MonthlyActiveUsersStore(hs)
+
+    @defer.inlineCallbacks
+    def test_can_insert_and_count_mau(self):
+        count = yield self.mau.get_monthly_active_count()
+        self.assertEqual(0, count)
+
+        yield self.mau.upsert_monthly_active_user("@user:server")
+        count = yield self.mau.get_monthly_active_count()
+
+        self.assertEqual(1, count)
+
+    @defer.inlineCallbacks
+    def test_is_user_monthly_active(self):
+        user_id1 = "@user1:server"
+        user_id2 = "@user2:server"
+        user_id3 = "@user3:server"
+        result = yield self.mau.is_user_monthly_active(user_id1)
+        self.assertFalse(result)
+        yield self.mau.upsert_monthly_active_user(user_id1)
+        yield self.mau.upsert_monthly_active_user(user_id2)
+        result = yield self.mau.is_user_monthly_active(user_id1)
+        self.assertTrue(result)
+        result = yield self.mau.is_user_monthly_active(user_id3)
+        self.assertFalse(result)

From 1841672c855c53a95cc586fe33662bcbbb156aed Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 31 Jul 2018 18:26:54 +0100
Subject: [PATCH 198/830] changelog

---
 changelog.d/3634.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3634.misc

diff --git a/changelog.d/3634.misc b/changelog.d/3634.misc
new file mode 100644
index 000000000000..2cd6af91ff16
--- /dev/null
+++ b/changelog.d/3634.misc
@@ -0,0 +1 @@
+rename replication_layer to federation_client

From 70af98e36155ee8d4a0ad79d9d33f891458fd3f6 Mon Sep 17 00:00:00 2001
From: Serban Constantin <serban.constantin@sparkware.ro>
Date: Fri, 27 Jul 2018 16:19:38 +0300
Subject: [PATCH 199/830] return NotFoundError if room not found

Per the Client-Server API[0] we should return
`M_NOT_FOUND` if the room isn't found instead
of generic SynapseError.

This ensures that /directory/list API returns
404 for room not found instead of 400.

[0]: https://matrix.org/docs/spec/client_server/unstable.html#get-matrix-client-r0-directory-list-room-roomid

Signed-off-by: Serban Constantin <serban.constantin@gmail.com>
---
 AUTHORS.rst                         | 5 ++++-
 changelog.d/2952.bugfix             | 1 +
 synapse/rest/client/v1/directory.py | 4 ++--
 3 files changed, 7 insertions(+), 3 deletions(-)
 create mode 100644 changelog.d/2952.bugfix

diff --git a/AUTHORS.rst b/AUTHORS.rst
index e13ac5ad348a..9a83d9015362 100644
--- a/AUTHORS.rst
+++ b/AUTHORS.rst
@@ -62,4 +62,7 @@ Christoph Witzany <christoph at web.crofting.com>
  * Add LDAP support for authentication
 
 Pierre Jaury <pierre at jaury.eu>
-* Docker packaging
\ No newline at end of file
+* Docker packaging
+
+Serban Constantin <serban.constantin at gmail dot com>
+ * Small bug fix
\ No newline at end of file
diff --git a/changelog.d/2952.bugfix b/changelog.d/2952.bugfix
new file mode 100644
index 000000000000..e5bc8a8762f0
--- /dev/null
+++ b/changelog.d/2952.bugfix
@@ -0,0 +1 @@
+/directory/list API returns 404 for room not found instead of 400
\ No newline at end of file
diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py
index 69dcd618cb6e..97733f302642 100644
--- a/synapse/rest/client/v1/directory.py
+++ b/synapse/rest/client/v1/directory.py
@@ -18,7 +18,7 @@
 
 from twisted.internet import defer
 
-from synapse.api.errors import AuthError, Codes, SynapseError
+from synapse.api.errors import AuthError, Codes, NotFoundError, SynapseError
 from synapse.http.servlet import parse_json_object_from_request
 from synapse.types import RoomAlias
 
@@ -159,7 +159,7 @@ def __init__(self, hs):
     def on_GET(self, request, room_id):
         room = yield self.store.get_room(room_id)
         if room is None:
-            raise SynapseError(400, "Unknown room")
+            raise NotFoundError("Unknown room")
 
         defer.returnValue((200, {
             "visibility": "public" if room["is_public"] else "private"

From 37be52ac34f3ec2e409f2961e428c1f42d25d729 Mon Sep 17 00:00:00 2001
From: Travis Ralston <travpc@gmail.com>
Date: Tue, 31 Jul 2018 16:29:09 -0600
Subject: [PATCH 200/830] limt -> limit

---
 synapse/federation/transport/client.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index 4529d454af46..625b4def9b62 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -106,7 +106,7 @@ def backfill(self, destination, room_id, event_tuples, limit):
             dest (str)
             room_id (str)
             event_tuples (list)
-            limt (int)
+            limit (int)
 
         Returns:
             Deferred: Results in a dict received from the remote homeserver.

From c507fa15ce068b496d320f08fcb6d6aedf4ace32 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 1 Aug 2018 10:20:42 +0100
Subject: [PATCH 201/830] only need to loop if mau limiting is enabled

---
 synapse/app/homeserver.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 82979e7d1b0a..2da60682ec74 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -531,7 +531,8 @@ def generate_monthly_active_users():
         limit_usage_by_mau_gauge.set(float(hs.config.limit_usage_by_mau))
 
     generate_monthly_active_users()
-    clock.looping_call(generate_monthly_active_users, 5 * 60 * 1000)
+    if hs.config.limit_usage_by_mau:
+        clock.looping_call(generate_monthly_active_users, 5 * 60 * 1000)
 
     if hs.config.report_stats:
         logger.info("Scheduling stats reporting for 3 hour intervals")

From 7931393495c76eef0af9b91c7904c88943197054 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 1 Aug 2018 10:21:56 +0100
Subject: [PATCH 202/830] make count_monthly_users async
 synapse/handlers/auth.py

---
 synapse/handlers/register.py    |  9 ++++----
 synapse/storage/__init__.py     | 26 +++++++++++-----------
 tests/handlers/test_auth.py     | 39 +++++++++++++++++++--------------
 tests/handlers/test_register.py | 10 +++++----
 4 files changed, 46 insertions(+), 38 deletions(-)

diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index f46b8355c0d0..cc935a5e844c 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -144,7 +144,7 @@ def register(
         Raises:
             RegistrationError if there was a problem registering.
         """
-        self._check_mau_limits()
+        yield self._check_mau_limits()
         password_hash = None
         if password:
             password_hash = yield self.auth_handler().hash(password)
@@ -289,7 +289,7 @@ def register_saml2(self, localpart):
                 400,
                 "User ID can only contain characters a-z, 0-9, or '=_-./'",
             )
-        self._check_mau_limits()
+        yield self._check_mau_limits()
         user = UserID(localpart, self.hs.hostname)
         user_id = user.to_string()
 
@@ -439,7 +439,7 @@ def get_or_create_user(self, requester, localpart, displayname,
         """
         if localpart is None:
             raise SynapseError(400, "Request must include user id")
-        self._check_mau_limits()
+        yield self._check_mau_limits()
         need_register = True
 
         try:
@@ -534,13 +534,14 @@ def _join_user_to_room(self, requester, room_identifier):
             action="join",
         )
 
+    @defer.inlineCallbacks
     def _check_mau_limits(self):
         """
         Do not accept registrations if monthly active user limits exceeded
          and limiting is enabled
         """
         if self.hs.config.limit_usage_by_mau is True:
-            current_mau = self.store.count_monthly_users()
+            current_mau = yield self.store.count_monthly_users()
             if current_mau >= self.hs.config.max_mau_value:
                 raise RegistrationError(
                     403, "MAU Limit Exceeded", Codes.MAU_LIMIT_EXCEEDED
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 4747118ed716..f9682832cad4 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -273,24 +273,24 @@ def count_monthly_users(self):
         This method should be refactored with count_daily_users - the only
         reason not to is waiting on definition of mau
         returns:
-            int: count of current monthly active users
+            defered: resolves to int
         """
+        def _count_monthly_users(txn):
+            thirty_days_ago = int(self._clock.time_msec()) - (1000 * 60 * 60 * 24 * 30)
+            sql = """
+                SELECT COALESCE(count(*), 0) FROM (
+                    SELECT user_id FROM user_ips
+                    WHERE last_seen > ?
+                    GROUP BY user_id
+                ) u
+            """
 
-        thirty_days_ago = int(self._clock.time_msec()) - (1000 * 60 * 60 * 24 * 30)
-        sql = """
-            SELECT COALESCE(count(*), 0) FROM (
-                SELECT user_id FROM user_ips
-                WHERE last_seen > ?
-                GROUP BY user_id
-            ) u
-        """
-        try:
-            txn = self.db_conn.cursor()
             txn.execute(sql, (thirty_days_ago,))
             count, = txn.fetchone()
+            print "Count is %d" % (count,)
             return count
-        finally:
-            txn.close()
+
+        return self.runInteraction("count_monthly_users", _count_monthly_users)
 
     def count_r30_users(self):
         """
diff --git a/tests/handlers/test_auth.py b/tests/handlers/test_auth.py
index e01f14a10aa1..440a45308222 100644
--- a/tests/handlers/test_auth.py
+++ b/tests/handlers/test_auth.py
@@ -77,38 +77,37 @@ def verify_nonce(caveat):
         v.satisfy_general(verify_nonce)
         v.verify(macaroon, self.hs.config.macaroon_secret_key)
 
+    @defer.inlineCallbacks
     def test_short_term_login_token_gives_user_id(self):
         self.hs.clock.now = 1000
 
         token = self.macaroon_generator.generate_short_term_login_token(
             "a_user", 5000
         )
-
-        self.assertEqual(
-            "a_user",
-            self.auth_handler.validate_short_term_login_token_and_get_user_id(
-                token
-            )
+        user_id = yield self.auth_handler.validate_short_term_login_token_and_get_user_id(
+            token
         )
+        self.assertEqual("a_user", user_id)
 
         # when we advance the clock, the token should be rejected
         self.hs.clock.now = 6000
         with self.assertRaises(synapse.api.errors.AuthError):
-            self.auth_handler.validate_short_term_login_token_and_get_user_id(
+            yield self.auth_handler.validate_short_term_login_token_and_get_user_id(
                 token
             )
 
+    @defer.inlineCallbacks
     def test_short_term_login_token_cannot_replace_user_id(self):
         token = self.macaroon_generator.generate_short_term_login_token(
             "a_user", 5000
         )
         macaroon = pymacaroons.Macaroon.deserialize(token)
 
+        user_id = yield self.auth_handler.validate_short_term_login_token_and_get_user_id(
+            macaroon.serialize()
+        )
         self.assertEqual(
-            "a_user",
-            self.auth_handler.validate_short_term_login_token_and_get_user_id(
-                macaroon.serialize()
-            )
+            "a_user", user_id
         )
 
         # add another "user_id" caveat, which might allow us to override the
@@ -116,7 +115,7 @@ def test_short_term_login_token_cannot_replace_user_id(self):
         macaroon.add_first_party_caveat("user_id = b_user")
 
         with self.assertRaises(synapse.api.errors.AuthError):
-            self.auth_handler.validate_short_term_login_token_and_get_user_id(
+            yield self.auth_handler.validate_short_term_login_token_and_get_user_id(
                 macaroon.serialize()
             )
 
@@ -126,7 +125,7 @@ def test_mau_limits_disabled(self):
         # Ensure does not throw exception
         yield self.auth_handler.get_access_token_for_user_id('user_a')
 
-        self.auth_handler.validate_short_term_login_token_and_get_user_id(
+        yield self.auth_handler.validate_short_term_login_token_and_get_user_id(
             self._get_macaroon().serialize()
         )
 
@@ -134,24 +133,30 @@ def test_mau_limits_disabled(self):
     def test_mau_limits_exceeded(self):
         self.hs.config.limit_usage_by_mau = True
         self.hs.get_datastore().count_monthly_users = Mock(
-            return_value=self.large_number_of_users
+            return_value=defer.succeed(self.large_number_of_users)
         )
+
         with self.assertRaises(AuthError):
             yield self.auth_handler.get_access_token_for_user_id('user_a')
+
+        self.hs.get_datastore().count_monthly_users = Mock(
+            return_value=defer.succeed(self.large_number_of_users)
+        )
         with self.assertRaises(AuthError):
-            self.auth_handler.validate_short_term_login_token_and_get_user_id(
+            yield self.auth_handler.validate_short_term_login_token_and_get_user_id(
                 self._get_macaroon().serialize()
             )
 
     @defer.inlineCallbacks
     def test_mau_limits_not_exceeded(self):
         self.hs.config.limit_usage_by_mau = True
+
         self.hs.get_datastore().count_monthly_users = Mock(
-            return_value=self.small_number_of_users
+            return_value=defer.succeed(self.small_number_of_users)
         )
         # Ensure does not raise exception
         yield self.auth_handler.get_access_token_for_user_id('user_a')
-        self.auth_handler.validate_short_term_login_token_and_get_user_id(
+        yield self.auth_handler.validate_short_term_login_token_and_get_user_id(
             self._get_macaroon().serialize()
         )
 
diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py
index a5a8e7c95402..0937d71cf60f 100644
--- a/tests/handlers/test_register.py
+++ b/tests/handlers/test_register.py
@@ -90,7 +90,7 @@ def test_cannot_register_when_mau_limits_exceeded(self):
         lots_of_users = 100
         small_number_users = 1
 
-        store.count_monthly_users = Mock(return_value=lots_of_users)
+        store.count_monthly_users = Mock(return_value=defer.succeed(lots_of_users))
 
         # Ensure does not throw exception
         yield self.handler.get_or_create_user(requester, 'a', display_name)
@@ -100,7 +100,7 @@ def test_cannot_register_when_mau_limits_exceeded(self):
         with self.assertRaises(RegistrationError):
             yield self.handler.get_or_create_user(requester, 'b', display_name)
 
-        store.count_monthly_users = Mock(return_value=small_number_users)
+        store.count_monthly_users = Mock(return_value=defer.succeed(small_number_users))
 
         self._macaroon_mock_generator("another_secret")
 
@@ -108,12 +108,14 @@ def test_cannot_register_when_mau_limits_exceeded(self):
         yield self.handler.get_or_create_user("@neil:matrix.org", 'c', "Neil")
 
         self._macaroon_mock_generator("another another secret")
-        store.count_monthly_users = Mock(return_value=lots_of_users)
+        store.count_monthly_users = Mock(return_value=defer.succeed(lots_of_users))
+
         with self.assertRaises(RegistrationError):
             yield self.handler.register(localpart=local_part)
 
         self._macaroon_mock_generator("another another secret")
-        store.count_monthly_users = Mock(return_value=lots_of_users)
+        store.count_monthly_users = Mock(return_value=defer.succeed(lots_of_users))
+
         with self.assertRaises(RegistrationError):
             yield self.handler.register_saml2(local_part)
 

From 6023cdd2275d08f25a255a95c1f1aeb2eef34eaf Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 1 Aug 2018 10:27:17 +0100
Subject: [PATCH 203/830] remove errant print

---
 synapse/storage/__init__.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index f9682832cad4..3ce1b34aea0b 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -287,7 +287,6 @@ def _count_monthly_users(txn):
 
             txn.execute(sql, (thirty_days_ago,))
             count, = txn.fetchone()
-            print "Count is %d" % (count,)
             return count
 
         return self.runInteraction("count_monthly_users", _count_monthly_users)

From 6e63d6868c29ed169082c941af04518b7011eda1 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Wed, 1 Aug 2018 10:31:22 +0100
Subject: [PATCH 204/830] Update 2952.bugfix

---
 changelog.d/2952.bugfix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/changelog.d/2952.bugfix b/changelog.d/2952.bugfix
index e5bc8a8762f0..07a3e483048d 100644
--- a/changelog.d/2952.bugfix
+++ b/changelog.d/2952.bugfix
@@ -1 +1 @@
-/directory/list API returns 404 for room not found instead of 400
\ No newline at end of file
+Make /directory/list API return 404 for room not found instead of 400

From 2c54f1c225365cc1da363a270f7f8d201193bc4a Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 1 Aug 2018 11:46:59 +0100
Subject: [PATCH 205/830] remove need to plot limit_usage_by_mau

---
 synapse/app/homeserver.py | 4 ----
 synapse/config/server.py  | 9 ++++++---
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 2da60682ec74..46325d8521f6 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -305,9 +305,6 @@ def run_startup_checks(self, db_conn, database_engine):
 # Gauges to expose monthly active user control metrics
 current_mau_gauge = Gauge("synapse_admin_current_mau", "Current MAU")
 max_mau_value_gauge = Gauge("synapse_admin_max_mau_value", "MAU Limit")
-limit_usage_by_mau_gauge = Gauge(
-    "synapse_admin_limit_usage_by_mau", "MAU Limiting enabled"
-)
 
 
 def setup(config_options):
@@ -528,7 +525,6 @@ def generate_monthly_active_users():
             count = hs.get_datastore().count_monthly_users()
         current_mau_gauge.set(float(count))
         max_mau_value_gauge.set(float(hs.config.max_mau_value))
-        limit_usage_by_mau_gauge.set(float(hs.config.limit_usage_by_mau))
 
     generate_monthly_active_users()
     if hs.config.limit_usage_by_mau:
diff --git a/synapse/config/server.py b/synapse/config/server.py
index 9af42a93ad09..a8014e9c5051 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -69,9 +69,12 @@ def read_config(self, config):
 
         # Options to control access by tracking MAU
         self.limit_usage_by_mau = config.get("limit_usage_by_mau", False)
-        self.max_mau_value = config.get(
-            "max_mau_value", 0,
-        )
+        if self.limit_usage_by_mau:
+            self.max_mau_value = config.get(
+                "max_mau_value", 0,
+            )
+        else:
+            self.max_mau_value = 0
         # FIXME: federation_domain_whitelist needs sytests
         self.federation_domain_whitelist = None
         federation_domain_whitelist = config.get(

From 0aba3d361a88c3d1c5b691e36d162ec6b28132c0 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 1 Aug 2018 11:47:58 +0100
Subject: [PATCH 206/830] count_monthly_users() async

---
 synapse/handlers/auth.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py
index 28f1c1afbb55..efe05d4de098 100644
--- a/synapse/handlers/auth.py
+++ b/synapse/handlers/auth.py
@@ -519,7 +519,7 @@ def get_access_token_for_user_id(self, user_id, device_id=None):
         """
         logger.info("Logging in user %s on device %s", user_id, device_id)
         access_token = yield self.issue_access_token(user_id, device_id)
-        self._check_mau_limits()
+        yield self._check_mau_limits()
 
         # the device *should* have been registered before we got here; however,
         # it's possible we raced against a DELETE operation. The thing we
@@ -729,16 +729,18 @@ def issue_access_token(self, user_id, device_id=None):
                                                   device_id)
         defer.returnValue(access_token)
 
+    @defer.inlineCallbacks
     def validate_short_term_login_token_and_get_user_id(self, login_token):
-        self._check_mau_limits()
+        yield self._check_mau_limits()
         auth_api = self.hs.get_auth()
+        user_id = None
         try:
             macaroon = pymacaroons.Macaroon.deserialize(login_token)
             user_id = auth_api.get_user_id_from_macaroon(macaroon)
             auth_api.validate_macaroon(macaroon, "login", True, user_id)
-            return user_id
         except Exception:
             raise AuthError(403, "Invalid token", errcode=Codes.FORBIDDEN)
+        defer.returnValue(user_id)
 
     @defer.inlineCallbacks
     def delete_access_token(self, access_token):
@@ -894,13 +896,14 @@ def _do_validate_hash():
         else:
             return defer.succeed(False)
 
+    @defer.inlineCallbacks
     def _check_mau_limits(self):
         """
         Ensure that if mau blocking is enabled that invalid users cannot
         log in.
         """
         if self.hs.config.limit_usage_by_mau is True:
-            current_mau = self.store.count_monthly_users()
+            current_mau = yield self.store.count_monthly_users()
             if current_mau >= self.hs.config.max_mau_value:
                 raise AuthError(
                     403, "MAU Limit Exceeded", errcode=Codes.MAU_LIMIT_EXCEEDED

From 4e6e00152c65ca9d1c53a290890dc0149c13e48b Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 1 Aug 2018 11:48:37 +0100
Subject: [PATCH 207/830] fix known broken test

---
 tests/storage/test__init__.py | 45 ++++++++++++++++-------------------
 1 file changed, 20 insertions(+), 25 deletions(-)

diff --git a/tests/storage/test__init__.py b/tests/storage/test__init__.py
index fe6eeeaf1076..acc59e4689f2 100644
--- a/tests/storage/test__init__.py
+++ b/tests/storage/test__init__.py
@@ -15,7 +15,7 @@
 
 from twisted.internet import defer
 
-import tests.unittest
+
 import tests.utils
 
 
@@ -33,39 +33,34 @@ def setUp(self):
         self.store = hs.get_datastore()
         self.clock = hs.get_clock()
 
+    @defer.inlineCallbacks
     def test_count_monthly_users(self):
-        count = self.store.count_monthly_users()
+        count = yield self.store.count_monthly_users()
         self.assertEqual(0, count)
 
-        self._insert_user_ips("@user:server1")
-        self._insert_user_ips("@user:server2")
+        yield self._insert_user_ips("@user:server1")
+        yield self._insert_user_ips("@user:server2")
 
-        count = self.store.count_monthly_users()
+        count = yield self.store.count_monthly_users()
         self.assertEqual(2, count)
 
+    @defer.inlineCallbacks
     def _insert_user_ips(self, user):
         """
         Helper function to populate user_ips without using batch insertion infra
         args:
             user (str):  specify username i.e. @user:server.com
         """
-        try:
-            txn = self.store.db_conn.cursor()
-            self.store.database_engine.lock_table(txn, "user_ips")
-            self.store._simple_upsert_txn(
-                txn,
-                table="user_ips",
-                keyvalues={
-                    "user_id": user,
-                    "access_token": "access_token",
-                    "ip": "ip",
-                    "user_agent": "user_agent",
-                    "device_id": "device_id",
-                },
-                values={
-                    "last_seen": self.clock.time_msec(),
-                },
-                lock=False,
-            )
-        finally:
-            txn.close()
+        yield self.store._simple_upsert(
+            table="user_ips",
+            keyvalues={
+                "user_id": user,
+                "access_token": "access_token",
+                "ip": "ip",
+                "user_agent": "user_agent",
+                "device_id": "device_id",
+            },
+            values={
+                "last_seen": self.clock.time_msec(),
+            }
+        )

From f9f55599718ba9849b2dee18e253dcaf8f5fcfe7 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 1 Aug 2018 12:03:42 +0100
Subject: [PATCH 208/830] fix comment

---
 synapse/storage/schema/delta/51/monthly_active_users.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/storage/schema/delta/51/monthly_active_users.sql b/synapse/storage/schema/delta/51/monthly_active_users.sql
index b3c0e1a67621..f2b6d3e31e4e 100644
--- a/synapse/storage/schema/delta/51/monthly_active_users.sql
+++ b/synapse/storage/schema/delta/51/monthly_active_users.sql
@@ -13,7 +13,7 @@
  * limitations under the License.
  */
 
--- a table of users who have requested that their details be erased
+-- a table of monthly active users, for use where blocking based on mau limits
 CREATE TABLE monthly_active_users (
     user_id TEXT NOT NULL,
     timestamp BIGINT NOT NULL

From 4e5ac901ddd15e8938605c5ac4312be804997ed5 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 1 Aug 2018 12:03:57 +0100
Subject: [PATCH 209/830] clean up

---
 synapse/storage/monthly_active_users.py | 18 +++---------------
 1 file changed, 3 insertions(+), 15 deletions(-)

diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py
index 373e828c0ae7..03eeea792e22 100644
--- a/synapse/storage/monthly_active_users.py
+++ b/synapse/storage/monthly_active_users.py
@@ -31,11 +31,8 @@ def get_monthly_active_count(self):
                 defered resolves to int
         """
         def _count_users(txn):
-            sql = """
-                SELECT COALESCE(count(*), 0) FROM (
-                    SELECT user_id FROM monthly_active_users
-                ) u
-            """
+            sql = "SELECT COALESCE(count(*), 0) FROM monthly_active_users"
+
             txn.execute(sql)
             count, = txn.fetchone()
             return count
@@ -59,9 +56,6 @@ def upsert_monthly_active_user(self, user_id):
             lock=False,
         )
 
-    def clean_out_monthly_active_users(self):
-        pass
-
     @defer.inlineCallbacks
     def is_user_monthly_active(self, user_id):
         """
@@ -79,11 +73,5 @@ def is_user_monthly_active(self, user_id):
             retcol="user_id",
             desc="is_user_monthly_active",
         )
-        # jeff = self.cursor_to_dict(res)
-        result = False
-        if user_present:
-            result = True
 
-        defer.returnValue(
-            result
-        )
+        defer.returnValue(bool(user_present))

From 4b256b92713624018d86f7966d8d2b02122b052c Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 1 Aug 2018 13:39:07 +0100
Subject: [PATCH 210/830] _persist_auth_tree no longer returns anything

---
 synapse/handlers/federation.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 617452be6c40..2e3cbe2aab65 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -400,7 +400,7 @@ def _process_received_pdu(self, origin, pdu, state, auth_chain):
             )
 
             try:
-                event_stream_id, max_stream_id = yield self._persist_auth_tree(
+                yield self._persist_auth_tree(
                     origin, auth_chain, state, event
                 )
             except AuthError as e:

From a6d7b749150d1673117f19c6a134ade8b8c2071b Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 1 Aug 2018 13:39:14 +0100
Subject: [PATCH 211/830] update docs

---
 synapse/handlers/federation.py | 8 +++++---
 synapse/storage/events.py      | 2 +-
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 2e3cbe2aab65..21e1c48eef39 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -1438,6 +1438,8 @@ def _handle_new_events(self, origin, event_infos, backfilled=False):
         should not depend on one another, e.g. this should be used to persist
         a bunch of outliers, but not a chunk of individual events that depend
         on each other for state calculations.
+
+        Notifies about the events where appropriate.
         """
         contexts = yield logcontext.make_deferred_yieldable(defer.gatherResults(
             [
@@ -1464,7 +1466,8 @@ def _handle_new_events(self, origin, event_infos, backfilled=False):
     def _persist_auth_tree(self, origin, auth_events, state, event):
         """Checks the auth chain is valid (and passes auth checks) for the
         state and event. Then persists the auth chain and state atomically.
-        Persists the event seperately.
+        Persists the event separately. Notifies about the persisted events
+        where appropriate.
 
         Will attempt to fetch missing auth events.
 
@@ -1475,8 +1478,7 @@ def _persist_auth_tree(self, origin, auth_events, state, event):
             event (Event)
 
         Returns:
-            2-tuple of (event_stream_id, max_stream_id) from the persist_event
-            call for `event`
+            Deferred
         """
         events_to_context = {}
         for e in itertools.chain(auth_events, state):
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index bbf6e42195f7..ee9159c10284 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -242,7 +242,7 @@ def persist_events(self, events_and_contexts, backfilled=False):
                 which might update the current state etc.
 
         Returns:
-            Deferred[int]: he stream ordering of the latest persisted event
+            Deferred[int]: the stream ordering of the latest persisted event
         """
         partitioned = {}
         for event, ctx in events_and_contexts:

From 6eed16d8a2335c97675b6b8661869848f397ea29 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 1 Aug 2018 14:02:10 +0100
Subject: [PATCH 212/830] fix test for py3

---
 tests/handlers/test_auth.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/handlers/test_auth.py b/tests/handlers/test_auth.py
index 440a45308222..55eab9e9cf9e 100644
--- a/tests/handlers/test_auth.py
+++ b/tests/handlers/test_auth.py
@@ -156,6 +156,10 @@ def test_mau_limits_not_exceeded(self):
         )
         # Ensure does not raise exception
         yield self.auth_handler.get_access_token_for_user_id('user_a')
+
+        self.hs.get_datastore().count_monthly_users = Mock(
+            return_value=defer.succeed(self.small_number_of_users)
+        )
         yield self.auth_handler.validate_short_term_login_token_and_get_user_id(
             self._get_macaroon().serialize()
         )

From c480c4c962eaf9dd979d35930162d00473635fcf Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 1 Aug 2018 14:25:58 +0100
Subject: [PATCH 213/830] fix isort

---
 tests/storage/test__init__.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/storage/test__init__.py b/tests/storage/test__init__.py
index acc59e4689f2..f19cb1265ca0 100644
--- a/tests/storage/test__init__.py
+++ b/tests/storage/test__init__.py
@@ -15,7 +15,6 @@
 
 from twisted.internet import defer
 
-
 import tests.utils
 
 

From da7785147df442eb9cdc1031fa5fea12b7b25334 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Thu, 2 Aug 2018 00:54:06 +1000
Subject: [PATCH 214/830] Python 3: Convert some unicode/bytes uses (#3569)

---
 changelog.d/3569.bugfix                  |  1 +
 synapse/api/auth.py                      |  4 +--
 synapse/federation/transport/server.py   |  2 +-
 synapse/handlers/auth.py                 | 29 ++++++++++++++------
 synapse/handlers/register.py             |  2 +-
 synapse/http/server.py                   | 35 +++++++++++++++++-------
 synapse/http/servlet.py                  | 10 ++++++-
 synapse/rest/client/v1/admin.py          | 22 ++++++++++-----
 synapse/rest/client/v2_alpha/register.py | 12 ++++----
 synapse/rest/media/v1/media_storage.py   |  2 +-
 synapse/state.py                         |  2 +-
 synapse/storage/events.py                | 14 +++++++---
 synapse/storage/signatures.py            |  2 +-
 synapse/types.py                         |  2 +-
 synapse/util/frozenutils.py              |  6 ++--
 tests/api/test_auth.py                   | 35 +++++++++++++-----------
 tests/utils.py                           |  9 ++++--
 17 files changed, 122 insertions(+), 67 deletions(-)
 create mode 100644 changelog.d/3569.bugfix

diff --git a/changelog.d/3569.bugfix b/changelog.d/3569.bugfix
new file mode 100644
index 000000000000..d77f035ee0e1
--- /dev/null
+++ b/changelog.d/3569.bugfix
@@ -0,0 +1 @@
+Unicode passwords are now normalised before hashing, preventing the instance where two different devices or browsers might send a different UTF-8 sequence for the password.
diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 073229b4c4bf..5bbbe8e2e71f 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -252,10 +252,10 @@ def _get_appservice_user_id(self, request):
             if ip_address not in app_service.ip_range_whitelist:
                 defer.returnValue((None, None))
 
-        if "user_id" not in request.args:
+        if b"user_id" not in request.args:
             defer.returnValue((app_service.sender, app_service))
 
-        user_id = request.args["user_id"][0]
+        user_id = request.args[b"user_id"][0].decode('utf8')
         if app_service.sender == user_id:
             defer.returnValue((app_service.sender, app_service))
 
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 3b5ea9515a5b..eae5f2b427bb 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -165,7 +165,7 @@ def _parse_auth_header(header_bytes):
         param_dict = dict(kv.split("=") for kv in params)
 
         def strip_quotes(value):
-            if value.startswith(b"\""):
+            if value.startswith("\""):
                 return value[1:-1]
             else:
                 return value
diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py
index 402e44cdef97..5d03bfa5f74b 100644
--- a/synapse/handlers/auth.py
+++ b/synapse/handlers/auth.py
@@ -15,6 +15,7 @@
 # limitations under the License.
 
 import logging
+import unicodedata
 
 import attr
 import bcrypt
@@ -626,6 +627,7 @@ def validate_login(self, username, login_submission):
         # special case to check for "password" for the check_password interface
         # for the auth providers
         password = login_submission.get("password")
+
         if login_type == LoginType.PASSWORD:
             if not self._password_enabled:
                 raise SynapseError(400, "Password login has been disabled.")
@@ -707,9 +709,10 @@ def _check_local_password(self, user_id, password):
         multiple inexact matches.
 
         Args:
-            user_id (str): complete @user:id
+            user_id (unicode): complete @user:id
+            password (unicode): the provided password
         Returns:
-            (str) the canonical_user_id, or None if unknown user / bad password
+            (unicode) the canonical_user_id, or None if unknown user / bad password
         """
         lookupres = yield self._find_user_id_and_pwd_hash(user_id)
         if not lookupres:
@@ -849,14 +852,19 @@ def hash(self, password):
         """Computes a secure hash of password.
 
         Args:
-            password (str): Password to hash.
+            password (unicode): Password to hash.
 
         Returns:
-            Deferred(str): Hashed password.
+            Deferred(unicode): Hashed password.
         """
         def _do_hash():
-            return bcrypt.hashpw(password.encode('utf8') + self.hs.config.password_pepper,
-                                 bcrypt.gensalt(self.bcrypt_rounds))
+            # Normalise the Unicode in the password
+            pw = unicodedata.normalize("NFKC", password)
+
+            return bcrypt.hashpw(
+                pw.encode('utf8') + self.hs.config.password_pepper.encode("utf8"),
+                bcrypt.gensalt(self.bcrypt_rounds),
+            ).decode('ascii')
 
         return make_deferred_yieldable(
             threads.deferToThreadPool(
@@ -868,16 +876,19 @@ def validate_hash(self, password, stored_hash):
         """Validates that self.hash(password) == stored_hash.
 
         Args:
-            password (str): Password to hash.
-            stored_hash (str): Expected hash value.
+            password (unicode): Password to hash.
+            stored_hash (unicode): Expected hash value.
 
         Returns:
             Deferred(bool): Whether self.hash(password) == stored_hash.
         """
 
         def _do_validate_hash():
+            # Normalise the Unicode in the password
+            pw = unicodedata.normalize("NFKC", password)
+
             return bcrypt.checkpw(
-                password.encode('utf8') + self.hs.config.password_pepper,
+                pw.encode('utf8') + self.hs.config.password_pepper.encode("utf8"),
                 stored_hash.encode('utf8')
             )
 
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index 7caff0cbc8d0..234f8e801939 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -131,7 +131,7 @@ def register(
         Args:
             localpart : The local part of the user ID to register. If None,
               one will be generated.
-            password (str) : The password to assign to this user so they can
+            password (unicode) : The password to assign to this user so they can
               login again. This can be None which means they cannot login again
               via a password (e.g. the user is an application service user).
             generate_token (bool): Whether a new access token should be
diff --git a/synapse/http/server.py b/synapse/http/server.py
index c70fdbdfd2f3..1940c1c4f4cb 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -13,12 +13,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 import cgi
 import collections
 import logging
-import urllib
 
-from six.moves import http_client
+from six import PY3
+from six.moves import http_client, urllib
 
 from canonicaljson import encode_canonical_json, encode_pretty_printed_json, json
 
@@ -264,6 +265,7 @@ def __init__(self, hs, canonical_json=True):
         self.hs = hs
 
     def register_paths(self, method, path_patterns, callback):
+        method = method.encode("utf-8")  # method is bytes on py3
         for path_pattern in path_patterns:
             logger.debug("Registering for %s %s", method, path_pattern.pattern)
             self.path_regexs.setdefault(method, []).append(
@@ -296,8 +298,19 @@ def _async_render(self, request):
         # here. If it throws an exception, that is handled by the wrapper
         # installed by @request_handler.
 
+        def _unquote(s):
+            if PY3:
+                # On Python 3, unquote is unicode -> unicode
+                return urllib.parse.unquote(s)
+            else:
+                # On Python 2, unquote is bytes -> bytes We need to encode the
+                # URL again (as it was decoded by _get_handler_for request), as
+                # ASCII because it's a URL, and then decode it to get the UTF-8
+                # characters that were quoted.
+                return urllib.parse.unquote(s.encode('ascii')).decode('utf8')
+
         kwargs = intern_dict({
-            name: urllib.unquote(value).decode("UTF-8") if value else value
+            name: _unquote(value) if value else value
             for name, value in group_dict.items()
         })
 
@@ -313,9 +326,9 @@ def _get_handler_for_request(self, request):
             request (twisted.web.http.Request):
 
         Returns:
-            Tuple[Callable, dict[str, str]]: callback method, and the dict
-                mapping keys to path components as specified in the handler's
-                path match regexp.
+            Tuple[Callable, dict[unicode, unicode]]: callback method, and the
+                dict mapping keys to path components as specified in the
+                handler's path match regexp.
 
                 The callback will normally be a method registered via
                 register_paths, so will return (possibly via Deferred) either
@@ -327,7 +340,7 @@ def _get_handler_for_request(self, request):
         # Loop through all the registered callbacks to check if the method
         # and path regex match
         for path_entry in self.path_regexs.get(request.method, []):
-            m = path_entry.pattern.match(request.path)
+            m = path_entry.pattern.match(request.path.decode('ascii'))
             if m:
                 # We found a match!
                 return path_entry.callback, m.groupdict()
@@ -383,7 +396,7 @@ def __init__(self, path):
         self.url = path
 
     def render_GET(self, request):
-        return redirectTo(self.url, request)
+        return redirectTo(self.url.encode('ascii'), request)
 
     def getChild(self, name, request):
         if len(name) == 0:
@@ -404,12 +417,14 @@ def respond_with_json(request, code, json_object, send_cors=False,
         return
 
     if pretty_print:
-        json_bytes = encode_pretty_printed_json(json_object) + "\n"
+        json_bytes = (encode_pretty_printed_json(json_object) + "\n"
+                      ).encode("utf-8")
     else:
         if canonical_json or synapse.events.USE_FROZEN_DICTS:
+            # canonicaljson already encodes to bytes
             json_bytes = encode_canonical_json(json_object)
         else:
-            json_bytes = json.dumps(json_object)
+            json_bytes = json.dumps(json_object).encode("utf-8")
 
     return respond_with_json_bytes(
         request, code, json_bytes,
diff --git a/synapse/http/servlet.py b/synapse/http/servlet.py
index 882816dc8f09..69f70852911c 100644
--- a/synapse/http/servlet.py
+++ b/synapse/http/servlet.py
@@ -171,8 +171,16 @@ def parse_json_value_from_request(request, allow_empty_body=False):
     if not content_bytes and allow_empty_body:
         return None
 
+    # Decode to Unicode so that simplejson will return Unicode strings on
+    # Python 2
     try:
-        content = json.loads(content_bytes)
+        content_unicode = content_bytes.decode('utf8')
+    except UnicodeDecodeError:
+        logger.warn("Unable to decode UTF-8")
+        raise SynapseError(400, "Content not JSON.", errcode=Codes.NOT_JSON)
+
+    try:
+        content = json.loads(content_unicode)
     except Exception as e:
         logger.warn("Unable to parse JSON: %s", e)
         raise SynapseError(400, "Content not JSON.", errcode=Codes.NOT_JSON)
diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py
index 99f6c6e3c389..80d625eecce6 100644
--- a/synapse/rest/client/v1/admin.py
+++ b/synapse/rest/client/v1/admin.py
@@ -18,6 +18,7 @@
 import hmac
 import logging
 
+from six import text_type
 from six.moves import http_client
 
 from twisted.internet import defer
@@ -131,7 +132,10 @@ def on_POST(self, request):
                 400, "username must be specified", errcode=Codes.BAD_JSON,
             )
         else:
-            if (not isinstance(body['username'], str) or len(body['username']) > 512):
+            if (
+                not isinstance(body['username'], text_type)
+                or len(body['username']) > 512
+            ):
                 raise SynapseError(400, "Invalid username")
 
             username = body["username"].encode("utf-8")
@@ -143,7 +147,10 @@ def on_POST(self, request):
                 400, "password must be specified", errcode=Codes.BAD_JSON,
             )
         else:
-            if (not isinstance(body['password'], str) or len(body['password']) > 512):
+            if (
+                not isinstance(body['password'], text_type)
+                or len(body['password']) > 512
+            ):
                 raise SynapseError(400, "Invalid password")
 
             password = body["password"].encode("utf-8")
@@ -166,17 +173,18 @@ def on_POST(self, request):
         want_mac.update(b"admin" if admin else b"notadmin")
         want_mac = want_mac.hexdigest()
 
-        if not hmac.compare_digest(want_mac, got_mac):
-            raise SynapseError(
-                403, "HMAC incorrect",
-            )
+        if not hmac.compare_digest(want_mac, got_mac.encode('ascii')):
+            raise SynapseError(403, "HMAC incorrect")
 
         # Reuse the parts of RegisterRestServlet to reduce code duplication
         from synapse.rest.client.v2_alpha.register import RegisterRestServlet
+
         register = RegisterRestServlet(self.hs)
 
         (user_id, _) = yield register.registration_handler.register(
-            localpart=username.lower(), password=password, admin=bool(admin),
+            localpart=body['username'].lower(),
+            password=body["password"],
+            admin=bool(admin),
             generate_token=False,
         )
 
diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index d6cf915d860c..2f64155d1316 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -193,15 +193,15 @@ def __init__(self, hs):
     def on_POST(self, request):
         body = parse_json_object_from_request(request)
 
-        kind = "user"
-        if "kind" in request.args:
-            kind = request.args["kind"][0]
+        kind = b"user"
+        if b"kind" in request.args:
+            kind = request.args[b"kind"][0]
 
-        if kind == "guest":
+        if kind == b"guest":
             ret = yield self._do_guest_registration(body)
             defer.returnValue(ret)
             return
-        elif kind != "user":
+        elif kind != b"user":
             raise UnrecognizedRequestError(
                 "Do not understand membership kind: %s" % (kind,)
             )
@@ -389,8 +389,8 @@ def on_POST(self, request):
             assert_params_in_dict(params, ["password"])
 
             desired_username = params.get("username", None)
-            new_password = params.get("password", None)
             guest_access_token = params.get("guest_access_token", None)
+            new_password = params.get("password", None)
 
             if desired_username is not None:
                 desired_username = desired_username.lower()
diff --git a/synapse/rest/media/v1/media_storage.py b/synapse/rest/media/v1/media_storage.py
index b25993fcb5b8..a6189224eeed 100644
--- a/synapse/rest/media/v1/media_storage.py
+++ b/synapse/rest/media/v1/media_storage.py
@@ -177,7 +177,7 @@ def ensure_media_is_in_local_cache(self, file_info):
             if res:
                 with res:
                     consumer = BackgroundFileConsumer(
-                        open(local_path, "w"), self.hs.get_reactor())
+                        open(local_path, "wb"), self.hs.get_reactor())
                     yield res.write_to_consumer(consumer)
                     yield consumer.wait()
                 defer.returnValue(local_path)
diff --git a/synapse/state.py b/synapse/state.py
index 033f55d967f1..e1092b97a9b4 100644
--- a/synapse/state.py
+++ b/synapse/state.py
@@ -577,7 +577,7 @@ def _make_state_cache_entry(
 
 def _ordered_events(events):
     def key_func(e):
-        return -int(e.depth), hashlib.sha1(e.event_id.encode()).hexdigest()
+        return -int(e.depth), hashlib.sha1(e.event_id.encode('ascii')).hexdigest()
 
     return sorted(events, key=key_func)
 
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index c98e524ba1c6..61223da1a57e 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -67,7 +67,13 @@
 
 
 def encode_json(json_object):
-    return frozendict_json_encoder.encode(json_object)
+    """
+    Encode a Python object as JSON and return it in a Unicode string.
+    """
+    out = frozendict_json_encoder.encode(json_object)
+    if isinstance(out, bytes):
+        out = out.decode('utf8')
+    return out
 
 
 class _EventPeristenceQueue(object):
@@ -1058,7 +1064,7 @@ def _update_outliers_txn(self, txn, events_and_contexts):
 
                 metadata_json = encode_json(
                     event.internal_metadata.get_dict()
-                ).decode("UTF-8")
+                )
 
                 sql = (
                     "UPDATE event_json SET internal_metadata = ?"
@@ -1172,8 +1178,8 @@ def event_dict(event):
                     "room_id": event.room_id,
                     "internal_metadata": encode_json(
                         event.internal_metadata.get_dict()
-                    ).decode("UTF-8"),
-                    "json": encode_json(event_dict(event)).decode("UTF-8"),
+                    ),
+                    "json": encode_json(event_dict(event)),
                 }
                 for event, _ in events_and_contexts
             ],
diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py
index 470212aa2ab4..5623391f6eb2 100644
--- a/synapse/storage/signatures.py
+++ b/synapse/storage/signatures.py
@@ -74,7 +74,7 @@ def _get_event_reference_hashes_txn(self, txn, event_id):
             txn (cursor):
             event_id (str): Id for the Event.
         Returns:
-            A dict of algorithm -> hash.
+            A dict[unicode, bytes] of algorithm -> hash.
         """
         query = (
             "SELECT algorithm, hash"
diff --git a/synapse/types.py b/synapse/types.py
index 08f058f71444..41afb27a746d 100644
--- a/synapse/types.py
+++ b/synapse/types.py
@@ -137,7 +137,7 @@ def __deepcopy__(self, memo):
     @classmethod
     def from_string(cls, s):
         """Parse the string given by 's' into a structure object."""
-        if len(s) < 1 or s[0] != cls.SIGIL:
+        if len(s) < 1 or s[0:1] != cls.SIGIL:
             raise SynapseError(400, "Expected %s string to start with '%s'" % (
                 cls.__name__, cls.SIGIL,
             ))
diff --git a/synapse/util/frozenutils.py b/synapse/util/frozenutils.py
index 581c6052ac18..014edea971a7 100644
--- a/synapse/util/frozenutils.py
+++ b/synapse/util/frozenutils.py
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from six import string_types
+from six import binary_type, text_type
 
 from canonicaljson import json
 from frozendict import frozendict
@@ -26,7 +26,7 @@ def freeze(o):
     if isinstance(o, frozendict):
         return o
 
-    if isinstance(o, string_types):
+    if isinstance(o, (binary_type, text_type)):
         return o
 
     try:
@@ -41,7 +41,7 @@ def unfreeze(o):
     if isinstance(o, (dict, frozendict)):
         return dict({k: unfreeze(v) for k, v in o.items()})
 
-    if isinstance(o, string_types):
+    if isinstance(o, (binary_type, text_type)):
         return o
 
     try:
diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py
index 5f158ec4b92f..a82d737e71da 100644
--- a/tests/api/test_auth.py
+++ b/tests/api/test_auth.py
@@ -46,7 +46,7 @@ def setUp(self):
         self.auth = Auth(self.hs)
 
         self.test_user = "@foo:bar"
-        self.test_token = "_test_token_"
+        self.test_token = b"_test_token_"
 
         # this is overridden for the appservice tests
         self.store.get_app_service_by_token = Mock(return_value=None)
@@ -61,7 +61,7 @@ def test_get_user_by_req_user_valid_token(self):
         self.store.get_user_by_access_token = Mock(return_value=user_info)
 
         request = Mock(args={})
-        request.args["access_token"] = [self.test_token]
+        request.args[b"access_token"] = [self.test_token]
         request.requestHeaders.getRawHeaders = mock_getRawHeaders()
         requester = yield self.auth.get_user_by_req(request)
         self.assertEquals(requester.user.to_string(), self.test_user)
@@ -70,7 +70,7 @@ def test_get_user_by_req_user_bad_token(self):
         self.store.get_user_by_access_token = Mock(return_value=None)
 
         request = Mock(args={})
-        request.args["access_token"] = [self.test_token]
+        request.args[b"access_token"] = [self.test_token]
         request.requestHeaders.getRawHeaders = mock_getRawHeaders()
         d = self.auth.get_user_by_req(request)
         self.failureResultOf(d, AuthError)
@@ -98,7 +98,7 @@ def test_get_user_by_req_appservice_valid_token(self):
 
         request = Mock(args={})
         request.getClientIP.return_value = "127.0.0.1"
-        request.args["access_token"] = [self.test_token]
+        request.args[b"access_token"] = [self.test_token]
         request.requestHeaders.getRawHeaders = mock_getRawHeaders()
         requester = yield self.auth.get_user_by_req(request)
         self.assertEquals(requester.user.to_string(), self.test_user)
@@ -115,7 +115,7 @@ def test_get_user_by_req_appservice_valid_token_good_ip(self):
 
         request = Mock(args={})
         request.getClientIP.return_value = "192.168.10.10"
-        request.args["access_token"] = [self.test_token]
+        request.args[b"access_token"] = [self.test_token]
         request.requestHeaders.getRawHeaders = mock_getRawHeaders()
         requester = yield self.auth.get_user_by_req(request)
         self.assertEquals(requester.user.to_string(), self.test_user)
@@ -131,7 +131,7 @@ def test_get_user_by_req_appservice_valid_token_bad_ip(self):
 
         request = Mock(args={})
         request.getClientIP.return_value = "131.111.8.42"
-        request.args["access_token"] = [self.test_token]
+        request.args[b"access_token"] = [self.test_token]
         request.requestHeaders.getRawHeaders = mock_getRawHeaders()
         d = self.auth.get_user_by_req(request)
         self.failureResultOf(d, AuthError)
@@ -141,7 +141,7 @@ def test_get_user_by_req_appservice_bad_token(self):
         self.store.get_user_by_access_token = Mock(return_value=None)
 
         request = Mock(args={})
-        request.args["access_token"] = [self.test_token]
+        request.args[b"access_token"] = [self.test_token]
         request.requestHeaders.getRawHeaders = mock_getRawHeaders()
         d = self.auth.get_user_by_req(request)
         self.failureResultOf(d, AuthError)
@@ -158,7 +158,7 @@ def test_get_user_by_req_appservice_missing_token(self):
 
     @defer.inlineCallbacks
     def test_get_user_by_req_appservice_valid_token_valid_user_id(self):
-        masquerading_user_id = "@doppelganger:matrix.org"
+        masquerading_user_id = b"@doppelganger:matrix.org"
         app_service = Mock(
             token="foobar", url="a_url", sender=self.test_user,
             ip_range_whitelist=None,
@@ -169,14 +169,17 @@ def test_get_user_by_req_appservice_valid_token_valid_user_id(self):
 
         request = Mock(args={})
         request.getClientIP.return_value = "127.0.0.1"
-        request.args["access_token"] = [self.test_token]
-        request.args["user_id"] = [masquerading_user_id]
+        request.args[b"access_token"] = [self.test_token]
+        request.args[b"user_id"] = [masquerading_user_id]
         request.requestHeaders.getRawHeaders = mock_getRawHeaders()
         requester = yield self.auth.get_user_by_req(request)
-        self.assertEquals(requester.user.to_string(), masquerading_user_id)
+        self.assertEquals(
+            requester.user.to_string(),
+            masquerading_user_id.decode('utf8')
+        )
 
     def test_get_user_by_req_appservice_valid_token_bad_user_id(self):
-        masquerading_user_id = "@doppelganger:matrix.org"
+        masquerading_user_id = b"@doppelganger:matrix.org"
         app_service = Mock(
             token="foobar", url="a_url", sender=self.test_user,
             ip_range_whitelist=None,
@@ -187,8 +190,8 @@ def test_get_user_by_req_appservice_valid_token_bad_user_id(self):
 
         request = Mock(args={})
         request.getClientIP.return_value = "127.0.0.1"
-        request.args["access_token"] = [self.test_token]
-        request.args["user_id"] = [masquerading_user_id]
+        request.args[b"access_token"] = [self.test_token]
+        request.args[b"user_id"] = [masquerading_user_id]
         request.requestHeaders.getRawHeaders = mock_getRawHeaders()
         d = self.auth.get_user_by_req(request)
         self.failureResultOf(d, AuthError)
@@ -418,7 +421,7 @@ def get_user(tok):
 
         # check the token works
         request = Mock(args={})
-        request.args["access_token"] = [token]
+        request.args[b"access_token"] = [token.encode('ascii')]
         request.requestHeaders.getRawHeaders = mock_getRawHeaders()
         requester = yield self.auth.get_user_by_req(request, allow_guest=True)
         self.assertEqual(UserID.from_string(USER_ID), requester.user)
@@ -431,7 +434,7 @@ def get_user(tok):
 
         # the token should *not* work now
         request = Mock(args={})
-        request.args["access_token"] = [guest_tok]
+        request.args[b"access_token"] = [guest_tok.encode('ascii')]
         request.requestHeaders.getRawHeaders = mock_getRawHeaders()
 
         with self.assertRaises(AuthError) as cm:
diff --git a/tests/utils.py b/tests/utils.py
index c3dbff85073c..9bff3ff3b9b5 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -193,7 +193,7 @@ def __init__(self, prefix=""):
         self.prefix = prefix
 
     def trigger_get(self, path):
-        return self.trigger("GET", path, None)
+        return self.trigger(b"GET", path, None)
 
     @patch('twisted.web.http.Request')
     @defer.inlineCallbacks
@@ -227,7 +227,7 @@ def trigger(self, http_method, path, content, mock_request, federation_auth=Fals
 
         headers = {}
         if federation_auth:
-            headers[b"Authorization"] = ["X-Matrix origin=test,key=,sig="]
+            headers[b"Authorization"] = [b"X-Matrix origin=test,key=,sig="]
         mock_request.requestHeaders.getRawHeaders = mock_getRawHeaders(headers)
 
         # return the right path if the event requires it
@@ -241,6 +241,9 @@ def trigger(self, http_method, path, content, mock_request, federation_auth=Fals
         except Exception:
             pass
 
+        if isinstance(path, bytes):
+            path = path.decode('utf8')
+
         for (method, pattern, func) in self.callbacks:
             if http_method != method:
                 continue
@@ -249,7 +252,7 @@ def trigger(self, http_method, path, content, mock_request, federation_auth=Fals
             if matcher:
                 try:
                     args = [
-                        urlparse.unquote(u).decode("UTF-8")
+                        urlparse.unquote(u)
                         for u in matcher.groups()
                     ]
 

From c82ccd3027dd9599ac0214adfd351996cf658681 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 1 Aug 2018 11:24:19 +0100
Subject: [PATCH 215/830] Factor out exception handling in federation_client

Factor out the error handling from make_membership_event, send_join, and
send_leave, so that it can be shared.
---
 synapse/federation/federation_client.py | 277 +++++++++++++-----------
 1 file changed, 148 insertions(+), 129 deletions(-)

diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index 62d7ed13cf97..baa9c3586fe8 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -48,6 +48,13 @@
 PDU_RETRY_TIME_MS = 1 * 60 * 1000
 
 
+class InvalidResponseError(RuntimeError):
+    """Helper for _try_destination_list: indicates that the server returned a response
+    we couldn't parse
+    """
+    pass
+
+
 class FederationClient(FederationBase):
     def __init__(self, hs):
         super(FederationClient, self).__init__(hs)
@@ -458,6 +465,61 @@ def get_event_auth(self, destination, room_id, event_id):
         defer.returnValue(signed_auth)
 
     @defer.inlineCallbacks
+    def _try_destination_list(self, description, destinations, callback):
+        """Try an operation on a series of servers, until it succeeds
+
+        Args:
+            description (unicode): description of the operation we're doing, for logging
+
+            destinations (Iterable[unicode]): list of server_names to try
+
+            callback (callable):  Function to run for each server. Passed a single
+                argument: the server_name to try. May return a deferred.
+
+                If the callback raises a CodeMessageException with a 300/400 code,
+                attempts to perform the operation stop immediately and the exception is
+                reraised.
+
+                Otherwise, if the callback raises an Exception the error is logged and the
+                next server tried. Normally the stacktrace is logged but this is
+                suppressed if the exception is an InvalidResponseError.
+
+        Returns:
+            The [Deferred] result of callback, if it succeeds
+
+        Raises:
+            CodeMessageException if the chosen remote server returns a 300/400 code.
+
+            RuntimeError if no servers were reachable.
+        """
+        for destination in destinations:
+            if destination == self.server_name:
+                continue
+
+            try:
+                res = yield callback(destination)
+                defer.returnValue(res)
+            except InvalidResponseError as e:
+                logger.warn(
+                    "Failed to %s via %s: %s",
+                    description, destination, e,
+                )
+            except CodeMessageException as e:
+                if not 500 <= e.code < 600:
+                    raise
+                else:
+                    logger.warn(
+                        "Failed to %s via %s: %i %s",
+                        description, destination, e.code, e.message,
+                    )
+            except Exception:
+                logger.warn(
+                    "Failed to %s via %s",
+                    description, destination, exc_info=1,
+                )
+
+        raise RuntimeError("Failed to %s via any server", description)
+
     def make_membership_event(self, destinations, room_id, user_id, membership,
                               content={},):
         """
@@ -492,50 +554,35 @@ def make_membership_event(self, destinations, room_id, user_id, membership,
                 "make_membership_event called with membership='%s', must be one of %s" %
                 (membership, ",".join(valid_memberships))
             )
-        for destination in destinations:
-            if destination == self.server_name:
-                continue
 
-            try:
-                ret = yield self.transport_layer.make_membership_event(
-                    destination, room_id, user_id, membership
-                )
+        @defer.inlineCallbacks
+        def send_request(destination):
+            ret = yield self.transport_layer.make_membership_event(
+                destination, room_id, user_id, membership
+            )
 
-                pdu_dict = ret["event"]
+            pdu_dict = ret["event"]
 
-                logger.debug("Got response to make_%s: %s", membership, pdu_dict)
+            logger.debug("Got response to make_%s: %s", membership, pdu_dict)
 
-                pdu_dict["content"].update(content)
+            pdu_dict["content"].update(content)
 
-                # The protoevent received over the JSON wire may not have all
-                # the required fields. Lets just gloss over that because
-                # there's some we never care about
-                if "prev_state" not in pdu_dict:
-                    pdu_dict["prev_state"] = []
+            # The protoevent received over the JSON wire may not have all
+            # the required fields. Lets just gloss over that because
+            # there's some we never care about
+            if "prev_state" not in pdu_dict:
+                pdu_dict["prev_state"] = []
 
-                ev = builder.EventBuilder(pdu_dict)
+            ev = builder.EventBuilder(pdu_dict)
 
-                defer.returnValue(
-                    (destination, ev)
-                )
-                break
-            except CodeMessageException as e:
-                if not 500 <= e.code < 600:
-                    raise
-                else:
-                    logger.warn(
-                        "Failed to make_%s via %s: %s",
-                        membership, destination, e.message
-                    )
-            except Exception as e:
-                logger.warn(
-                    "Failed to make_%s via %s: %s",
-                    membership, destination, e.message
-                )
+            defer.returnValue(
+                (destination, ev)
+            )
 
-        raise RuntimeError("Failed to send to any server.")
+        return self._try_destination_list(
+            "make_" + membership, destinations, send_request,
+        )
 
-    @defer.inlineCallbacks
     def send_join(self, destinations, pdu):
         """Sends a join event to one of a list of homeservers.
 
@@ -558,87 +605,70 @@ def send_join(self, destinations, pdu):
             Fails with a ``RuntimeError`` if no servers were reachable.
         """
 
-        for destination in destinations:
-            if destination == self.server_name:
-                continue
-
-            try:
-                time_now = self._clock.time_msec()
-                _, content = yield self.transport_layer.send_join(
-                    destination=destination,
-                    room_id=pdu.room_id,
-                    event_id=pdu.event_id,
-                    content=pdu.get_pdu_json(time_now),
-                )
+        @defer.inlineCallbacks
+        def send_request(destination):
+            time_now = self._clock.time_msec()
+            _, content = yield self.transport_layer.send_join(
+                destination=destination,
+                room_id=pdu.room_id,
+                event_id=pdu.event_id,
+                content=pdu.get_pdu_json(time_now),
+            )
 
-                logger.debug("Got content: %s", content)
+            logger.debug("Got content: %s", content)
 
-                state = [
-                    event_from_pdu_json(p, outlier=True)
-                    for p in content.get("state", [])
-                ]
+            state = [
+                event_from_pdu_json(p, outlier=True)
+                for p in content.get("state", [])
+            ]
 
-                auth_chain = [
-                    event_from_pdu_json(p, outlier=True)
-                    for p in content.get("auth_chain", [])
-                ]
+            auth_chain = [
+                event_from_pdu_json(p, outlier=True)
+                for p in content.get("auth_chain", [])
+            ]
 
-                pdus = {
-                    p.event_id: p
-                    for p in itertools.chain(state, auth_chain)
-                }
+            pdus = {
+                p.event_id: p
+                for p in itertools.chain(state, auth_chain)
+            }
 
-                valid_pdus = yield self._check_sigs_and_hash_and_fetch(
-                    destination, list(pdus.values()),
-                    outlier=True,
-                )
+            valid_pdus = yield self._check_sigs_and_hash_and_fetch(
+                destination, list(pdus.values()),
+                outlier=True,
+            )
 
-                valid_pdus_map = {
-                    p.event_id: p
-                    for p in valid_pdus
-                }
-
-                # NB: We *need* to copy to ensure that we don't have multiple
-                # references being passed on, as that causes... issues.
-                signed_state = [
-                    copy.copy(valid_pdus_map[p.event_id])
-                    for p in state
-                    if p.event_id in valid_pdus_map
-                ]
+            valid_pdus_map = {
+                p.event_id: p
+                for p in valid_pdus
+            }
 
-                signed_auth = [
-                    valid_pdus_map[p.event_id]
-                    for p in auth_chain
-                    if p.event_id in valid_pdus_map
-                ]
+            # NB: We *need* to copy to ensure that we don't have multiple
+            # references being passed on, as that causes... issues.
+            signed_state = [
+                copy.copy(valid_pdus_map[p.event_id])
+                for p in state
+                if p.event_id in valid_pdus_map
+            ]
 
-                # NB: We *need* to copy to ensure that we don't have multiple
-                # references being passed on, as that causes... issues.
-                for s in signed_state:
-                    s.internal_metadata = copy.deepcopy(s.internal_metadata)
+            signed_auth = [
+                valid_pdus_map[p.event_id]
+                for p in auth_chain
+                if p.event_id in valid_pdus_map
+            ]
 
-                auth_chain.sort(key=lambda e: e.depth)
+            # NB: We *need* to copy to ensure that we don't have multiple
+            # references being passed on, as that causes... issues.
+            for s in signed_state:
+                s.internal_metadata = copy.deepcopy(s.internal_metadata)
 
-                defer.returnValue({
-                    "state": signed_state,
-                    "auth_chain": signed_auth,
-                    "origin": destination,
-                })
-            except CodeMessageException as e:
-                if not 500 <= e.code < 600:
-                    raise
-                else:
-                    logger.exception(
-                        "Failed to send_join via %s: %s",
-                        destination, e.message
-                    )
-            except Exception as e:
-                logger.exception(
-                    "Failed to send_join via %s: %s",
-                    destination, e.message
-                )
+            auth_chain.sort(key=lambda e: e.depth)
 
-        raise RuntimeError("Failed to send to any server.")
+            defer.returnValue({
+                "state": signed_state,
+                "auth_chain": signed_auth,
+                "origin": destination,
+            })
+        return self._try_destination_list("send_join", destinations, send_request)
 
     @defer.inlineCallbacks
     def send_invite(self, destination, room_id, event_id, pdu):
@@ -663,7 +693,6 @@ def send_invite(self, destination, room_id, event_id, pdu):
 
         defer.returnValue(pdu)
 
-    @defer.inlineCallbacks
     def send_leave(self, destinations, pdu):
         """Sends a leave event to one of a list of homeservers.
 
@@ -681,34 +710,24 @@ def send_leave(self, destinations, pdu):
             Deferred: resolves to None.
 
             Fails with a ``CodeMessageException`` if the chosen remote server
-            returns a non-200 code.
+            returns a 300/400 code.
 
             Fails with a ``RuntimeError`` if no servers were reachable.
         """
-        for destination in destinations:
-            if destination == self.server_name:
-                continue
-
-            try:
-                time_now = self._clock.time_msec()
-                _, content = yield self.transport_layer.send_leave(
-                    destination=destination,
-                    room_id=pdu.room_id,
-                    event_id=pdu.event_id,
-                    content=pdu.get_pdu_json(time_now),
-                )
+        @defer.inlineCallbacks
+        def send_request(destination):
+            time_now = self._clock.time_msec()
+            _, content = yield self.transport_layer.send_leave(
+                destination=destination,
+                room_id=pdu.room_id,
+                event_id=pdu.event_id,
+                content=pdu.get_pdu_json(time_now),
+            )
 
-                logger.debug("Got content: %s", content)
-                defer.returnValue(None)
-            except CodeMessageException:
-                raise
-            except Exception as e:
-                logger.exception(
-                    "Failed to send_leave via %s: %s",
-                    destination, e.message
-                )
+            logger.debug("Got content: %s", content)
+            defer.returnValue(None)
 
-        raise RuntimeError("Failed to send to any server.")
+        return self._try_destination_list("send_leave", destinations, send_request)
 
     def get_public_rooms(self, destination, limit=None, since_token=None,
                          search_filter=None, include_all_networks=False,

From fa7dc889f19f581e81624245ce7820525066eff3 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 1 Aug 2018 13:47:07 +0100
Subject: [PATCH 216/830] Be more careful which errors we send back over the
 C-S API

We really shouldn't be sending all CodeMessageExceptions back over the C-S API;
it will include things like 401s which we shouldn't proxy.

That means that we need to explicitly turn a few HttpResponseExceptions into
SynapseErrors in the federation layer.

The effect of the latter is that the matrix errcode will get passed through
correctly to calling clients, which might help with some of the random
M_UNKNOWN errors when trying to join rooms.
---
 synapse/api/errors.py                   | 11 ----------
 synapse/federation/federation_client.py | 29 +++++++++++++++----------
 synapse/http/server.py                  | 14 +++++-------
 3 files changed, 22 insertions(+), 32 deletions(-)

diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index 6074df292f55..cf48829c8bce 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -69,9 +69,6 @@ def __init__(self, code, msg):
         self.code = code
         self.msg = msg
 
-    def error_dict(self):
-        return cs_error(self.msg)
-
 
 class MatrixCodeMessageException(CodeMessageException):
     """An error from a general matrix endpoint, eg. from a proxied Matrix API call.
@@ -308,14 +305,6 @@ def error_dict(self):
         )
 
 
-def cs_exception(exception):
-    if isinstance(exception, CodeMessageException):
-        return exception.error_dict()
-    else:
-        logger.error("Unknown exception type: %s", type(exception))
-        return {}
-
-
 def cs_error(msg, code=Codes.UNKNOWN, **kwargs):
     """ Utility method for constructing an error response for client-server
     interactions.
diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index baa9c3586fe8..0b09f93ca928 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -488,7 +488,7 @@ def _try_destination_list(self, description, destinations, callback):
             The [Deferred] result of callback, if it succeeds
 
         Raises:
-            CodeMessageException if the chosen remote server returns a 300/400 code.
+            SynapseError if the chosen remote server returns a 300/400 code.
 
             RuntimeError if no servers were reachable.
         """
@@ -504,9 +504,9 @@ def _try_destination_list(self, description, destinations, callback):
                     "Failed to %s via %s: %s",
                     description, destination, e,
                 )
-            except CodeMessageException as e:
+            except HttpResponseException as e:
                 if not 500 <= e.code < 600:
-                    raise
+                    raise SynapseError.from_http_response_exception(e)
                 else:
                     logger.warn(
                         "Failed to %s via %s: %i %s",
@@ -543,7 +543,7 @@ def make_membership_event(self, destinations, room_id, user_id, membership,
             Deferred: resolves to a tuple of (origin (str), event (object))
             where origin is the remote homeserver which generated the event.
 
-            Fails with a ``CodeMessageException`` if the chosen remote server
+            Fails with a ``SynapseError`` if the chosen remote server
             returns a 300/400 code.
 
             Fails with a ``RuntimeError`` if no servers were reachable.
@@ -599,7 +599,7 @@ def send_join(self, destinations, pdu):
             giving the serer the event was sent to, ``state`` (?) and
             ``auth_chain``.
 
-            Fails with a ``CodeMessageException`` if the chosen remote server
+            Fails with a ``SynapseError`` if the chosen remote server
             returns a 300/400 code.
 
             Fails with a ``RuntimeError`` if no servers were reachable.
@@ -673,12 +673,17 @@ def send_request(destination):
     @defer.inlineCallbacks
     def send_invite(self, destination, room_id, event_id, pdu):
         time_now = self._clock.time_msec()
-        code, content = yield self.transport_layer.send_invite(
-            destination=destination,
-            room_id=room_id,
-            event_id=event_id,
-            content=pdu.get_pdu_json(time_now),
-        )
+        try:
+            code, content = yield self.transport_layer.send_invite(
+                destination=destination,
+                room_id=room_id,
+                event_id=event_id,
+                content=pdu.get_pdu_json(time_now),
+            )
+        except HttpResponseException as e:
+            if e.code == 403:
+                raise SynapseError.from_http_response_exception(e)
+            raise
 
         pdu_dict = content["event"]
 
@@ -709,7 +714,7 @@ def send_leave(self, destinations, pdu):
         Return:
             Deferred: resolves to None.
 
-            Fails with a ``CodeMessageException`` if the chosen remote server
+            Fails with a ``SynapseError`` if the chosen remote server
             returns a 300/400 code.
 
             Fails with a ``RuntimeError`` if no servers were reachable.
diff --git a/synapse/http/server.py b/synapse/http/server.py
index 1940c1c4f4cb..6dacb31037aa 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -36,7 +36,6 @@
     Codes,
     SynapseError,
     UnrecognizedRequestError,
-    cs_exception,
 )
 from synapse.http.request_metrics import requests_counter
 from synapse.util.caches import intern_dict
@@ -77,16 +76,13 @@ def wrap_json_request_handler(h):
     def wrapped_request_handler(self, request):
         try:
             yield h(self, request)
-        except CodeMessageException as e:
+        except SynapseError as e:
             code = e.code
-            if isinstance(e, SynapseError):
-                logger.info(
-                    "%s SynapseError: %s - %s", request, code, e.msg
-                )
-            else:
-                logger.exception(e)
+            logger.info(
+                "%s SynapseError: %s - %s", request, code, e.msg
+            )
             respond_with_json(
-                request, code, cs_exception(e), send_cors=True,
+                request, code, e.error_dict(), send_cors=True,
                 pretty_print=_request_user_agent_is_curl(request),
             )
 

From 018d75a148ced6945aca7b095a272e0edba5aae1 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 1 Aug 2018 14:58:16 +0100
Subject: [PATCH 217/830] Refactor code for turning HttpResponseException into
 SynapseError

This commit replaces SynapseError.from_http_response_exception with
HttpResponseException.to_synapse_error.

The new method actually returns a ProxiedRequestError, which allows us to pass
through additional metadata from the API call.
---
 synapse/api/errors.py                     | 84 ++++++++++++++---------
 synapse/federation/federation_client.py   |  4 +-
 synapse/rest/media/v1/media_repository.py |  2 +-
 3 files changed, 56 insertions(+), 34 deletions(-)

diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index cf48829c8bce..7476c90ed343 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -105,38 +105,28 @@ def error_dict(self):
             self.errcode,
         )
 
-    @classmethod
-    def from_http_response_exception(cls, err):
-        """Make a SynapseError based on an HTTPResponseException
-
-        This is useful when a proxied request has failed, and we need to
-        decide how to map the failure onto a matrix error to send back to the
-        client.
 
-        An attempt is made to parse the body of the http response as a matrix
-        error. If that succeeds, the errcode and error message from the body
-        are used as the errcode and error message in the new synapse error.
-
-        Otherwise, the errcode is set to M_UNKNOWN, and the error message is
-        set to the reason code from the HTTP response.
-
-        Args:
-            err (HttpResponseException):
+class ProxiedRequestError(SynapseError):
+    """An error from a general matrix endpoint, eg. from a proxied Matrix API call.
 
-        Returns:
-            SynapseError:
-        """
-        # try to parse the body as json, to get better errcode/msg, but
-        # default to M_UNKNOWN with the HTTP status as the error text
-        try:
-            j = json.loads(err.response)
-        except ValueError:
-            j = {}
-        errcode = j.get('errcode', Codes.UNKNOWN)
-        errmsg = j.get('error', err.msg)
+    Attributes:
+        errcode (str): Matrix error code e.g 'M_FORBIDDEN'
+    """
+    def __init__(self, code, msg, errcode=Codes.UNKNOWN, additional_fields=None):
+        super(ProxiedRequestError, self).__init__(
+            code, msg, errcode
+        )
+        if additional_fields is None:
+            self._additional_fields = {}
+        else:
+            self._additional_fields = dict(additional_fields)
 
-        res = SynapseError(err.code, errmsg, errcode)
-        return res
+    def error_dict(self):
+        return cs_error(
+            self.msg,
+            self.errcode,
+            **self._additional_fields
+        )
 
 
 class ConsentNotGivenError(SynapseError):
@@ -361,7 +351,7 @@ class HttpResponseException(CodeMessageException):
     Represents an HTTP-level failure of an outbound request
 
     Attributes:
-        response (str): body of response
+        response (bytes): body of response
     """
     def __init__(self, code, msg, response):
         """
@@ -369,7 +359,39 @@ def __init__(self, code, msg, response):
         Args:
             code (int): HTTP status code
             msg (str): reason phrase from HTTP response status line
-            response (str): body of response
+            response (bytes): body of response
         """
         super(HttpResponseException, self).__init__(code, msg)
         self.response = response
+
+    def to_synapse_error(self):
+        """Make a SynapseError based on an HTTPResponseException
+
+        This is useful when a proxied request has failed, and we need to
+        decide how to map the failure onto a matrix error to send back to the
+        client.
+
+        An attempt is made to parse the body of the http response as a matrix
+        error. If that succeeds, the errcode and error message from the body
+        are used as the errcode and error message in the new synapse error.
+
+        Otherwise, the errcode is set to M_UNKNOWN, and the error message is
+        set to the reason code from the HTTP response.
+
+        Returns:
+            SynapseError:
+        """
+        # try to parse the body as json, to get better errcode/msg, but
+        # default to M_UNKNOWN with the HTTP status as the error text
+        try:
+            j = json.loads(self.response)
+        except ValueError:
+            j = {}
+
+        if not isinstance(j, dict):
+            j = {}
+
+        errcode = j.pop('errcode', Codes.UNKNOWN)
+        errmsg = j.pop('error', self.msg)
+
+        return ProxiedRequestError(self.code, errmsg, errcode, j)
diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index 0b09f93ca928..7550e11b6ec2 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -506,7 +506,7 @@ def _try_destination_list(self, description, destinations, callback):
                 )
             except HttpResponseException as e:
                 if not 500 <= e.code < 600:
-                    raise SynapseError.from_http_response_exception(e)
+                    raise e.to_synapse_error()
                 else:
                     logger.warn(
                         "Failed to %s via %s: %i %s",
@@ -682,7 +682,7 @@ def send_invite(self, destination, room_id, event_id, pdu):
             )
         except HttpResponseException as e:
             if e.code == 403:
-                raise SynapseError.from_http_response_exception(e)
+                raise e.to_synapse_error()
             raise
 
         pdu_dict = content["event"]
diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py
index 174ad20123cd..8fb413d8253b 100644
--- a/synapse/rest/media/v1/media_repository.py
+++ b/synapse/rest/media/v1/media_repository.py
@@ -379,7 +379,7 @@ def _download_remote_file(self, server_name, media_id, file_id):
                 logger.warn("HTTP error fetching remote media %s/%s: %s",
                             server_name, media_id, e.response)
                 if e.code == twisted.web.http.NOT_FOUND:
-                    raise SynapseError.from_http_response_exception(e)
+                    raise e.to_synapse_error()
                 raise SynapseError(502, "Failed to fetch remote media")
 
             except SynapseError:

From 01e93f48ed3dd78fda45a37733251659af19dde3 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 1 Aug 2018 15:04:50 +0100
Subject: [PATCH 218/830] Kill off MatrixCodeMessageException

This code brings the SimpleHttpClient into line with the
MatrixFederationHttpClient by having it raise HttpResponseExceptions when a
request fails (rather than trying to parse for matrix errors and maybe raising
MatrixCodeMessageException).

Then, whenever we were checking for MatrixCodeMessageException and turning them
into SynapseErrors, we now need to check for HttpResponseExceptions and call
to_synapse_error.
---
 synapse/api/errors.py                  | 11 -----
 synapse/handlers/identity.py           | 25 ++++-------
 synapse/http/client.py                 | 61 ++++++++++++--------------
 synapse/replication/http/membership.py | 18 ++++----
 synapse/replication/http/send_event.py | 10 ++---
 5 files changed, 47 insertions(+), 78 deletions(-)

diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index 7476c90ed343..356836238945 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -70,17 +70,6 @@ def __init__(self, code, msg):
         self.msg = msg
 
 
-class MatrixCodeMessageException(CodeMessageException):
-    """An error from a general matrix endpoint, eg. from a proxied Matrix API call.
-
-    Attributes:
-        errcode (str): Matrix error code e.g 'M_FORBIDDEN'
-    """
-    def __init__(self, code, msg, errcode=Codes.UNKNOWN):
-        super(MatrixCodeMessageException, self).__init__(code, msg)
-        self.errcode = errcode
-
-
 class SynapseError(CodeMessageException):
     """A base exception type for matrix errors which have an errcode and error
     message (as well as an HTTP status code).
diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py
index 8c8aedb2b84b..1d36d967c3b5 100644
--- a/synapse/handlers/identity.py
+++ b/synapse/handlers/identity.py
@@ -26,7 +26,7 @@
 from synapse.api.errors import (
     CodeMessageException,
     Codes,
-    MatrixCodeMessageException,
+    HttpResponseException,
     SynapseError,
 )
 
@@ -85,7 +85,6 @@ def threepid_from_creds(self, creds):
             )
             defer.returnValue(None)
 
-        data = {}
         try:
             data = yield self.http_client.get_json(
                 "https://%s%s" % (
@@ -94,11 +93,9 @@ def threepid_from_creds(self, creds):
                 ),
                 {'sid': creds['sid'], 'client_secret': client_secret}
             )
-        except MatrixCodeMessageException as e:
+        except HttpResponseException as e:
             logger.info("getValidated3pid failed with Matrix error: %r", e)
-            raise SynapseError(e.code, e.msg, e.errcode)
-        except CodeMessageException as e:
-            data = json.loads(e.msg)
+            raise e.to_synapse_error()
 
         if 'medium' in data:
             defer.returnValue(data)
@@ -136,7 +133,7 @@ def bind_threepid(self, creds, mxid):
             )
             logger.debug("bound threepid %r to %s", creds, mxid)
         except CodeMessageException as e:
-            data = json.loads(e.msg)
+            data = json.loads(e.msg)  # XXX WAT?
         defer.returnValue(data)
 
     @defer.inlineCallbacks
@@ -209,12 +206,9 @@ def requestEmailToken(self, id_server, email, client_secret, send_attempt, **kwa
                 params
             )
             defer.returnValue(data)
-        except MatrixCodeMessageException as e:
-            logger.info("Proxied requestToken failed with Matrix error: %r", e)
-            raise SynapseError(e.code, e.msg, e.errcode)
-        except CodeMessageException as e:
+        except HttpResponseException as e:
             logger.info("Proxied requestToken failed: %r", e)
-            raise e
+            raise e.to_synapse_error()
 
     @defer.inlineCallbacks
     def requestMsisdnToken(
@@ -244,9 +238,6 @@ def requestMsisdnToken(
                 params
             )
             defer.returnValue(data)
-        except MatrixCodeMessageException as e:
-            logger.info("Proxied requestToken failed with Matrix error: %r", e)
-            raise SynapseError(e.code, e.msg, e.errcode)
-        except CodeMessageException as e:
+        except HttpResponseException as e:
             logger.info("Proxied requestToken failed: %r", e)
-            raise e
+            raise e.to_synapse_error()
diff --git a/synapse/http/client.py b/synapse/http/client.py
index 25b63078848f..3771e0b3f627 100644
--- a/synapse/http/client.py
+++ b/synapse/http/client.py
@@ -39,12 +39,7 @@
 from twisted.web.http import PotentialDataLoss
 from twisted.web.http_headers import Headers
 
-from synapse.api.errors import (
-    CodeMessageException,
-    Codes,
-    MatrixCodeMessageException,
-    SynapseError,
-)
+from synapse.api.errors import Codes, HttpResponseException, SynapseError
 from synapse.http import cancelled_to_request_timed_out_error, redact_uri
 from synapse.http.endpoint import SpiderEndpoint
 from synapse.util.async import add_timeout_to_deferred
@@ -132,6 +127,11 @@ def post_urlencoded_get_json(self, uri, args={}, headers=None):
 
         Returns:
             Deferred[object]: parsed json
+
+        Raises:
+            HttpResponseException: On a non-2xx HTTP response.
+
+            ValueError: if the response was not JSON
         """
 
         # TODO: Do we ever want to log message contents?
@@ -155,7 +155,10 @@ def post_urlencoded_get_json(self, uri, args={}, headers=None):
 
         body = yield make_deferred_yieldable(readBody(response))
 
-        defer.returnValue(json.loads(body))
+        if 200 <= response.code < 300:
+            defer.returnValue(json.loads(body))
+        else:
+            raise HttpResponseException(response.code, response.phrase, body)
 
     @defer.inlineCallbacks
     def post_json_get_json(self, uri, post_json, headers=None):
@@ -169,6 +172,11 @@ def post_json_get_json(self, uri, post_json, headers=None):
 
         Returns:
             Deferred[object]: parsed json
+
+        Raises:
+            HttpResponseException: On a non-2xx HTTP response.
+
+            ValueError: if the response was not JSON
         """
         json_str = encode_canonical_json(post_json)
 
@@ -193,9 +201,7 @@ def post_json_get_json(self, uri, post_json, headers=None):
         if 200 <= response.code < 300:
             defer.returnValue(json.loads(body))
         else:
-            raise self._exceptionFromFailedRequest(response, body)
-
-        defer.returnValue(json.loads(body))
+            raise HttpResponseException(response.code, response.phrase, body)
 
     @defer.inlineCallbacks
     def get_json(self, uri, args={}, headers=None):
@@ -213,14 +219,12 @@ def get_json(self, uri, args={}, headers=None):
             Deferred: Succeeds when we get *any* 2xx HTTP response, with the
             HTTP body as JSON.
         Raises:
-            On a non-2xx HTTP response. The response body will be used as the
-            error message.
+            HttpResponseException On a non-2xx HTTP response.
+
+            ValueError: if the response was not JSON
         """
-        try:
-            body = yield self.get_raw(uri, args, headers=headers)
-            defer.returnValue(json.loads(body))
-        except CodeMessageException as e:
-            raise self._exceptionFromFailedRequest(e.code, e.msg)
+        body = yield self.get_raw(uri, args, headers=headers)
+        defer.returnValue(json.loads(body))
 
     @defer.inlineCallbacks
     def put_json(self, uri, json_body, args={}, headers=None):
@@ -239,7 +243,9 @@ def put_json(self, uri, json_body, args={}, headers=None):
             Deferred: Succeeds when we get *any* 2xx HTTP response, with the
             HTTP body as JSON.
         Raises:
-            On a non-2xx HTTP response.
+            HttpResponseException On a non-2xx HTTP response.
+
+            ValueError: if the response was not JSON
         """
         if len(args):
             query_bytes = urllib.urlencode(args, True)
@@ -266,10 +272,7 @@ def put_json(self, uri, json_body, args={}, headers=None):
         if 200 <= response.code < 300:
             defer.returnValue(json.loads(body))
         else:
-            # NB: This is explicitly not json.loads(body)'d because the contract
-            # of CodeMessageException is a *string* message. Callers can always
-            # load it into JSON if they want.
-            raise CodeMessageException(response.code, body)
+            raise HttpResponseException(response.code, response.phrase, body)
 
     @defer.inlineCallbacks
     def get_raw(self, uri, args={}, headers=None):
@@ -287,8 +290,7 @@ def get_raw(self, uri, args={}, headers=None):
             Deferred: Succeeds when we get *any* 2xx HTTP response, with the
             HTTP body at text.
         Raises:
-            On a non-2xx HTTP response. The response body will be used as the
-            error message.
+            HttpResponseException on a non-2xx HTTP response.
         """
         if len(args):
             query_bytes = urllib.urlencode(args, True)
@@ -311,16 +313,7 @@ def get_raw(self, uri, args={}, headers=None):
         if 200 <= response.code < 300:
             defer.returnValue(body)
         else:
-            raise CodeMessageException(response.code, body)
-
-    def _exceptionFromFailedRequest(self, response, body):
-        try:
-            jsonBody = json.loads(body)
-            errcode = jsonBody['errcode']
-            error = jsonBody['error']
-            return MatrixCodeMessageException(response.code, error, errcode)
-        except (ValueError, KeyError):
-            return CodeMessageException(response.code, body)
+            raise HttpResponseException(response.code, response.phrase, body)
 
     # XXX: FIXME: This is horribly copy-pasted from matrixfederationclient.
     # The two should be factored out.
diff --git a/synapse/replication/http/membership.py b/synapse/replication/http/membership.py
index 6bfc8a5b89d4..7a3cfb159cde 100644
--- a/synapse/replication/http/membership.py
+++ b/synapse/replication/http/membership.py
@@ -18,7 +18,7 @@
 
 from twisted.internet import defer
 
-from synapse.api.errors import MatrixCodeMessageException, SynapseError
+from synapse.api.errors import HttpResponseException
 from synapse.http.servlet import RestServlet, parse_json_object_from_request
 from synapse.types import Requester, UserID
 from synapse.util.distributor import user_joined_room, user_left_room
@@ -56,11 +56,11 @@ def remote_join(client, host, port, requester, remote_room_hosts,
 
     try:
         result = yield client.post_json_get_json(uri, payload)
-    except MatrixCodeMessageException as e:
+    except HttpResponseException as e:
         # We convert to SynapseError as we know that it was a SynapseError
         # on the master process that we should send to the client. (And
         # importantly, not stack traces everywhere)
-        raise SynapseError(e.code, e.msg, e.errcode)
+        raise e.to_synapse_error()
     defer.returnValue(result)
 
 
@@ -92,11 +92,11 @@ def remote_reject_invite(client, host, port, requester, remote_room_hosts,
 
     try:
         result = yield client.post_json_get_json(uri, payload)
-    except MatrixCodeMessageException as e:
+    except HttpResponseException as e:
         # We convert to SynapseError as we know that it was a SynapseError
         # on the master process that we should send to the client. (And
         # importantly, not stack traces everywhere)
-        raise SynapseError(e.code, e.msg, e.errcode)
+        raise e.to_synapse_error()
     defer.returnValue(result)
 
 
@@ -131,11 +131,11 @@ def get_or_register_3pid_guest(client, host, port, requester,
 
     try:
         result = yield client.post_json_get_json(uri, payload)
-    except MatrixCodeMessageException as e:
+    except HttpResponseException as e:
         # We convert to SynapseError as we know that it was a SynapseError
         # on the master process that we should send to the client. (And
         # importantly, not stack traces everywhere)
-        raise SynapseError(e.code, e.msg, e.errcode)
+        raise e.to_synapse_error()
     defer.returnValue(result)
 
 
@@ -165,11 +165,11 @@ def notify_user_membership_change(client, host, port, user_id, room_id, change):
 
     try:
         result = yield client.post_json_get_json(uri, payload)
-    except MatrixCodeMessageException as e:
+    except HttpResponseException as e:
         # We convert to SynapseError as we know that it was a SynapseError
         # on the master process that we should send to the client. (And
         # importantly, not stack traces everywhere)
-        raise SynapseError(e.code, e.msg, e.errcode)
+        raise e.to_synapse_error()
     defer.returnValue(result)
 
 
diff --git a/synapse/replication/http/send_event.py b/synapse/replication/http/send_event.py
index 5227bc333d19..d3509dc288f1 100644
--- a/synapse/replication/http/send_event.py
+++ b/synapse/replication/http/send_event.py
@@ -18,11 +18,7 @@
 
 from twisted.internet import defer
 
-from synapse.api.errors import (
-    CodeMessageException,
-    MatrixCodeMessageException,
-    SynapseError,
-)
+from synapse.api.errors import CodeMessageException, HttpResponseException
 from synapse.events import FrozenEvent
 from synapse.events.snapshot import EventContext
 from synapse.http.servlet import RestServlet, parse_json_object_from_request
@@ -83,11 +79,11 @@ def send_event_to_master(clock, store, client, host, port, requester, event, con
             # If we timed out we probably don't need to worry about backing
             # off too much, but lets just wait a little anyway.
             yield clock.sleep(1)
-    except MatrixCodeMessageException as e:
+    except HttpResponseException as e:
         # We convert to SynapseError as we know that it was a SynapseError
         # on the master process that we should send to the client. (And
         # importantly, not stack traces everywhere)
-        raise SynapseError(e.code, e.msg, e.errcode)
+        raise e.to_synapse_error()
     defer.returnValue(result)
 
 

From 38b98e5a98414eeba2d8e3671875f66a7b27645a Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 1 Aug 2018 16:07:49 +0100
Subject: [PATCH 219/830] changelog

---
 changelog.d/3638.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3638.misc

diff --git a/changelog.d/3638.misc b/changelog.d/3638.misc
new file mode 100644
index 000000000000..9faab15cf265
--- /dev/null
+++ b/changelog.d/3638.misc
@@ -0,0 +1 @@
+Factor out exception handling in federation_client

From 7ff44d92154875c55addc079b0ad7f303c699933 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 1 Aug 2018 16:17:00 +0100
Subject: [PATCH 220/830] improve clarity

---
 changelog.d/3630.feature    | 2 +-
 synapse/storage/__init__.py | 9 +++++----
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/changelog.d/3630.feature b/changelog.d/3630.feature
index 20398da9e0aa..8007a048405b 100644
--- a/changelog.d/3630.feature
+++ b/changelog.d/3630.feature
@@ -1 +1 @@
-Blocks registration and sign in if max mau value exceeded
+Add ability to limit number of monthly active users on the server
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 12282a8b3dcb..134e4a80f1f0 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -268,12 +268,13 @@ def _count_users(txn):
         return self.runInteraction("count_users", _count_users)
 
     def count_monthly_users(self):
-        """
-        Counts the number of users who used this homeserver in the last 30 days
+        """Counts the number of users who used this homeserver in the last 30 days
+
         This method should be refactored with count_daily_users - the only
         reason not to is waiting on definition of mau
-        returns:
-            defered: resolves to int
+
+        Returns:
+            Defered[int]
         """
         def _count_monthly_users(txn):
             thirty_days_ago = int(self._clock.time_msec()) - (1000 * 60 * 60 * 24 * 30)

From b7f203a56658bdd6d7bf39e82fc1ffd0dae1f61a Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 1 Aug 2018 16:17:42 +0100
Subject: [PATCH 221/830] count_monthly_users is now async

---
 synapse/app/homeserver.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 46325d8521f6..fba51c26e869 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -519,10 +519,11 @@ def generate_user_daily_visit_stats():
     # table will decrease
     clock.looping_call(generate_user_daily_visit_stats, 5 * 60 * 1000)
 
+    @defer.inlineCallbacks
     def generate_monthly_active_users():
         count = 0
         if hs.config.limit_usage_by_mau:
-            count = hs.get_datastore().count_monthly_users()
+            count = yield hs.get_datastore().count_monthly_users()
         current_mau_gauge.set(float(count))
         max_mau_value_gauge.set(float(hs.config.max_mau_value))
 

From 908be65e646ba933d595f3d8874cee69774e7243 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 1 Aug 2018 16:23:31 +0100
Subject: [PATCH 222/830] changelog

---
 changelog.d/3639.feature | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3639.feature

diff --git a/changelog.d/3639.feature b/changelog.d/3639.feature
new file mode 100644
index 000000000000..c8c387e219c1
--- /dev/null
+++ b/changelog.d/3639.feature
@@ -0,0 +1 @@
+When we fail to join a room over federation, pass the error code back to the client.
\ No newline at end of file

From ec716a35b219d147dee51733b55573952799a549 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 1 Aug 2018 17:54:37 +0100
Subject: [PATCH 223/830] change monthly_active_users table to be a single
 column

---
 synapse/storage/monthly_active_users.py                | 10 +++-------
 .../storage/schema/delta/51/monthly_active_users.sql   |  4 +---
 tests/storage/test_monthly_active_users.py             |  6 +++---
 3 files changed, 7 insertions(+), 13 deletions(-)

diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py
index 03eeea792e22..7b3f13aedf17 100644
--- a/synapse/storage/monthly_active_users.py
+++ b/synapse/storage/monthly_active_users.py
@@ -38,22 +38,18 @@ def _count_users(txn):
             return count
         return self.runInteraction("count_users", _count_users)
 
-    def upsert_monthly_active_user(self, user_id):
+    def insert_monthly_active_user(self, user_id):
         """
             Updates or inserts monthly active user member
             Arguments:
                 user_id (str): user to add/update
         """
-        return self._simple_upsert(
+        return self._simple_insert(
             desc="upsert_monthly_active_user",
             table="monthly_active_users",
-            keyvalues={
-                "user_id": user_id,
-            },
             values={
-                "timestamp": int(self._clock.time_msec()),
+                "user_id": user_id,
             },
-            lock=False,
         )
 
     @defer.inlineCallbacks
diff --git a/synapse/storage/schema/delta/51/monthly_active_users.sql b/synapse/storage/schema/delta/51/monthly_active_users.sql
index f2b6d3e31e4e..590b1abab2f9 100644
--- a/synapse/storage/schema/delta/51/monthly_active_users.sql
+++ b/synapse/storage/schema/delta/51/monthly_active_users.sql
@@ -15,9 +15,7 @@
 
 -- a table of monthly active users, for use where blocking based on mau limits
 CREATE TABLE monthly_active_users (
-    user_id TEXT NOT NULL,
-    timestamp BIGINT NOT NULL
+    user_id TEXT NOT NULL
 );
 
 CREATE UNIQUE INDEX monthly_active_users_users ON monthly_active_users(user_id);
-CREATE INDEX monthly_active_users_time_stamp ON monthly_active_users(timestamp);
diff --git a/tests/storage/test_monthly_active_users.py b/tests/storage/test_monthly_active_users.py
index 9b1ffc6369ee..7a8432ce6942 100644
--- a/tests/storage/test_monthly_active_users.py
+++ b/tests/storage/test_monthly_active_users.py
@@ -22,7 +22,7 @@ def test_can_insert_and_count_mau(self):
         count = yield self.mau.get_monthly_active_count()
         self.assertEqual(0, count)
 
-        yield self.mau.upsert_monthly_active_user("@user:server")
+        yield self.mau.insert_monthly_active_user("@user:server")
         count = yield self.mau.get_monthly_active_count()
 
         self.assertEqual(1, count)
@@ -34,8 +34,8 @@ def test_is_user_monthly_active(self):
         user_id3 = "@user3:server"
         result = yield self.mau.is_user_monthly_active(user_id1)
         self.assertFalse(result)
-        yield self.mau.upsert_monthly_active_user(user_id1)
-        yield self.mau.upsert_monthly_active_user(user_id2)
+        yield self.mau.insert_monthly_active_user(user_id1)
+        yield self.mau.insert_monthly_active_user(user_id2)
         result = yield self.mau.is_user_monthly_active(user_id1)
         self.assertTrue(result)
         result = yield self.mau.is_user_monthly_active(user_id3)

From c21d82bab3b32e2f59c9ef09a1a10b337ce45db4 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 1 Aug 2018 23:24:38 +0100
Subject: [PATCH 224/830] normalise reaping query

---
 synapse/storage/monthly_active_users.py | 41 +++++++++++++++++++++++--
 1 file changed, 38 insertions(+), 3 deletions(-)

diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py
index 7b3f13aedf17..0741c7fa614e 100644
--- a/synapse/storage/monthly_active_users.py
+++ b/synapse/storage/monthly_active_users.py
@@ -7,6 +7,7 @@ class MonthlyActiveUsersStore(SQLBaseStore):
     def __init__(self, hs):
         super(MonthlyActiveUsersStore, self).__init__(None, hs)
         self._clock = hs.get_clock()
+        self.max_mau_value = hs.config.max_mau_value
 
     def reap_monthly_active_users(self):
         """
@@ -19,8 +20,42 @@ def _reap_users(txn):
             thirty_days_ago = (
                 int(self._clock.time_msec()) - (1000 * 60 * 60 * 24 * 30)
             )
-            sql = "DELETE FROM monthly_active_users WHERE timestamp < ?"
-            txn.execute(sql, (thirty_days_ago,))
+
+            # Query deletes the union of users that have either:
+            #     * not visited in the last 30 days
+            #     * exceeded the total max_mau_value threshold. Where there is
+            #       an excess, more recent users are favoured - this is to cover
+            #       the case where the limit has been step change reduced.
+            #
+            sql = """
+            DELETE FROM monthly_active_users
+            WHERE user_id
+            IN (
+                SELECT * FROM (
+                    SELECT monthly_active_users.user_id
+                    FROM monthly_active_users
+                    LEFT JOIN (
+                        SELECT user_id, max(last_seen) AS last_seen
+                        FROM user_ips
+                        GROUP BY user_id
+                    ) AS uip ON uip.user_id=monthly_active_users.user_id
+                    ORDER BY uip.last_seen desc LIMIT -1 OFFSET ?
+                )
+                UNION
+                SELECT * FROM (
+                    SELECT monthly_active_users.user_id
+                    FROM monthly_active_users
+                    LEFT JOIN (
+                        SELECT user_id, max(last_seen) AS last_seen
+                        FROM user_ips
+                        GROUP BY user_id
+                    ) AS uip  ON uip.user_id=monthly_active_users.user_id
+                    WHERE uip.last_seen < ?
+                )
+            )
+            """
+
+            txn.execute(sql, (self.max_mau_value, thirty_days_ago,))
 
         return self.runInteraction("reap_monthly_active_users", _reap_users)
 
@@ -45,7 +80,7 @@ def insert_monthly_active_user(self, user_id):
                 user_id (str): user to add/update
         """
         return self._simple_insert(
-            desc="upsert_monthly_active_user",
+            desc="insert_monthly_active_user",
             table="monthly_active_users",
             values={
                 "user_id": user_id,

From 08281fe6b7cdd9620d28d2aa6b725bdb0a351bbc Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 1 Aug 2018 23:26:24 +0100
Subject: [PATCH 225/830] self.db_conn unused

---
 synapse/storage/__init__.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 134e4a80f1f0..04ff1f8006a2 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -94,7 +94,6 @@ def __init__(self, db_conn, hs):
         self._clock = hs.get_clock()
         self.database_engine = hs.database_engine
 
-        self.db_conn = db_conn
         self._stream_id_gen = StreamIdGenerator(
             db_conn, "events", "stream_ordering",
             extra_tables=[("local_invites", "stream_id")]

From 165e06703340667dd88eb73dfe299a4d3298b94b Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Thu, 2 Aug 2018 10:59:58 +0100
Subject: [PATCH 226/830] Revert "change monthly_active_users table to be a
 single column"

This reverts commit ec716a35b219d147dee51733b55573952799a549.
---
 synapse/storage/monthly_active_users.py                | 10 +++++++---
 .../storage/schema/delta/51/monthly_active_users.sql   |  4 +++-
 tests/storage/test_monthly_active_users.py             |  6 +++---
 3 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py
index 7b3f13aedf17..03eeea792e22 100644
--- a/synapse/storage/monthly_active_users.py
+++ b/synapse/storage/monthly_active_users.py
@@ -38,18 +38,22 @@ def _count_users(txn):
             return count
         return self.runInteraction("count_users", _count_users)
 
-    def insert_monthly_active_user(self, user_id):
+    def upsert_monthly_active_user(self, user_id):
         """
             Updates or inserts monthly active user member
             Arguments:
                 user_id (str): user to add/update
         """
-        return self._simple_insert(
+        return self._simple_upsert(
             desc="upsert_monthly_active_user",
             table="monthly_active_users",
-            values={
+            keyvalues={
                 "user_id": user_id,
             },
+            values={
+                "timestamp": int(self._clock.time_msec()),
+            },
+            lock=False,
         )
 
     @defer.inlineCallbacks
diff --git a/synapse/storage/schema/delta/51/monthly_active_users.sql b/synapse/storage/schema/delta/51/monthly_active_users.sql
index 590b1abab2f9..f2b6d3e31e4e 100644
--- a/synapse/storage/schema/delta/51/monthly_active_users.sql
+++ b/synapse/storage/schema/delta/51/monthly_active_users.sql
@@ -15,7 +15,9 @@
 
 -- a table of monthly active users, for use where blocking based on mau limits
 CREATE TABLE monthly_active_users (
-    user_id TEXT NOT NULL
+    user_id TEXT NOT NULL,
+    timestamp BIGINT NOT NULL
 );
 
 CREATE UNIQUE INDEX monthly_active_users_users ON monthly_active_users(user_id);
+CREATE INDEX monthly_active_users_time_stamp ON monthly_active_users(timestamp);
diff --git a/tests/storage/test_monthly_active_users.py b/tests/storage/test_monthly_active_users.py
index 7a8432ce6942..9b1ffc6369ee 100644
--- a/tests/storage/test_monthly_active_users.py
+++ b/tests/storage/test_monthly_active_users.py
@@ -22,7 +22,7 @@ def test_can_insert_and_count_mau(self):
         count = yield self.mau.get_monthly_active_count()
         self.assertEqual(0, count)
 
-        yield self.mau.insert_monthly_active_user("@user:server")
+        yield self.mau.upsert_monthly_active_user("@user:server")
         count = yield self.mau.get_monthly_active_count()
 
         self.assertEqual(1, count)
@@ -34,8 +34,8 @@ def test_is_user_monthly_active(self):
         user_id3 = "@user3:server"
         result = yield self.mau.is_user_monthly_active(user_id1)
         self.assertFalse(result)
-        yield self.mau.insert_monthly_active_user(user_id1)
-        yield self.mau.insert_monthly_active_user(user_id2)
+        yield self.mau.upsert_monthly_active_user(user_id1)
+        yield self.mau.upsert_monthly_active_user(user_id2)
         result = yield self.mau.is_user_monthly_active(user_id1)
         self.assertTrue(result)
         result = yield self.mau.is_user_monthly_active(user_id3)

From 00f99f74b1b875bb7ac6b0623994cabad4a59cc6 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Thu, 2 Aug 2018 13:47:19 +0100
Subject: [PATCH 227/830] insertion into monthly_active_users

---
 synapse/api/auth.py                     |  2 +-
 synapse/storage/__init__.py             |  2 +
 synapse/storage/client_ips.py           | 22 ++++++++-
 synapse/storage/monthly_active_users.py | 18 ++++---
 tests/storage/test_client_ips.py        | 66 +++++++++++++++++++++++--
 5 files changed, 99 insertions(+), 11 deletions(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 5bbbe8e2e71f..d8022bcf8e95 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -213,7 +213,7 @@ def get_user_by_req(self, request, allow_guest=False, rights="access"):
                 default=[b""]
             )[0]
             if user and access_token and ip_addr:
-                self.store.insert_client_ip(
+                yield self.store.insert_client_ip(
                     user_id=user.to_string(),
                     access_token=access_token,
                     ip=ip_addr,
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 04ff1f8006a2..2120f46ed5be 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -39,6 +39,7 @@
 from .group_server import GroupServerStore
 from .keys import KeyStore
 from .media_repository import MediaRepositoryStore
+from .monthly_active_users import MonthlyActiveUsersStore
 from .openid import OpenIdStore
 from .presence import PresenceStore, UserPresenceState
 from .profile import ProfileStore
@@ -87,6 +88,7 @@ class DataStore(RoomMemberStore, RoomStore,
                 UserDirectoryStore,
                 GroupServerStore,
                 UserErasureStore,
+                MonthlyActiveUsersStore,
                 ):
 
     def __init__(self, db_conn, hs):
diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py
index b8cefd43d6df..506915a1ef65 100644
--- a/synapse/storage/client_ips.py
+++ b/synapse/storage/client_ips.py
@@ -35,6 +35,7 @@
 
 class ClientIpStore(background_updates.BackgroundUpdateStore):
     def __init__(self, db_conn, hs):
+
         self.client_ip_last_seen = Cache(
             name="client_ip_last_seen",
             keylen=4,
@@ -74,6 +75,7 @@ def __init__(self, db_conn, hs):
             "before", "shutdown", self._update_client_ips_batch
         )
 
+    @defer.inlineCallbacks
     def insert_client_ip(self, user_id, access_token, ip, user_agent, device_id,
                          now=None):
         if not now:
@@ -84,7 +86,7 @@ def insert_client_ip(self, user_id, access_token, ip, user_agent, device_id,
             last_seen = self.client_ip_last_seen.get(key)
         except KeyError:
             last_seen = None
-
+        yield self._populate_monthly_active_users(user_id)
         # Rate-limited inserts
         if last_seen is not None and (now - last_seen) < LAST_SEEN_GRANULARITY:
             return
@@ -93,6 +95,24 @@ def insert_client_ip(self, user_id, access_token, ip, user_agent, device_id,
 
         self._batch_row_update[key] = (user_agent, device_id, now)
 
+    @defer.inlineCallbacks
+    def _populate_monthly_active_users(self, user_id):
+        store = self.hs.get_datastore()
+        print "entering _populate_monthly_active_users"
+        if self.hs.config.limit_usage_by_mau:
+            print "self.hs.config.limit_usage_by_mau is TRUE"
+            is_user_monthly_active = yield store.is_user_monthly_active(user_id)
+            print "is_user_monthly_active is %r" % is_user_monthly_active
+            if is_user_monthly_active:
+                yield store.upsert_monthly_active_user(user_id)
+            else:
+                count = yield store.get_monthly_active_count()
+                print "count is %d" % count
+                if count < self.hs.config.max_mau_value:
+                    print "count is less than self.hs.config.max_mau_value "
+                    res = yield store.upsert_monthly_active_user(user_id)
+                    print "upsert response is %r" % res
+
     def _update_client_ips_batch(self):
         def update():
             to_update = self._batch_row_update
diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py
index 2337438c583b..aada9bd2b954 100644
--- a/synapse/storage/monthly_active_users.py
+++ b/synapse/storage/monthly_active_users.py
@@ -4,7 +4,7 @@
 
 
 class MonthlyActiveUsersStore(SQLBaseStore):
-    def __init__(self, hs):
+    def __init__(self, dbconn, hs):
         super(MonthlyActiveUsersStore, self).__init__(None, hs)
         self._clock = hs.get_clock()
         self.max_mau_value = hs.config.max_mau_value
@@ -14,24 +14,28 @@ def reap_monthly_active_users(self):
         Cleans out monthly active user table to ensure that no stale
         entries exist.
         Return:
-            defered, no return type
+            Defered()
         """
         def _reap_users(txn):
             thirty_days_ago = (
                 int(self._clock.time_msec()) - (1000 * 60 * 60 * 24 * 30)
             )
-
             sql = "DELETE FROM monthly_active_users WHERE timestamp < ?"
-
             txn.execute(sql, (thirty_days_ago,))
+            sql = """
+                DELETE FROM monthly_active_users
+                ORDER BY timestamp desc
+                LIMIT -1 OFFSET ?
+                """
+            txn.execute(sql, (self.max_mau_value,))
 
         return self.runInteraction("reap_monthly_active_users", _reap_users)
 
     def get_monthly_active_count(self):
         """
             Generates current count of monthly active users.abs
-            return:
-                defered resolves to int
+            Return:
+                Defered(int): Number of current monthly active users
         """
         def _count_users(txn):
             sql = "SELECT COALESCE(count(*), 0) FROM monthly_active_users"
@@ -46,6 +50,8 @@ def upsert_monthly_active_user(self, user_id):
             Updates or inserts monthly active user member
             Arguments:
                 user_id (str): user to add/update
+            Deferred(bool): True if a new entry was created, False if an
+                existing one was updated.
         """
         return self._simple_upsert(
             desc="upsert_monthly_active_user",
diff --git a/tests/storage/test_client_ips.py b/tests/storage/test_client_ips.py
index bd6fda6cb12f..e1552510cc50 100644
--- a/tests/storage/test_client_ips.py
+++ b/tests/storage/test_client_ips.py
@@ -12,6 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from mock import Mock
 
 from twisted.internet import defer
 
@@ -27,9 +28,9 @@ def __init__(self, *args, **kwargs):
 
     @defer.inlineCallbacks
     def setUp(self):
-        hs = yield tests.utils.setup_test_homeserver()
-        self.store = hs.get_datastore()
-        self.clock = hs.get_clock()
+        self.hs = yield tests.utils.setup_test_homeserver()
+        self.store = self.hs.get_datastore()
+        self.clock = self.hs.get_clock()
 
     @defer.inlineCallbacks
     def test_insert_new_client_ip(self):
@@ -54,3 +55,62 @@ def test_insert_new_client_ip(self):
             },
             r
         )
+
+    @defer.inlineCallbacks
+    def test_disabled_monthly_active_user(self):
+        self.hs.config.limit_usage_by_mau = False
+        self.hs.config.max_mau_value = 50
+        user_id = "@user:server"
+        yield self.store.insert_client_ip(
+            user_id, "access_token", "ip", "user_agent", "device_id",
+        )
+        active = yield self.store.is_user_monthly_active(user_id)
+        self.assertFalse(active)
+
+    @defer.inlineCallbacks
+    def test_adding_monthly_active_user_when_full(self):
+        self.hs.config.limit_usage_by_mau = True
+        self.hs.config.max_mau_value = 50
+        lots_of_users = 100
+        user_id = "@user:server"
+
+        self.store.get_monthly_active_count = Mock(
+            return_value=defer.succeed(lots_of_users)
+        )
+        yield self.store.insert_client_ip(
+            user_id, "access_token", "ip", "user_agent", "device_id",
+        )
+        active = yield self.store.is_user_monthly_active(user_id)
+        self.assertFalse(active)
+
+    @defer.inlineCallbacks
+    def test_adding_monthly_active_user_when_space(self):
+        self.hs.config.limit_usage_by_mau = True
+        self.hs.config.max_mau_value = 50
+        user_id = "@user:server"
+        active = yield self.store.is_user_monthly_active(user_id)
+        self.assertFalse(active)
+
+        yield self.store.insert_client_ip(
+            user_id, "access_token", "ip", "user_agent", "device_id",
+        )
+        active = yield self.store.is_user_monthly_active(user_id)
+        self.assertTrue(active)
+
+    @defer.inlineCallbacks
+    def test_updating_monthly_active_user_when_space(self):
+        self.hs.config.limit_usage_by_mau = True
+        self.hs.config.max_mau_value = 50
+        user_id = "@user:server"
+
+        active = yield self.store.is_user_monthly_active(user_id)
+        self.assertFalse(active)
+
+        yield self.store.insert_client_ip(
+            user_id, "access_token", "ip", "user_agent", "device_id",
+        )
+        yield self.store.insert_client_ip(
+            user_id, "access_token", "ip", "user_agent", "device_id",
+        )
+        active = yield self.store.is_user_monthly_active(user_id)
+        self.assertTrue(active)

From 0a65450d044fb580d789013dcdac48b10c930761 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 2 Aug 2018 11:53:52 +0100
Subject: [PATCH 228/830] Validation for events/rooms in fed requests

When we get a federation request which refers to an event id, make sure that
said event is in the room the caller claims it is in.

(patch supplied by @turt2live)
---
 synapse/federation/federation_server.py |  1 +
 synapse/handlers/federation.py          | 35 ++++++++++++++++++++++++-
 synapse/storage/event_federation.py     | 29 ++++++++++++++++++++
 3 files changed, 64 insertions(+), 1 deletion(-)

diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index 48f26db67ca0..10e71c78ce73 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -425,6 +425,7 @@ def on_query_auth_request(self, origin, content, room_id, event_id):
             ret = yield self.handler.on_query_auth(
                 origin,
                 event_id,
+                room_id,
                 signed_auth,
                 content.get("rejects", []),
                 content.get("missing", []),
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 20fb46fc89a7..12eeb7c4cd00 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -1349,6 +1349,9 @@ def on_send_leave_request(self, origin, pdu):
     def get_state_for_pdu(self, room_id, event_id):
         """Returns the state at the event. i.e. not including said event.
         """
+
+        yield self._verify_events_in_room([event_id], room_id)
+
         state_groups = yield self.store.get_state_groups(
             room_id, [event_id]
         )
@@ -1391,6 +1394,9 @@ def get_state_for_pdu(self, room_id, event_id):
     def get_state_ids_for_pdu(self, room_id, event_id):
         """Returns the state at the event. i.e. not including said event.
         """
+
+        yield self._verify_events_in_room([event_id], room_id)
+
         state_groups = yield self.store.get_state_groups_ids(
             room_id, [event_id]
         )
@@ -1420,6 +1426,8 @@ def on_backfill_request(self, origin, room_id, pdu_list, limit):
         if not in_room:
             raise AuthError(403, "Host not in room.")
 
+        yield self._verify_events_in_room(pdu_list, room_id)
+
         events = yield self.store.get_backfill_events(
             room_id,
             pdu_list,
@@ -1706,8 +1714,17 @@ def _prep_event(self, origin, event, state=None, auth_events=None):
         defer.returnValue(context)
 
     @defer.inlineCallbacks
-    def on_query_auth(self, origin, event_id, remote_auth_chain, rejects,
+    def on_query_auth(self, origin, event_id, room_id, remote_auth_chain, rejects,
                       missing):
+        in_room = yield self.auth.check_host_in_room(
+            room_id,
+            origin
+        )
+        if not in_room:
+            raise AuthError(403, "Host not in room.")
+
+        yield self._verify_events_in_room([event_id], room_id)
+
         # Just go through and process each event in `remote_auth_chain`. We
         # don't want to fall into the trap of `missing` being wrong.
         for e in remote_auth_chain:
@@ -2368,3 +2385,19 @@ def _check_key_revocation(self, public_key, url):
             )
         if "valid" not in response or not response["valid"]:
             raise AuthError(403, "Third party certificate was invalid")
+
+    @defer.inlineCallbacks
+    def _verify_events_in_room(self, pdu_ids, room_id):
+        """Checks whether the given PDU IDs are in the given room or not.
+
+        Args:
+            pdu_ids (list): list of PDU IDs
+            room_id (str): the room ID that the PDUs should be in
+
+        Raises:
+            AuthError: if one or more of the PDUs does not belong to the
+                given room.
+        """
+        room_ids = yield self.store.get_room_ids_for_events(pdu_ids)
+        if len(room_ids) != 1 or room_ids[0] != room_id:
+            raise AuthError(403, "Events must belong to the given room")
diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py
index 8d366d1b915e..e860fe1a1eda 100644
--- a/synapse/storage/event_federation.py
+++ b/synapse/storage/event_federation.py
@@ -295,6 +295,35 @@ def get_forward_extremeties_for_room_txn(txn):
             get_forward_extremeties_for_room_txn
         )
 
+    def get_room_ids_for_events(self, event_ids):
+        """Get a list of room IDs for which the given events belong.
+
+        Args:
+            event_ids (list): the events to look up the room of
+
+        Returns:
+            list, the room IDs for the events
+        """
+        return self.runInteraction(
+            "get_room_ids_for_events",
+            self._get_room_ids_for_events, event_ids
+        )
+
+    def _get_room_ids_for_events(self, txn, event_ids):
+        logger.debug("_get_room_ids_for_events: %s", repr(event_ids))
+
+        base_sql = (
+            "SELECT DISTINCT room_id FROM events"
+            " WHERE event_id IN (%s)"
+        )
+
+        txn.execute(
+            base_sql % (",".join(["?"] * len(event_ids)),),
+            event_ids
+        )
+
+        return [r[0] for r in txn]
+
     def get_backfill_events(self, room_id, event_list, limit):
         """Get a list of Events for a given topic that occurred before (and
         including) the events in event_list. Return a list of max size `limit`

From c4ffbecb6856602743198f1a8d4c25a8bb7afd4d Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Thu, 2 Aug 2018 13:51:05 +0100
Subject: [PATCH 229/830] fix test, update constructor call

---
 tests/storage/test_monthly_active_users.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/storage/test_monthly_active_users.py b/tests/storage/test_monthly_active_users.py
index 9b1ffc6369ee..d8d25a606911 100644
--- a/tests/storage/test_monthly_active_users.py
+++ b/tests/storage/test_monthly_active_users.py
@@ -15,7 +15,7 @@ def __init__(self, *args, **kwargs):
     @defer.inlineCallbacks
     def setUp(self):
         hs = yield setup_test_homeserver()
-        self.mau = MonthlyActiveUsersStore(hs)
+        self.mau = MonthlyActiveUsersStore(None, hs)
 
     @defer.inlineCallbacks
     def test_can_insert_and_count_mau(self):

From 14fa9d4d92eaa242d44a2823bbd9908be2f02d81 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 2 Aug 2018 13:23:48 +0100
Subject: [PATCH 230/830] Avoid extra db lookups

Since we're about to look up the events themselves anyway, we can skip the
extra db queries here.
---
 synapse/handlers/federation.py      | 38 +++++++++--------------------
 synapse/storage/event_federation.py | 30 +----------------------
 synapse/storage/events_worker.py    | 20 ++++++++++-----
 3 files changed, 26 insertions(+), 62 deletions(-)

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 12eeb7c4cd00..60391d07c404 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -1350,7 +1350,9 @@ def get_state_for_pdu(self, room_id, event_id):
         """Returns the state at the event. i.e. not including said event.
         """
 
-        yield self._verify_events_in_room([event_id], room_id)
+        event = yield self.store.get_event(
+            event_id, allow_none=False, check_room_id=room_id,
+        )
 
         state_groups = yield self.store.get_state_groups(
             room_id, [event_id]
@@ -1362,8 +1364,7 @@ def get_state_for_pdu(self, room_id, event_id):
                 (e.type, e.state_key): e for e in state
             }
 
-            event = yield self.store.get_event(event_id)
-            if event and event.is_state():
+            if event.is_state():
                 # Get previous state
                 if "replaces_state" in event.unsigned:
                     prev_id = event.unsigned["replaces_state"]
@@ -1394,8 +1395,9 @@ def get_state_for_pdu(self, room_id, event_id):
     def get_state_ids_for_pdu(self, room_id, event_id):
         """Returns the state at the event. i.e. not including said event.
         """
-
-        yield self._verify_events_in_room([event_id], room_id)
+        event = yield self.store.get_event(
+            event_id, allow_none=False, check_room_id=room_id,
+        )
 
         state_groups = yield self.store.get_state_groups_ids(
             room_id, [event_id]
@@ -1405,8 +1407,7 @@ def get_state_ids_for_pdu(self, room_id, event_id):
             _, state = state_groups.items().pop()
             results = state
 
-            event = yield self.store.get_event(event_id)
-            if event and event.is_state():
+            if event.is_state():
                 # Get previous state
                 if "replaces_state" in event.unsigned:
                     prev_id = event.unsigned["replaces_state"]
@@ -1426,8 +1427,6 @@ def on_backfill_request(self, origin, room_id, pdu_list, limit):
         if not in_room:
             raise AuthError(403, "Host not in room.")
 
-        yield self._verify_events_in_room(pdu_list, room_id)
-
         events = yield self.store.get_backfill_events(
             room_id,
             pdu_list,
@@ -1723,7 +1722,9 @@ def on_query_auth(self, origin, event_id, room_id, remote_auth_chain, rejects,
         if not in_room:
             raise AuthError(403, "Host not in room.")
 
-        yield self._verify_events_in_room([event_id], room_id)
+        event = yield self.store.get_event(
+            event_id, allow_none=False, check_room_id=room_id
+        )
 
         # Just go through and process each event in `remote_auth_chain`. We
         # don't want to fall into the trap of `missing` being wrong.
@@ -1734,7 +1735,6 @@ def on_query_auth(self, origin, event_id, room_id, remote_auth_chain, rejects,
                 pass
 
         # Now get the current auth_chain for the event.
-        event = yield self.store.get_event(event_id)
         local_auth_chain = yield self.store.get_auth_chain(
             [auth_id for auth_id, _ in event.auth_events],
             include_given=True
@@ -2385,19 +2385,3 @@ def _check_key_revocation(self, public_key, url):
             )
         if "valid" not in response or not response["valid"]:
             raise AuthError(403, "Third party certificate was invalid")
-
-    @defer.inlineCallbacks
-    def _verify_events_in_room(self, pdu_ids, room_id):
-        """Checks whether the given PDU IDs are in the given room or not.
-
-        Args:
-            pdu_ids (list): list of PDU IDs
-            room_id (str): the room ID that the PDUs should be in
-
-        Raises:
-            AuthError: if one or more of the PDUs does not belong to the
-                given room.
-        """
-        room_ids = yield self.store.get_room_ids_for_events(pdu_ids)
-        if len(room_ids) != 1 or room_ids[0] != room_id:
-            raise AuthError(403, "Events must belong to the given room")
diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py
index e860fe1a1eda..7cd77c1c29a1 100644
--- a/synapse/storage/event_federation.py
+++ b/synapse/storage/event_federation.py
@@ -295,35 +295,6 @@ def get_forward_extremeties_for_room_txn(txn):
             get_forward_extremeties_for_room_txn
         )
 
-    def get_room_ids_for_events(self, event_ids):
-        """Get a list of room IDs for which the given events belong.
-
-        Args:
-            event_ids (list): the events to look up the room of
-
-        Returns:
-            list, the room IDs for the events
-        """
-        return self.runInteraction(
-            "get_room_ids_for_events",
-            self._get_room_ids_for_events, event_ids
-        )
-
-    def _get_room_ids_for_events(self, txn, event_ids):
-        logger.debug("_get_room_ids_for_events: %s", repr(event_ids))
-
-        base_sql = (
-            "SELECT DISTINCT room_id FROM events"
-            " WHERE event_id IN (%s)"
-        )
-
-        txn.execute(
-            base_sql % (",".join(["?"] * len(event_ids)),),
-            event_ids
-        )
-
-        return [r[0] for r in txn]
-
     def get_backfill_events(self, room_id, event_list, limit):
         """Get a list of Events for a given topic that occurred before (and
         including) the events in event_list. Return a list of max size `limit`
@@ -372,6 +343,7 @@ def _get_backfill_events(self, txn, room_id, event_list, limit):
                 table="events",
                 keyvalues={
                     "event_id": event_id,
+                    "room_id": room_id,
                 },
                 retcol="depth",
                 allow_none=True,
diff --git a/synapse/storage/events_worker.py b/synapse/storage/events_worker.py
index 67433606c647..6b8a8e908f7e 100644
--- a/synapse/storage/events_worker.py
+++ b/synapse/storage/events_worker.py
@@ -19,7 +19,7 @@
 
 from twisted.internet import defer
 
-from synapse.api.errors import SynapseError
+from synapse.api.errors import NotFoundError
 # these are only included to make the type annotations work
 from synapse.events import EventBase  # noqa: F401
 from synapse.events import FrozenEvent
@@ -76,7 +76,7 @@ def get_received_ts(self, event_id):
     @defer.inlineCallbacks
     def get_event(self, event_id, check_redacted=True,
                   get_prev_content=False, allow_rejected=False,
-                  allow_none=False):
+                  allow_none=False, check_room_id=None):
         """Get an event from the database by event_id.
 
         Args:
@@ -87,7 +87,9 @@ def get_event(self, event_id, check_redacted=True,
                 include the previous states content in the unsigned field.
             allow_rejected (bool): If True return rejected events.
             allow_none (bool): If True, return None if no event found, if
-                False throw an exception.
+                False throw a NotFoundError
+            check_room_id (str|None): if not None, check the room of the found event.
+                If there is a mismatch, behave as per allow_none.
 
         Returns:
             Deferred : A FrozenEvent.
@@ -99,10 +101,16 @@ def get_event(self, event_id, check_redacted=True,
             allow_rejected=allow_rejected,
         )
 
-        if not events and not allow_none:
-            raise SynapseError(404, "Could not find event %s" % (event_id,))
+        event = events[0] if events else None
 
-        defer.returnValue(events[0] if events else None)
+        if event is not None and check_room_id is not None:
+            if event.room_id != check_room_id:
+                event = None
+
+        if event is None and not allow_none:
+            raise NotFoundError("Could not find event %s" % (event_id,))
+
+        defer.returnValue(event)
 
     @defer.inlineCallbacks
     def get_events(self, event_ids, check_redacted=True,

From a0134042922b7d1a128150e1be0d7213c1e9e138 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 2 Aug 2018 14:00:29 +0100
Subject: [PATCH 231/830] changelog

---
 changelog.d/3641.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3641.bugfix

diff --git a/changelog.d/3641.bugfix b/changelog.d/3641.bugfix
new file mode 100644
index 000000000000..02181975c011
--- /dev/null
+++ b/changelog.d/3641.bugfix
@@ -0,0 +1 @@
+Fix a potential event disclosure issue
\ No newline at end of file

From 0bf5ec0db700f189ba36360ea8424d9761658905 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 2 Aug 2018 15:03:27 +0100
Subject: [PATCH 232/830] Check room visibility for /event/ requests

Make sure that the user has permission to view the requeseted event for
/event/{eventId} and /room/{roomId}/event/{eventId} requests.

Also check that the event is in the given room for
/room/{roomId}/event/{eventId}, for sanity.
---
 synapse/handlers/events.py       | 25 +++++++++++++++++++++----
 synapse/rest/client/v1/events.py |  2 +-
 synapse/rest/client/v1/room.py   |  2 +-
 3 files changed, 23 insertions(+), 6 deletions(-)

diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py
index c3f2d7feff3a..f772e62c28f6 100644
--- a/synapse/handlers/events.py
+++ b/synapse/handlers/events.py
@@ -19,10 +19,12 @@
 from twisted.internet import defer
 
 from synapse.api.constants import EventTypes, Membership
+from synapse.api.errors import AuthError
 from synapse.events import EventBase
 from synapse.events.utils import serialize_event
 from synapse.types import UserID
 from synapse.util.logutils import log_function
+from synapse.visibility import filter_events_for_client
 
 from ._base import BaseHandler
 
@@ -129,11 +131,13 @@ def get_stream(self, auth_user_id, pagin_config, timeout=0,
 class EventHandler(BaseHandler):
 
     @defer.inlineCallbacks
-    def get_event(self, user, event_id):
+    def get_event(self, user, room_id, event_id):
         """Retrieve a single specified event.
 
         Args:
             user (synapse.types.UserID): The user requesting the event
+            room_id (str|None): The expected room id. We'll return None if the
+                event's room does not match.
             event_id (str): The event ID to obtain.
         Returns:
             dict: An event, or None if there is no event matching this ID.
@@ -142,13 +146,26 @@ def get_event(self, user, event_id):
             AuthError if the user does not have the rights to inspect this
             event.
         """
-        event = yield self.store.get_event(event_id)
+        event = yield self.store.get_event(event_id, check_room_id=room_id)
 
         if not event:
             defer.returnValue(None)
             return
 
-        if hasattr(event, "room_id"):
-            yield self.auth.check_joined_room(event.room_id, user.to_string())
+        users = yield self.store.get_users_in_room(event.room_id)
+        is_peeking = user.to_string() not in users
+
+        filtered = yield filter_events_for_client(
+            self.store,
+            user.to_string(),
+            [event],
+            is_peeking=is_peeking
+        )
+
+        if not filtered:
+            raise AuthError(
+                403,
+                "You don't have permission to access that event."
+            )
 
         defer.returnValue(event)
diff --git a/synapse/rest/client/v1/events.py b/synapse/rest/client/v1/events.py
index b70c9c280682..0f3a2e8b5104 100644
--- a/synapse/rest/client/v1/events.py
+++ b/synapse/rest/client/v1/events.py
@@ -88,7 +88,7 @@ def __init__(self, hs):
     @defer.inlineCallbacks
     def on_GET(self, request, event_id):
         requester = yield self.auth.get_user_by_req(request)
-        event = yield self.event_handler.get_event(requester.user, event_id)
+        event = yield self.event_handler.get_event(requester.user, None, event_id)
 
         time_now = self.clock.time_msec()
         if event:
diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py
index 3d6244785427..2a679ac8303e 100644
--- a/synapse/rest/client/v1/room.py
+++ b/synapse/rest/client/v1/room.py
@@ -508,7 +508,7 @@ def __init__(self, hs):
     @defer.inlineCallbacks
     def on_GET(self, request, room_id, event_id):
         requester = yield self.auth.get_user_by_req(request)
-        event = yield self.event_handler.get_event(requester.user, event_id)
+        event = yield self.event_handler.get_event(requester.user, room_id, event_id)
 
         time_now = self.clock.time_msec()
         if event:

From 8cefc690c9206c8f0936b4cd6a5b0823d9c7beb6 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 2 Aug 2018 15:11:19 +0100
Subject: [PATCH 233/830] changelogs

---
 changelog.d/3641.bugfix | 2 +-
 changelog.d/3642.bugfix | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/3642.bugfix

diff --git a/changelog.d/3641.bugfix b/changelog.d/3641.bugfix
index 02181975c011..149349baa23c 100644
--- a/changelog.d/3641.bugfix
+++ b/changelog.d/3641.bugfix
@@ -1 +1 @@
-Fix a potential event disclosure issue
\ No newline at end of file
+Fix a potential issue where servers could request events for rooms they have not joined.
diff --git a/changelog.d/3642.bugfix b/changelog.d/3642.bugfix
new file mode 100644
index 000000000000..e2e9b209def6
--- /dev/null
+++ b/changelog.d/3642.bugfix
@@ -0,0 +1 @@
+Fix a potential issue where users could see events in private joins before they joined

From 14a4e7d5a4e3c90195a907aa9677c058f2932cdb Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 2 Aug 2018 15:31:04 +0100
Subject: [PATCH 234/830] Prepare 0.33.1

---
 CHANGES.rst             | 10 ++++++++++
 changelog.d/3641.bugfix |  1 -
 changelog.d/3642.bugfix |  1 -
 synapse/__init__.py     |  2 +-
 4 files changed, 11 insertions(+), 3 deletions(-)
 delete mode 100644 changelog.d/3641.bugfix
 delete mode 100644 changelog.d/3642.bugfix

diff --git a/CHANGES.rst b/CHANGES.rst
index da6e0eb195fd..dff195be6ef8 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,3 +1,13 @@
+Synapse 0.33.1 (2018-08-02)
+===========================
+
+Bugfixes
+--------
+
+- Fix a potential issue where servers could request events for rooms they have not joined. (`#3641 <https://github.com/matrix-org/synapse/issues/3641>`_)
+- Fix a potential issue where users could see events in private joins before they joined (`#3642 <https://github.com/matrix-org/synapse/issues/3642>`_)
+
+
 Synapse 0.33.0 (2018-07-19)
 ===========================
 
diff --git a/changelog.d/3641.bugfix b/changelog.d/3641.bugfix
deleted file mode 100644
index 149349baa23c..000000000000
--- a/changelog.d/3641.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a potential issue where servers could request events for rooms they have not joined.
diff --git a/changelog.d/3642.bugfix b/changelog.d/3642.bugfix
deleted file mode 100644
index e2e9b209def6..000000000000
--- a/changelog.d/3642.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a potential issue where users could see events in private joins before they joined
diff --git a/synapse/__init__.py b/synapse/__init__.py
index 5c0f2f83aab0..1810cb6fcd3f 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -17,4 +17,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.33.0"
+__version__ = "0.33.1"

From db1f33fb363e827d0b9225a9544cb512bbfa2886 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 2 Aug 2018 15:33:53 +0100
Subject: [PATCH 235/830] fix changelog typos

---
 CHANGES.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index dff195be6ef8..60be8e7b5dc1 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -5,7 +5,7 @@ Bugfixes
 --------
 
 - Fix a potential issue where servers could request events for rooms they have not joined. (`#3641 <https://github.com/matrix-org/synapse/issues/3641>`_)
-- Fix a potential issue where users could see events in private joins before they joined (`#3642 <https://github.com/matrix-org/synapse/issues/3642>`_)
+- Fix a potential issue where users could see events in private rooms before they joined. (`#3642 <https://github.com/matrix-org/synapse/issues/3642>`_)
 
 
 Synapse 0.33.0 (2018-07-19)

From c2a83349f026c964302c6ad50a402c4cd664367f Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 2 Aug 2018 15:35:42 +0100
Subject: [PATCH 236/830] changelog: this is a security release

---
 CHANGES.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 60be8e7b5dc1..a1cc88fe21f5 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,8 +1,8 @@
 Synapse 0.33.1 (2018-08-02)
 ===========================
 
-Bugfixes
---------
+SECURITY FIXES
+--------------
 
 - Fix a potential issue where servers could request events for rooms they have not joined. (`#3641 <https://github.com/matrix-org/synapse/issues/3641>`_)
 - Fix a potential issue where users could see events in private rooms before they joined. (`#3642 <https://github.com/matrix-org/synapse/issues/3642>`_)

From 9180061b49907da363f2e3bfa0061f913d7ccf7a Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Thu, 2 Aug 2018 15:55:29 +0100
Subject: [PATCH 237/830] remove unused count_monthly_users

---
 synapse/storage/__init__.py   | 25 --------------
 tests/storage/test__init__.py | 65 -----------------------------------
 2 files changed, 90 deletions(-)
 delete mode 100644 tests/storage/test__init__.py

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 2120f46ed5be..23b4a8d76d04 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -268,31 +268,6 @@ def _count_users(txn):
 
         return self.runInteraction("count_users", _count_users)
 
-    def count_monthly_users(self):
-        """Counts the number of users who used this homeserver in the last 30 days
-
-        This method should be refactored with count_daily_users - the only
-        reason not to is waiting on definition of mau
-
-        Returns:
-            Defered[int]
-        """
-        def _count_monthly_users(txn):
-            thirty_days_ago = int(self._clock.time_msec()) - (1000 * 60 * 60 * 24 * 30)
-            sql = """
-                SELECT COALESCE(count(*), 0) FROM (
-                    SELECT user_id FROM user_ips
-                    WHERE last_seen > ?
-                    GROUP BY user_id
-                ) u
-            """
-
-            txn.execute(sql, (thirty_days_ago,))
-            count, = txn.fetchone()
-            return count
-
-        return self.runInteraction("count_monthly_users", _count_monthly_users)
-
     def count_r30_users(self):
         """
         Counts the number of 30 day retained users, defined as:-
diff --git a/tests/storage/test__init__.py b/tests/storage/test__init__.py
deleted file mode 100644
index f19cb1265ca0..000000000000
--- a/tests/storage/test__init__.py
+++ /dev/null
@@ -1,65 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2018 New Vector Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from twisted.internet import defer
-
-import tests.utils
-
-
-class InitTestCase(tests.unittest.TestCase):
-    def __init__(self, *args, **kwargs):
-        super(InitTestCase, self).__init__(*args, **kwargs)
-        self.store = None  # type: synapse.storage.DataStore
-
-    @defer.inlineCallbacks
-    def setUp(self):
-        hs = yield tests.utils.setup_test_homeserver()
-
-        hs.config.max_mau_value = 50
-        hs.config.limit_usage_by_mau = True
-        self.store = hs.get_datastore()
-        self.clock = hs.get_clock()
-
-    @defer.inlineCallbacks
-    def test_count_monthly_users(self):
-        count = yield self.store.count_monthly_users()
-        self.assertEqual(0, count)
-
-        yield self._insert_user_ips("@user:server1")
-        yield self._insert_user_ips("@user:server2")
-
-        count = yield self.store.count_monthly_users()
-        self.assertEqual(2, count)
-
-    @defer.inlineCallbacks
-    def _insert_user_ips(self, user):
-        """
-        Helper function to populate user_ips without using batch insertion infra
-        args:
-            user (str):  specify username i.e. @user:server.com
-        """
-        yield self.store._simple_upsert(
-            table="user_ips",
-            keyvalues={
-                "user_id": user,
-                "access_token": "access_token",
-                "ip": "ip",
-                "user_agent": "user_agent",
-                "device_id": "device_id",
-            },
-            values={
-                "last_seen": self.clock.time_msec(),
-            }
-        )

From 74b1d46ad9ae692774f2e9d71cbbe1cea91b4070 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Thu, 2 Aug 2018 16:57:35 +0100
Subject: [PATCH 238/830] do mau checks based on monthly_active_users table

---
 synapse/api/auth.py             | 13 ++++++
 synapse/handlers/auth.py        | 10 ++---
 synapse/handlers/register.py    | 10 ++---
 synapse/storage/client_ips.py   | 15 +++----
 tests/api/test_auth.py          | 31 +++++++++++++-
 tests/handlers/test_auth.py     |  8 ++--
 tests/handlers/test_register.py | 71 ++++++++++++++++-----------------
 7 files changed, 97 insertions(+), 61 deletions(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index d8022bcf8e95..943a488339bb 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -773,3 +773,16 @@ def check_in_room_or_world_readable(self, room_id, user_id):
             raise AuthError(
                 403, "Guest access not allowed", errcode=Codes.GUEST_ACCESS_FORBIDDEN
             )
+
+    @defer.inlineCallbacks
+    def check_auth_blocking(self, error):
+        """Checks if the user should be rejected for some external reason,
+        such as monthly active user limiting or global disable flag
+        Args:
+            error (Error): The error that should be raised if user is to be
+            blocked
+            """
+        if self.hs.config.limit_usage_by_mau is True:
+            current_mau = yield self.store.get_monthly_active_count()
+            if current_mau >= self.hs.config.max_mau_value:
+                raise error
diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py
index 184eef09d03b..8f9cff92e84b 100644
--- a/synapse/handlers/auth.py
+++ b/synapse/handlers/auth.py
@@ -913,12 +913,10 @@ def _check_mau_limits(self):
         Ensure that if mau blocking is enabled that invalid users cannot
         log in.
         """
-        if self.hs.config.limit_usage_by_mau is True:
-            current_mau = yield self.store.count_monthly_users()
-            if current_mau >= self.hs.config.max_mau_value:
-                raise AuthError(
-                    403, "MAU Limit Exceeded", errcode=Codes.MAU_LIMIT_EXCEEDED
-                )
+        error = AuthError(
+            403, "Monthly Active User limits exceeded", errcode=Codes.MAU_LIMIT_EXCEEDED
+        )
+        yield self.auth.check_auth_blocking(error)
 
 
 @attr.s
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index 289704b24162..706ed8c292db 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -540,9 +540,7 @@ def _check_mau_limits(self):
         Do not accept registrations if monthly active user limits exceeded
          and limiting is enabled
         """
-        if self.hs.config.limit_usage_by_mau is True:
-            current_mau = yield self.store.count_monthly_users()
-            if current_mau >= self.hs.config.max_mau_value:
-                raise RegistrationError(
-                    403, "MAU Limit Exceeded", Codes.MAU_LIMIT_EXCEEDED
-                )
+        error = RegistrationError(
+            403, "Monthly Active User limits exceeded", errcode=Codes.MAU_LIMIT_EXCEEDED
+        )
+        yield self.auth.check_auth_blocking(error)
diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py
index 506915a1ef65..83d64d1563c2 100644
--- a/synapse/storage/client_ips.py
+++ b/synapse/storage/client_ips.py
@@ -97,21 +97,22 @@ def insert_client_ip(self, user_id, access_token, ip, user_agent, device_id,
 
     @defer.inlineCallbacks
     def _populate_monthly_active_users(self, user_id):
+        """Checks on the state of monthly active user limits and optionally
+        add the user to the monthly active tables
+
+        Args:
+            user_id(str): the user_id to query
+        """
+
         store = self.hs.get_datastore()
-        print "entering _populate_monthly_active_users"
         if self.hs.config.limit_usage_by_mau:
-            print "self.hs.config.limit_usage_by_mau is TRUE"
             is_user_monthly_active = yield store.is_user_monthly_active(user_id)
-            print "is_user_monthly_active is %r" % is_user_monthly_active
             if is_user_monthly_active:
                 yield store.upsert_monthly_active_user(user_id)
             else:
                 count = yield store.get_monthly_active_count()
-                print "count is %d" % count
                 if count < self.hs.config.max_mau_value:
-                    print "count is less than self.hs.config.max_mau_value "
-                    res = yield store.upsert_monthly_active_user(user_id)
-                    print "upsert response is %r" % res
+                    yield store.upsert_monthly_active_user(user_id)
 
     def _update_client_ips_batch(self):
         def update():
diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py
index a82d737e71da..54bdf28663f9 100644
--- a/tests/api/test_auth.py
+++ b/tests/api/test_auth.py
@@ -21,7 +21,7 @@
 
 import synapse.handlers.auth
 from synapse.api.auth import Auth
-from synapse.api.errors import AuthError
+from synapse.api.errors import AuthError, Codes
 from synapse.types import UserID
 
 from tests import unittest
@@ -444,3 +444,32 @@ def get_user(tok):
         self.assertEqual("Guest access token used for regular user", cm.exception.msg)
 
         self.store.get_user_by_id.assert_called_with(USER_ID)
+
+    @defer.inlineCallbacks
+    def test_blocking_mau(self):
+        self.hs.config.limit_usage_by_mau = False
+        self.hs.config.max_mau_value = 50
+        lots_of_users = 100
+        small_number_of_users = 1
+
+        error = AuthError(
+            403, "MAU Limit Exceeded", errcode=Codes.MAU_LIMIT_EXCEEDED
+        )
+
+        # Ensure no error thrown
+        yield self.auth.check_auth_blocking(error)
+
+        self.hs.config.limit_usage_by_mau = True
+
+        self.store.get_monthly_active_count = Mock(
+            return_value=defer.succeed(lots_of_users)
+        )
+
+        with self.assertRaises(AuthError):
+            yield self.auth.check_auth_blocking(error)
+
+        # Ensure does not throw an error
+        self.store.get_monthly_active_count = Mock(
+            return_value=defer.succeed(small_number_of_users)
+        )
+        yield self.auth.check_auth_blocking(error)
diff --git a/tests/handlers/test_auth.py b/tests/handlers/test_auth.py
index 55eab9e9cf9e..8a9bf2d5fd02 100644
--- a/tests/handlers/test_auth.py
+++ b/tests/handlers/test_auth.py
@@ -132,14 +132,14 @@ def test_mau_limits_disabled(self):
     @defer.inlineCallbacks
     def test_mau_limits_exceeded(self):
         self.hs.config.limit_usage_by_mau = True
-        self.hs.get_datastore().count_monthly_users = Mock(
+        self.hs.get_datastore().get_monthly_active_count = Mock(
             return_value=defer.succeed(self.large_number_of_users)
         )
 
         with self.assertRaises(AuthError):
             yield self.auth_handler.get_access_token_for_user_id('user_a')
 
-        self.hs.get_datastore().count_monthly_users = Mock(
+        self.hs.get_datastore().get_monthly_active_count = Mock(
             return_value=defer.succeed(self.large_number_of_users)
         )
         with self.assertRaises(AuthError):
@@ -151,13 +151,13 @@ def test_mau_limits_exceeded(self):
     def test_mau_limits_not_exceeded(self):
         self.hs.config.limit_usage_by_mau = True
 
-        self.hs.get_datastore().count_monthly_users = Mock(
+        self.hs.get_datastore().get_monthly_active_count = Mock(
             return_value=defer.succeed(self.small_number_of_users)
         )
         # Ensure does not raise exception
         yield self.auth_handler.get_access_token_for_user_id('user_a')
 
-        self.hs.get_datastore().count_monthly_users = Mock(
+        self.hs.get_datastore().get_monthly_active_count = Mock(
             return_value=defer.succeed(self.small_number_of_users)
         )
         yield self.auth_handler.validate_short_term_login_token_and_get_user_id(
diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py
index 0937d71cf60f..6b5b8b3772a4 100644
--- a/tests/handlers/test_register.py
+++ b/tests/handlers/test_register.py
@@ -50,6 +50,10 @@ def setUp(self):
         self.hs.get_macaroon_generator = Mock(return_value=self.macaroon_generator)
         self.hs.handlers = RegistrationHandlers(self.hs)
         self.handler = self.hs.get_handlers().registration_handler
+        self.store = self.hs.get_datastore()
+        self.hs.config.max_mau_value = 50
+        self.lots_of_users = 100
+        self.small_number_of_users = 1
 
     @defer.inlineCallbacks
     def test_user_is_created_and_logged_in_if_doesnt_exist(self):
@@ -80,51 +84,44 @@ def test_if_user_exists(self):
         self.assertEquals(result_token, 'secret')
 
     @defer.inlineCallbacks
-    def test_cannot_register_when_mau_limits_exceeded(self):
-        local_part = "someone"
-        display_name = "someone"
-        requester = create_requester("@as:test")
-        store = self.hs.get_datastore()
+    def test_mau_limits_when_disabled(self):
         self.hs.config.limit_usage_by_mau = False
-        self.hs.config.max_mau_value = 50
-        lots_of_users = 100
-        small_number_users = 1
-
-        store.count_monthly_users = Mock(return_value=defer.succeed(lots_of_users))
-
         # Ensure does not throw exception
-        yield self.handler.get_or_create_user(requester, 'a', display_name)
+        yield self.handler.get_or_create_user("requester", 'a', "display_name")
 
+    @defer.inlineCallbacks
+    def test_get_or_create_user_mau_not_blocked(self):
         self.hs.config.limit_usage_by_mau = True
-
-        with self.assertRaises(RegistrationError):
-            yield self.handler.get_or_create_user(requester, 'b', display_name)
-
-        store.count_monthly_users = Mock(return_value=defer.succeed(small_number_users))
-
-        self._macaroon_mock_generator("another_secret")
-
+        self.store.count_monthly_users = Mock(
+            return_value=defer.succeed(self.small_number_of_users)
+        )
         # Ensure does not throw exception
-        yield self.handler.get_or_create_user("@neil:matrix.org", 'c', "Neil")
+        yield self.handler.get_or_create_user("@user:server", 'c', "User")
 
-        self._macaroon_mock_generator("another another secret")
-        store.count_monthly_users = Mock(return_value=defer.succeed(lots_of_users))
+    @defer.inlineCallbacks
+    def test_get_or_create_user_mau_blocked(self):
+        self.hs.config.limit_usage_by_mau = True
+        self.store.get_monthly_active_count = Mock(
+            return_value=defer.succeed(self.lots_of_users)
+        )
 
         with self.assertRaises(RegistrationError):
-            yield self.handler.register(localpart=local_part)
+            yield self.handler.get_or_create_user("requester", 'b', "display_name")
 
-        self._macaroon_mock_generator("another another secret")
-        store.count_monthly_users = Mock(return_value=defer.succeed(lots_of_users))
+    @defer.inlineCallbacks
+    def test_register_mau_blocked(self):
+        self.hs.config.limit_usage_by_mau = True
+        self.store.get_monthly_active_count = Mock(
+            return_value=defer.succeed(self.lots_of_users)
+        )
+        with self.assertRaises(RegistrationError):
+            yield self.handler.register(localpart="local_part")
 
+    @defer.inlineCallbacks
+    def test_register_saml2_mau_blocked(self):
+        self.hs.config.limit_usage_by_mau = True
+        self.store.get_monthly_active_count = Mock(
+            return_value=defer.succeed(self.lots_of_users)
+        )
         with self.assertRaises(RegistrationError):
-            yield self.handler.register_saml2(local_part)
-
-    def _macaroon_mock_generator(self, secret):
-        """
-        Reset macaroon generator in the case where the test creates multiple users
-        """
-        macaroon_generator = Mock(
-            generate_access_token=Mock(return_value=secret))
-        self.hs.get_macaroon_generator = Mock(return_value=macaroon_generator)
-        self.hs.handlers = RegistrationHandlers(self.hs)
-        self.handler = self.hs.get_handlers().registration_handler
+            yield self.handler.register_saml2(localpart="local_part")

From 0d25724419c43313f9b8345b07aaebe3eeeeee7e Mon Sep 17 00:00:00 2001
From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com>
Date: Mon, 21 May 2018 14:45:33 +0100
Subject: [PATCH 239/830] Refactor docker locations and README.

This addresses #3224
---
 README.rst                                    |   2 +-
 contrib/docker/README.md                      | 110 +---------------
 Dockerfile => docker/Dockerfile               |   2 +-
 docker/README.md                              | 122 ++++++++++++++++++
 .../docker => docker}/conf/homeserver.yaml    |   0
 {contrib/docker => docker}/conf/log.config    |   0
 {contrib/docker => docker}/start.py           |   0
 7 files changed, 126 insertions(+), 110 deletions(-)
 rename Dockerfile => docker/Dockerfile (90%)
 create mode 100644 docker/README.md
 rename {contrib/docker => docker}/conf/homeserver.yaml (100%)
 rename {contrib/docker => docker}/conf/log.config (100%)
 rename {contrib/docker => docker}/start.py (100%)

diff --git a/README.rst b/README.rst
index 5fdfad345fa7..90bd49c9a169 100644
--- a/README.rst
+++ b/README.rst
@@ -157,7 +157,7 @@ if you prefer.
 
 In case of problems, please see the _`Troubleshooting` section below.
 
-There is an offical synapse image available at https://hub.docker.com/r/matrixdotorg/synapse/tags/ which can be used with the docker-compose file available at `contrib/docker`. Further information on this including configuration options is available in `contrib/docker/README.md`.
+There is an offical synapse image available at https://hub.docker.com/r/matrixdotorg/synapse/tags/ which can be used with the docker-compose file available at `contrib/docker`. Further information on this including configuration options is available in the README on hub.docker.com.
 
 Alternatively, Andreas Peters (previously Silvio Fricke) has contributed a Dockerfile to automate a synapse server in a single Docker image, at https://hub.docker.com/r/avhost/docker-matrix/tags/
 
diff --git a/contrib/docker/README.md b/contrib/docker/README.md
index 562cdaac2b8c..10b15b040757 100644
--- a/contrib/docker/README.md
+++ b/contrib/docker/README.md
@@ -1,23 +1,5 @@
 # Synapse Docker
 
-The `matrixdotorg/synapse` Docker image will run Synapse as a single process. It does not provide a
-database server or a TURN server, you should run these separately.
-
-If you run a Postgres server, you should simply include it in the same Compose
-project or set the proper environment variables and the image will automatically
-use that server.
-
-## Build
-
-Build the docker image with the `docker-compose build` command.
-
-You may have a local Python wheel cache available, in which case copy the relevant packages in the ``cache/`` directory at the root of the project.
-
-## Run
-
-This image is designed to run either with an automatically generated configuration
-file or with a custom configuration that requires manual edition.
-
 ### Automated configuration
 
 It is recommended that you use Docker Compose to run your containers, including
@@ -54,94 +36,6 @@ Then, customize your configuration and run the server:
 docker-compose up -d
 ```
 
-### Without Compose
-
-If you do not wish to use Compose, you may still run this image using plain
-Docker commands. Note that the following is just a guideline and you may need
-to add parameters to the docker run command to account for the network situation
-with your postgres database.
-
-```
-docker run \
-    -d \
-    --name synapse \
-    -v ${DATA_PATH}:/data \
-    -e SYNAPSE_SERVER_NAME=my.matrix.host \
-    -e SYNAPSE_REPORT_STATS=yes \
-    docker.io/matrixdotorg/synapse:latest
-```
-
-## Volumes
-
-The image expects a single volume, located at ``/data``, that will hold:
-
-* temporary files during uploads;
-* uploaded media and thumbnails;
-* the SQLite database if you do not configure postgres;
-* the appservices configuration.
-
-You are free to use separate volumes depending on storage endpoints at your
-disposal. For instance, ``/data/media`` coud be stored on a large but low
-performance hdd storage while other files could be stored on high performance
-endpoints.
-
-In order to setup an application service, simply create an ``appservices``
-directory in the data volume and write the application service Yaml
-configuration file there. Multiple application services are supported.
-
-## Environment
-
-Unless you specify a custom path for the configuration file, a very generic
-file will be generated, based on the following environment settings.
-These are a good starting point for setting up your own deployment.
-
-Global settings:
-
-* ``UID``, the user id Synapse will run as [default 991]
-* ``GID``, the group id Synapse will run as [default 991]
-* ``SYNAPSE_CONFIG_PATH``, path to a custom config file
-
-If ``SYNAPSE_CONFIG_PATH`` is set, you should generate a configuration file
-then customize it manually. No other environment variable is required.
-
-Otherwise, a dynamic configuration file will be used. The following environment
-variables are available for configuration:
-
-* ``SYNAPSE_SERVER_NAME`` (mandatory), the current server public hostname.
-* ``SYNAPSE_REPORT_STATS``, (mandatory, ``yes`` or ``no``), enable anonymous
-  statistics reporting back to the Matrix project which helps us to get funding.
-* ``SYNAPSE_NO_TLS``, set this variable to disable TLS in Synapse (use this if
-  you run your own TLS-capable reverse proxy).
-* ``SYNAPSE_ENABLE_REGISTRATION``, set this variable to enable registration on
-  the Synapse instance.
-* ``SYNAPSE_ALLOW_GUEST``, set this variable to allow guest joining this server.
-* ``SYNAPSE_EVENT_CACHE_SIZE``, the event cache size [default `10K`].
-* ``SYNAPSE_CACHE_FACTOR``, the cache factor [default `0.5`].
-* ``SYNAPSE_RECAPTCHA_PUBLIC_KEY``, set this variable to the recaptcha public
-  key in order to enable recaptcha upon registration.
-* ``SYNAPSE_RECAPTCHA_PRIVATE_KEY``, set this variable to the recaptcha private
-  key in order to enable recaptcha upon registration.
-* ``SYNAPSE_TURN_URIS``, set this variable to the coma-separated list of TURN
-  uris to enable TURN for this homeserver.
-* ``SYNAPSE_TURN_SECRET``, set this to the TURN shared secret if required.
-
-Shared secrets, that will be initialized to random values if not set:
-
-* ``SYNAPSE_REGISTRATION_SHARED_SECRET``, secret for registrering users if
-  registration is disable.
-* ``SYNAPSE_MACAROON_SECRET_KEY`` secret for signing access tokens
-  to the server.
-
-Database specific values (will use SQLite if not set):
-
-* `POSTGRES_DB` - The database name for the synapse postgres database. [default: `synapse`]
-* `POSTGRES_HOST` - The host of the postgres database if you wish to use postgresql instead of sqlite3. [default: `db` which is useful when using a container on the same docker network in a compose file where the postgres service is called `db`]
-* `POSTGRES_PASSWORD` - The password for the synapse postgres database. **If this is set then postgres will be used instead of sqlite3.** [default: none] **NOTE**: You are highly encouraged to use postgresql! Please use the compose file to make it easier to deploy.
-* `POSTGRES_USER` - The user for the synapse postgres database. [default: `matrix`]
-
-Mail server specific values (will not send emails if not set):
+### More information
 
-* ``SYNAPSE_SMTP_HOST``, hostname to the mail server.
-* ``SYNAPSE_SMTP_PORT``, TCP port for accessing the mail server [default ``25``].
-* ``SYNAPSE_SMTP_USER``, username for authenticating against the mail server if any.
-* ``SYNAPSE_SMTP_PASSWORD``, password for authenticating against the mail server if any.
+For more information on required environment variables and mounts, see the main docker documentation at `docker/README.md`
diff --git a/Dockerfile b/docker/Dockerfile
similarity index 90%
rename from Dockerfile
rename to docker/Dockerfile
index 0242be5f6868..26fb3a6bfffd 100644
--- a/Dockerfile
+++ b/docker/Dockerfile
@@ -22,7 +22,7 @@ RUN cd /synapse \
         setuptools \
  && mkdir -p /synapse/cache \
  && pip install -f /synapse/cache --upgrade --process-dependency-links . \
- && mv /synapse/contrib/docker/start.py /synapse/contrib/docker/conf / \
+ && mv /synapse/docker/start.py /synapse/docker/conf / \
  && rm -rf \
         setup.cfg \
         setup.py \
diff --git a/docker/README.md b/docker/README.md
new file mode 100644
index 000000000000..f60ea49234c8
--- /dev/null
+++ b/docker/README.md
@@ -0,0 +1,122 @@
+# Synapse Docker
+
+This Docker image will run Synapse as a single process. It does not provide a database
+server or a TURN server, you should run these separately.
+
+## Run
+
+We do not currently offer a `latest` image, as this has somewhat undefined semantics.
+We instead release only tagged versions so upgrading between releases is entirely
+within your control.
+
+### Using docker-compose (easier)
+
+This image is designed to run either with an automatically generated configuration
+file or with a custom configuration that requires manual edition.
+
+An easy way to make use of this image is via docker-compose, see the (https://github.com/matrix-org/synapse/tree/develop/contrib/docker)[contrib] section of the synapse project for examples.
+
+### Without Compose (harder)
+
+If you do not wish to use Compose, you may still run this image using plain
+Docker commands. Note that the following is just a guideline and you may need
+to add parameters to the docker run command to account for the network situation
+with your postgres database.
+
+```
+docker run \
+    -d \
+    --name synapse \
+    -v ${DATA_PATH}:/data \
+    -e SYNAPSE_SERVER_NAME=my.matrix.host \
+    -e SYNAPSE_REPORT_STATS=yes \
+    docker.io/matrixdotorg/synapse:latest
+```
+
+## Volumes
+
+The image expects a single volume, located at ``/data``, that will hold:
+
+* temporary files during uploads;
+* uploaded media and thumbnails;
+* the SQLite database if you do not configure postgres;
+* the appservices configuration.
+
+You are free to use separate volumes depending on storage endpoints at your
+disposal. For instance, ``/data/media`` coud be stored on a large but low
+performance hdd storage while other files could be stored on high performance
+endpoints.
+
+In order to setup an application service, simply create an ``appservices``
+directory in the data volume and write the application service Yaml
+configuration file there. Multiple application services are supported.
+
+## Environment
+
+Unless you specify a custom path for the configuration file, a very generic
+file will be generated, based on the following environment settings.
+These are a good starting point for setting up your own deployment.
+
+Global settings:
+
+* ``UID``, the user id Synapse will run as [default 991]
+* ``GID``, the group id Synapse will run as [default 991]
+* ``SYNAPSE_CONFIG_PATH``, path to a custom config file
+
+If ``SYNAPSE_CONFIG_PATH`` is set, you should generate a configuration file
+then customize it manually. No other environment variable is required.
+
+Otherwise, a dynamic configuration file will be used. The following environment
+variables are available for configuration:
+
+* ``SYNAPSE_SERVER_NAME`` (mandatory), the current server public hostname.
+* ``SYNAPSE_REPORT_STATS``, (mandatory, ``yes`` or ``no``), enable anonymous
+  statistics reporting back to the Matrix project which helps us to get funding.
+* ``SYNAPSE_NO_TLS``, set this variable to disable TLS in Synapse (use this if
+  you run your own TLS-capable reverse proxy).
+* ``SYNAPSE_ENABLE_REGISTRATION``, set this variable to enable registration on
+  the Synapse instance.
+* ``SYNAPSE_ALLOW_GUEST``, set this variable to allow guest joining this server.
+* ``SYNAPSE_EVENT_CACHE_SIZE``, the event cache size [default `10K`].
+* ``SYNAPSE_CACHE_FACTOR``, the cache factor [default `0.5`].
+* ``SYNAPSE_RECAPTCHA_PUBLIC_KEY``, set this variable to the recaptcha public
+  key in order to enable recaptcha upon registration.
+* ``SYNAPSE_RECAPTCHA_PRIVATE_KEY``, set this variable to the recaptcha private
+  key in order to enable recaptcha upon registration.
+* ``SYNAPSE_TURN_URIS``, set this variable to the coma-separated list of TURN
+  uris to enable TURN for this homeserver.
+* ``SYNAPSE_TURN_SECRET``, set this to the TURN shared secret if required.
+
+Shared secrets, that will be initialized to random values if not set:
+
+* ``SYNAPSE_REGISTRATION_SHARED_SECRET``, secret for registrering users if
+  registration is disable.
+* ``SYNAPSE_MACAROON_SECRET_KEY`` secret for signing access tokens
+  to the server.
+
+Database specific values (will use SQLite if not set):
+
+* `POSTGRES_DB` - The database name for the synapse postgres database. [default: `synapse`]
+* `POSTGRES_HOST` - The host of the postgres database if you wish to use postgresql instead of sqlite3. [default: `db` which is useful when using a container on the same docker network in a compose file where the postgres service is called `db`]
+* `POSTGRES_PASSWORD` - The password for the synapse postgres database. **If this is set then postgres will be used instead of sqlite3.** [default: none] **NOTE**: You are highly encouraged to use postgresql! Please use the compose file to make it easier to deploy.
+* `POSTGRES_USER` - The user for the synapse postgres database. [default: `matrix`]
+
+Mail server specific values (will not send emails if not set):
+
+* ``SYNAPSE_SMTP_HOST``, hostname to the mail server.
+* ``SYNAPSE_SMTP_PORT``, TCP port for accessing the mail server [default ``25``].
+* ``SYNAPSE_SMTP_USER``, username for authenticating against the mail server if any.
+* ``SYNAPSE_SMTP_PASSWORD``, password for authenticating against the mail server if any.
+
+## Build
+
+Build the docker image with the `docker build` command from the root of the synapse repository.
+
+```
+docker build -t docker.io/matrixdotorg/synapse . -f docker/Dockerfile
+```
+
+The `-t` option sets the image tag. Official images are tagged `matrixdotorg/synapse:<version>` where `<version>` is the same as the release tag in the synapse git repository.
+
+You may have a local Python wheel cache available, in which case copy the relevant
+packages in the ``cache/`` directory at the root of the project.
diff --git a/contrib/docker/conf/homeserver.yaml b/docker/conf/homeserver.yaml
similarity index 100%
rename from contrib/docker/conf/homeserver.yaml
rename to docker/conf/homeserver.yaml
diff --git a/contrib/docker/conf/log.config b/docker/conf/log.config
similarity index 100%
rename from contrib/docker/conf/log.config
rename to docker/conf/log.config
diff --git a/contrib/docker/start.py b/docker/start.py
similarity index 100%
rename from contrib/docker/start.py
rename to docker/start.py

From 26a37f3d4dcd2257005d9afe1d90af470198e111 Mon Sep 17 00:00:00 2001
From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com>
Date: Fri, 8 Jun 2018 13:35:07 +0100
Subject: [PATCH 240/830] Do not include docker files in python build

---
 MANIFEST.in | 1 +
 1 file changed, 1 insertion(+)

diff --git a/MANIFEST.in b/MANIFEST.in
index 7076b608d4c7..1ff98d95dff0 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -35,3 +35,4 @@ recursive-include changelog.d *
 
 prune .github
 prune demo/etc
+prune docker

From 1758f4e1c793cd5f219469c9d5c340eea071b6c8 Mon Sep 17 00:00:00 2001
From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com>
Date: Fri, 8 Jun 2018 13:37:39 +0100
Subject: [PATCH 241/830] Address SPAG issues

---
 docker/README.md | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/docker/README.md b/docker/README.md
index f60ea49234c8..8303a7fecd9c 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -12,9 +12,11 @@ within your control.
 ### Using docker-compose (easier)
 
 This image is designed to run either with an automatically generated configuration
-file or with a custom configuration that requires manual edition.
+file or with a custom configuration that requires manual editing.
 
-An easy way to make use of this image is via docker-compose, see the (https://github.com/matrix-org/synapse/tree/develop/contrib/docker)[contrib] section of the synapse project for examples.
+An easy way to make use of this image is via docker-compose. See the
+(https://github.com/matrix-org/synapse/tree/develop/contrib/docker)[contrib]
+section of the synapse project for examples.
 
 ### Without Compose (harder)
 

From 489949879ede04f104852f11993c734e9109cd16 Mon Sep 17 00:00:00 2001
From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com>
Date: Thu, 2 Aug 2018 18:49:50 +0100
Subject: [PATCH 242/830] Add news entry

---
 changelog.d/3644.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3644.misc

diff --git a/changelog.d/3644.misc b/changelog.d/3644.misc
new file mode 100644
index 000000000000..2347fc850047
--- /dev/null
+++ b/changelog.d/3644.misc
@@ -0,0 +1 @@
+Refactor location of docker build script.

From 4c4a123bd0ab21b4e3df008c21dc8727e6230dba Mon Sep 17 00:00:00 2001
From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com>
Date: Thu, 2 Aug 2018 18:53:44 +0100
Subject: [PATCH 243/830] Mention the word "newsfragment" in CONTRIBUTING.rst

---
 CONTRIBUTING.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
index 6c295cfbfebf..aa2738eea39d 100644
--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -51,7 +51,7 @@ makes it horribly hard to review otherwise.
 Changelog
 ~~~~~~~~~
 
-All changes, even minor ones, need a corresponding changelog
+All changes, even minor ones, need a corresponding changelog / newsfragment
 entry. These are managed by Towncrier
 (https://github.com/hawkowl/towncrier).
 

From 70baa61e95ae09cc920e79bf4a9b23f278ab9f33 Mon Sep 17 00:00:00 2001
From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com>
Date: Thu, 2 Aug 2018 18:58:43 +0100
Subject: [PATCH 244/830] newsfragment

---
 changelog.d/3645.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3645.misc

diff --git a/changelog.d/3645.misc b/changelog.d/3645.misc
new file mode 100644
index 000000000000..0fe6b28da173
--- /dev/null
+++ b/changelog.d/3645.misc
@@ -0,0 +1 @@
+Update CONTRIBUTING to mention newsfragments.

From 4ecb4bdac963ae967da810de6134d85b50c999f9 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Thu, 2 Aug 2018 22:41:05 +0100
Subject: [PATCH 245/830] wip attempt at caching

---
 synapse/storage/monthly_active_users.py | 56 +++++++++++++++++++++----
 1 file changed, 47 insertions(+), 9 deletions(-)

diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py
index aada9bd2b954..19846cefd9f9 100644
--- a/synapse/storage/monthly_active_users.py
+++ b/synapse/storage/monthly_active_users.py
@@ -1,4 +1,6 @@
 from twisted.internet import defer
+from synapse.util.caches.descriptors import cachedInlineCallbacks
+from synapse.storage.engines import PostgresEngine, Sqlite3Engine
 
 from ._base import SQLBaseStore
 
@@ -20,17 +22,53 @@ def _reap_users(txn):
             thirty_days_ago = (
                 int(self._clock.time_msec()) - (1000 * 60 * 60 * 24 * 30)
             )
-            sql = "DELETE FROM monthly_active_users WHERE timestamp < ?"
-            txn.execute(sql, (thirty_days_ago,))
-            sql = """
-                DELETE FROM monthly_active_users
-                ORDER BY timestamp desc
-                LIMIT -1 OFFSET ?
-                """
-            txn.execute(sql, (self.max_mau_value,))
+
+            if isinstance(self.database_engine, PostgresEngine):
+                sql = """
+                    DELETE FROM monthly_active_users
+                    WHERE timestamp < ?
+                    RETURNING user_id
+                    """
+                txn.execute(sql, (thirty_days_ago,))
+                res = txn.fetchall()
+                for r in res:
+                    self.is_user_monthly_active.invalidate(r)
+
+                sql = """
+                    DELETE FROM monthly_active_users
+                    ORDER BY timestamp desc
+                    LIMIT -1 OFFSET ?
+                    RETURNING user_id
+                    """
+                txn.execute(sql, (self.max_mau_value,))
+                res = txn.fetchall()
+                for r in res:
+                    self.is_user_monthly_active.invalidate(r)
+                    print r
+                self.get_monthly_active_count.invalidate()
+            elif isinstance(self.database_engine, Sqlite3Engine):
+                sql = "DELETE FROM monthly_active_users WHERE timestamp < ?"
+
+                txn.execute(sql, (thirty_days_ago,))
+                sql = """
+                    DELETE FROM monthly_active_users
+                    ORDER BY timestamp desc
+                    LIMIT -1 OFFSET ?
+                    """
+                txn.execute(sql, (self.max_mau_value,))
+
+                # It seems poor to invalidate the whole cache, but the alternative
+                # is to select then delete which has it's own problem.
+                # It seems unlikely that anyone using this feature on large datasets
+                # would be using sqlite and if they are then there will be
+                # larger perf issues than this one to encourage an upgrade to postgres.
+
+                self.is_user_monthly_active.invalidate_all()
+                self.get_monthly_active_count.invalidate_all()
 
         return self.runInteraction("reap_monthly_active_users", _reap_users)
 
+    @cachedInlineCallbacks(num_args=0)
     def get_monthly_active_count(self):
         """
             Generates current count of monthly active users.abs
@@ -65,7 +103,7 @@ def upsert_monthly_active_user(self, user_id):
             lock=False,
         )
 
-    @defer.inlineCallbacks
+    @cachedInlineCallbacks(num_args=1)
     def is_user_monthly_active(self, user_id):
         """
             Checks if a given user is part of the monthly active user group

From 5e2f7b80844bc8ee401beab494948c093338f404 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Thu, 2 Aug 2018 22:47:48 +0100
Subject: [PATCH 246/830] typo

---
 synapse/storage/monthly_active_users.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py
index 19846cefd9f9..2872ba4cae1b 100644
--- a/synapse/storage/monthly_active_users.py
+++ b/synapse/storage/monthly_active_users.py
@@ -58,7 +58,7 @@ def _reap_users(txn):
                 txn.execute(sql, (self.max_mau_value,))
 
                 # It seems poor to invalidate the whole cache, but the alternative
-                # is to select then delete which has it's own problem.
+                # is to select then delete which has its own problems.
                 # It seems unlikely that anyone using this feature on large datasets
                 # would be using sqlite and if they are then there will be
                 # larger perf issues than this one to encourage an upgrade to postgres.

From c0affa7b4ff1e0775a8a5e6d704596b5383112a5 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Thu, 2 Aug 2018 23:03:01 +0100
Subject: [PATCH 247/830] update generate_monthly_active_users, and
 reap_monthly_active_users

---
 synapse/app/homeserver.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 5f0ca51ac76d..7d4ea493bcde 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -520,14 +520,14 @@ def generate_user_daily_visit_stats():
     # table will decrease
     clock.looping_call(generate_user_daily_visit_stats, 5 * 60 * 1000)
     clock.looping_call(
-        MonthlyActiveUsersStore(hs).reap_monthly_active_users, 1000 * 60 * 60
+        hs.get_datastore().reap_monthly_active_users, 1000 * 60 * 60
     )
 
     @defer.inlineCallbacks
     def generate_monthly_active_users():
         count = 0
         if hs.config.limit_usage_by_mau:
-            count = yield hs.get_datastore().count_monthly_users()
+            count = yield hs.get_datastore().get_monthly_active_count()
         current_mau_gauge.set(float(count))
         max_mau_value_gauge.set(float(hs.config.max_mau_value))
 

From feacd13932b848546f75a8f646546c3215d35316 Mon Sep 17 00:00:00 2001
From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com>
Date: Fri, 3 Aug 2018 09:10:41 +0100
Subject: [PATCH 248/830] Update README.rst

wrap at 80ish
---
 README.rst | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/README.rst b/README.rst
index 90bd49c9a169..1d602adc7e78 100644
--- a/README.rst
+++ b/README.rst
@@ -157,12 +157,19 @@ if you prefer.
 
 In case of problems, please see the _`Troubleshooting` section below.
 
-There is an offical synapse image available at https://hub.docker.com/r/matrixdotorg/synapse/tags/ which can be used with the docker-compose file available at `contrib/docker`. Further information on this including configuration options is available in the README on hub.docker.com.
+There is an offical synapse image available at 
+https://hub.docker.com/r/matrixdotorg/synapse/tags/ which can be used with
+the docker-compose file available at `contrib/docker`. Further information on
+this including configuration options is available in the README on
+hub.docker.com.
 
-Alternatively, Andreas Peters (previously Silvio Fricke) has contributed a Dockerfile to automate a synapse server in a single Docker image, at https://hub.docker.com/r/avhost/docker-matrix/tags/
+Alternatively, Andreas Peters (previously Silvio Fricke) has contributed a
+Dockerfile to automate a synapse server in a single Docker image, at
+https://hub.docker.com/r/avhost/docker-matrix/tags/
 
 Also, Martin Giess has created an auto-deployment process with vagrant/ansible,
-tested with VirtualBox/AWS/DigitalOcean - see https://github.com/EMnify/matrix-synapse-auto-deploy
+tested with VirtualBox/AWS/DigitalOcean - see 
+https://github.com/EMnify/matrix-synapse-auto-deploy
 for details.
 
 Configuring synapse

From 637b11b9edd7de93aeabb3688ab2dda6c58015a3 Mon Sep 17 00:00:00 2001
From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com>
Date: Fri, 3 Aug 2018 09:13:54 +0100
Subject: [PATCH 249/830] Update README.rst

Link to contrib/docker
---
 README.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index 1d602adc7e78..4c5971d043c8 100644
--- a/README.rst
+++ b/README.rst
@@ -159,7 +159,7 @@ In case of problems, please see the _`Troubleshooting` section below.
 
 There is an offical synapse image available at 
 https://hub.docker.com/r/matrixdotorg/synapse/tags/ which can be used with
-the docker-compose file available at `contrib/docker`. Further information on
+the docker-compose file available at `contrib/docker <contrib/docker>`_. Further information on
 this including configuration options is available in the README on
 hub.docker.com.
 

From c3f596180f19299aee6735cb9c91e3f61d073ccc Mon Sep 17 00:00:00 2001
From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com>
Date: Fri, 3 Aug 2018 09:15:19 +0100
Subject: [PATCH 250/830] Update README.md

Link to docker/README.md
---
 contrib/docker/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/docker/README.md b/contrib/docker/README.md
index 10b15b040757..05254e51927b 100644
--- a/contrib/docker/README.md
+++ b/contrib/docker/README.md
@@ -38,4 +38,4 @@ docker-compose up -d
 
 ### More information
 
-For more information on required environment variables and mounts, see the main docker documentation at `docker/README.md`
+For more information on required environment variables and mounts, see the main docker documentation at [/docker/README.md](../../docker/README.md)

From 42960aa0475edc59f83d6a47c423d9e5f12e9417 Mon Sep 17 00:00:00 2001
From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com>
Date: Fri, 3 Aug 2018 09:16:01 +0100
Subject: [PATCH 251/830] Update README.md

Link to contrib/docker
---
 docker/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/README.md b/docker/README.md
index 8303a7fecd9c..038c78f7c052 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -15,7 +15,7 @@ This image is designed to run either with an automatically generated configurati
 file or with a custom configuration that requires manual editing.
 
 An easy way to make use of this image is via docker-compose. See the
-(https://github.com/matrix-org/synapse/tree/develop/contrib/docker)[contrib]
+[contrib/docker](../contrib/docker)
 section of the synapse project for examples.
 
 ### Without Compose (harder)

From 950807d93a264b6d10ece386d227dc4069f7d0da Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Fri, 3 Aug 2018 13:49:53 +0100
Subject: [PATCH 252/830] fix caching and tests

---
 synapse/app/homeserver.py                  |  1 -
 synapse/storage/monthly_active_users.py    | 91 ++++++++++------------
 tests/storage/test_monthly_active_users.py | 50 ++++++++----
 3 files changed, 80 insertions(+), 62 deletions(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 7d4ea493bcde..3a67db8b30db 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -64,7 +64,6 @@
 from synapse.server import HomeServer
 from synapse.storage import are_all_users_on_domain
 from synapse.storage.engines import IncorrectDatabaseSetup, create_engine
-from synapse.storage.monthly_active_users import MonthlyActiveUsersStore
 from synapse.storage.prepare_database import UpgradeDatabaseException, prepare_database
 from synapse.util.caches import CACHE_SIZE_FACTOR
 from synapse.util.httpresourcetree import create_resource_tree
diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py
index 2872ba4cae1b..d06c90cbcc83 100644
--- a/synapse/storage/monthly_active_users.py
+++ b/synapse/storage/monthly_active_users.py
@@ -1,6 +1,21 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from twisted.internet import defer
-from synapse.util.caches.descriptors import cachedInlineCallbacks
-from synapse.storage.engines import PostgresEngine, Sqlite3Engine
+
+from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
 
 from ._base import SQLBaseStore
 
@@ -9,7 +24,7 @@ class MonthlyActiveUsersStore(SQLBaseStore):
     def __init__(self, dbconn, hs):
         super(MonthlyActiveUsersStore, self).__init__(None, hs)
         self._clock = hs.get_clock()
-        self.max_mau_value = hs.config.max_mau_value
+        self.hs = hs
 
     def reap_monthly_active_users(self):
         """
@@ -19,62 +34,41 @@ def reap_monthly_active_users(self):
             Defered()
         """
         def _reap_users(txn):
+
             thirty_days_ago = (
                 int(self._clock.time_msec()) - (1000 * 60 * 60 * 24 * 30)
             )
 
-            if isinstance(self.database_engine, PostgresEngine):
-                sql = """
-                    DELETE FROM monthly_active_users
-                    WHERE timestamp < ?
-                    RETURNING user_id
-                    """
-                txn.execute(sql, (thirty_days_ago,))
-                res = txn.fetchall()
-                for r in res:
-                    self.is_user_monthly_active.invalidate(r)
-
-                sql = """
-                    DELETE FROM monthly_active_users
-                    ORDER BY timestamp desc
-                    LIMIT -1 OFFSET ?
-                    RETURNING user_id
-                    """
-                txn.execute(sql, (self.max_mau_value,))
-                res = txn.fetchall()
-                for r in res:
-                    self.is_user_monthly_active.invalidate(r)
-                    print r
-                self.get_monthly_active_count.invalidate()
-            elif isinstance(self.database_engine, Sqlite3Engine):
-                sql = "DELETE FROM monthly_active_users WHERE timestamp < ?"
+            sql = "DELETE FROM monthly_active_users WHERE timestamp < ?"
 
-                txn.execute(sql, (thirty_days_ago,))
-                sql = """
-                    DELETE FROM monthly_active_users
-                    ORDER BY timestamp desc
-                    LIMIT -1 OFFSET ?
-                    """
-                txn.execute(sql, (self.max_mau_value,))
+            txn.execute(sql, (thirty_days_ago,))
+            sql = """
+                DELETE FROM monthly_active_users
+                ORDER BY timestamp desc
+                LIMIT -1 OFFSET ?
+                """
+            txn.execute(sql, (self.hs.config.max_mau_value,))
 
-                # It seems poor to invalidate the whole cache, but the alternative
-                # is to select then delete which has its own problems.
-                # It seems unlikely that anyone using this feature on large datasets
-                # would be using sqlite and if they are then there will be
-                # larger perf issues than this one to encourage an upgrade to postgres.
+        res = self.runInteraction("reap_monthly_active_users", _reap_users)
+        # It seems poor to invalidate the whole cache, Postgres supports
+        # 'Returning' which would allow me to invalidate only the
+        # specific users, but sqlite has no way to do this and instead
+        # I would need to SELECT and the DELETE which without locking
+        # is racy.
+        # Have resolved to invalidate the whole cache for now and do
+        # something about it if and when the perf becomes significant
+        self.is_user_monthly_active.invalidate_all()
+        self.get_monthly_active_count.invalidate_all()
+        return res
 
-                self.is_user_monthly_active.invalidate_all()
-                self.get_monthly_active_count.invalidate_all()
-
-        return self.runInteraction("reap_monthly_active_users", _reap_users)
-
-    @cachedInlineCallbacks(num_args=0)
+    @cached(num_args=0)
     def get_monthly_active_count(self):
         """
             Generates current count of monthly active users.abs
             Return:
                 Defered(int): Number of current monthly active users
         """
+
         def _count_users(txn):
             sql = "SELECT COALESCE(count(*), 0) FROM monthly_active_users"
 
@@ -91,7 +85,7 @@ def upsert_monthly_active_user(self, user_id):
             Deferred(bool): True if a new entry was created, False if an
                 existing one was updated.
         """
-        return self._simple_upsert(
+        self._simple_upsert(
             desc="upsert_monthly_active_user",
             table="monthly_active_users",
             keyvalues={
@@ -102,6 +96,8 @@ def upsert_monthly_active_user(self, user_id):
             },
             lock=False,
         )
+        self.is_user_monthly_active.invalidate((user_id,))
+        self.get_monthly_active_count.invalidate(())
 
     @cachedInlineCallbacks(num_args=1)
     def is_user_monthly_active(self, user_id):
@@ -120,5 +116,4 @@ def is_user_monthly_active(self, user_id):
             retcol="user_id",
             desc="is_user_monthly_active",
         )
-
         defer.returnValue(bool(user_present))
diff --git a/tests/storage/test_monthly_active_users.py b/tests/storage/test_monthly_active_users.py
index d8d25a606911..c32109ecc5d9 100644
--- a/tests/storage/test_monthly_active_users.py
+++ b/tests/storage/test_monthly_active_users.py
@@ -1,6 +1,19 @@
-from twisted.internet import defer
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
-from synapse.storage.monthly_active_users import MonthlyActiveUsersStore
+from twisted.internet import defer
 
 import tests.unittest
 import tests.utils
@@ -10,20 +23,19 @@
 class MonthlyActiveUsersTestCase(tests.unittest.TestCase):
     def __init__(self, *args, **kwargs):
         super(MonthlyActiveUsersTestCase, self).__init__(*args, **kwargs)
-        self.mau = None
 
     @defer.inlineCallbacks
     def setUp(self):
-        hs = yield setup_test_homeserver()
-        self.mau = MonthlyActiveUsersStore(None, hs)
+        self.hs = yield setup_test_homeserver()
+        self.store = self.hs.get_datastore()
 
     @defer.inlineCallbacks
     def test_can_insert_and_count_mau(self):
-        count = yield self.mau.get_monthly_active_count()
+        count = yield self.store.get_monthly_active_count()
         self.assertEqual(0, count)
 
-        yield self.mau.upsert_monthly_active_user("@user:server")
-        count = yield self.mau.get_monthly_active_count()
+        yield self.store.upsert_monthly_active_user("@user:server")
+        count = yield self.store.get_monthly_active_count()
 
         self.assertEqual(1, count)
 
@@ -32,11 +44,23 @@ def test_is_user_monthly_active(self):
         user_id1 = "@user1:server"
         user_id2 = "@user2:server"
         user_id3 = "@user3:server"
-        result = yield self.mau.is_user_monthly_active(user_id1)
+        result = yield self.store.is_user_monthly_active(user_id1)
         self.assertFalse(result)
-        yield self.mau.upsert_monthly_active_user(user_id1)
-        yield self.mau.upsert_monthly_active_user(user_id2)
-        result = yield self.mau.is_user_monthly_active(user_id1)
+        yield self.store.upsert_monthly_active_user(user_id1)
+        yield self.store.upsert_monthly_active_user(user_id2)
+        result = yield self.store.is_user_monthly_active(user_id1)
         self.assertTrue(result)
-        result = yield self.mau.is_user_monthly_active(user_id3)
+        result = yield self.store.is_user_monthly_active(user_id3)
         self.assertFalse(result)
+
+    @defer.inlineCallbacks
+    def test_reap_monthly_active_users(self):
+        self.hs.config.max_mau_value = 5
+        initial_users = 10
+        for i in range(initial_users):
+            yield self.store.upsert_monthly_active_user("@user%d:server" % i)
+        count = yield self.store.get_monthly_active_count()
+        self.assertTrue(count, initial_users)
+        yield self.store.reap_monthly_active_users()
+        count = yield self.store.get_monthly_active_count()
+        self.assertTrue(count, initial_users - self.hs.config.max_mau_value)

From da90337d8952999f6c2c7dbcbb006b46bce17020 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Fri, 3 Aug 2018 14:05:20 +0100
Subject: [PATCH 253/830] Add ability to limit number of monthly active users
 on the server

---
 changelog.d/3633.feature | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3633.feature

diff --git a/changelog.d/3633.feature b/changelog.d/3633.feature
new file mode 100644
index 000000000000..8007a048405b
--- /dev/null
+++ b/changelog.d/3633.feature
@@ -0,0 +1 @@
+Add ability to limit number of monthly active users on the server

From b2aab04d2c6021e6999fdc3c204db63d7c6f9334 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Fri, 3 Aug 2018 14:12:56 +0100
Subject: [PATCH 254/830] fix py3 test failure

---
 tests/storage/test_client_ips.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/storage/test_client_ips.py b/tests/storage/test_client_ips.py
index e1552510cc50..0cd290176ea0 100644
--- a/tests/storage/test_client_ips.py
+++ b/tests/storage/test_client_ips.py
@@ -34,6 +34,7 @@ def setUp(self):
 
     @defer.inlineCallbacks
     def test_insert_new_client_ip(self):
+        self.hs.config.max_mau_value = 50
         self.clock.now = 12345678
         user_id = "@user:id"
         yield self.store.insert_client_ip(

From 5593ff6773c7556b85514ee172c4b6c06898f35e Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Fri, 3 Aug 2018 14:59:17 +0100
Subject: [PATCH 255/830] fix (lots of) py3 test failures

---
 synapse/config/server.py         | 4 ++--
 tests/storage/test_client_ips.py | 1 -
 tests/utils.py                   | 1 +
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/synapse/config/server.py b/synapse/config/server.py
index 6a471a0a5ec7..8fd23197590c 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -69,12 +69,12 @@ def read_config(self, config):
 
         # Options to control access by tracking MAU
         self.limit_usage_by_mau = config.get("limit_usage_by_mau", False)
+        self.max_mau_value = 0
         if self.limit_usage_by_mau:
             self.max_mau_value = config.get(
                 "max_mau_value", 0,
             )
-        else:
-            self.max_mau_value = 0
+
         # FIXME: federation_domain_whitelist needs sytests
         self.federation_domain_whitelist = None
         federation_domain_whitelist = config.get(
diff --git a/tests/storage/test_client_ips.py b/tests/storage/test_client_ips.py
index 0cd290176ea0..e1552510cc50 100644
--- a/tests/storage/test_client_ips.py
+++ b/tests/storage/test_client_ips.py
@@ -34,7 +34,6 @@ def setUp(self):
 
     @defer.inlineCallbacks
     def test_insert_new_client_ip(self):
-        self.hs.config.max_mau_value = 50
         self.clock.now = 12345678
         user_id = "@user:id"
         yield self.store.insert_client_ip(
diff --git a/tests/utils.py b/tests/utils.py
index 9bff3ff3b9b5..ec40428e7438 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -73,6 +73,7 @@ def setup_test_homeserver(name="test", datastore=None, config=None, reactor=None
         config.block_events_without_consent_error = None
         config.media_storage_providers = []
         config.auto_join_rooms = []
+        config.limit_usage_by_mau = False
 
         # disable user directory updates, because they get done in the
         # background, which upsets the test runner.

From 15c1ae45e52c31d03f162c5b201f0ccafa032885 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 3 Aug 2018 16:04:29 +0100
Subject: [PATCH 256/830] Docstrings for BaseFederationServlet

... to save me reverse-engineering this stuff again.
---
 synapse/federation/transport/server.py | 47 ++++++++++++++++++++++++++
 1 file changed, 47 insertions(+)

diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index eae5f2b427bb..93bd899b865c 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -190,6 +190,41 @@ def strip_quotes(value):
 
 
 class BaseFederationServlet(object):
+    """Abstract base class for federation servlet classes.
+
+    The servlet object should have a PATH attribute which takes the form of a regexp to
+    match against the request path (excluding the /federation/v1 prefix).
+
+    The servlet should also implement one or more of on_GET, on_POST, on_PUT, to match
+    the appropriate HTTP method. These methods have the signature:
+
+        on_<METHOD>(self, origin, content, query, **kwargs)
+
+        With arguments:
+
+            origin (unicode|None): The authenticated server_name of the calling server,
+                unless REQUIRE_AUTH is set to False and authentication failed.
+
+            content (unicode|None): decoded json body of the request. None if the
+                request was a GET.
+
+            query (dict[bytes, list[bytes]]): Query params from the request. url-decoded
+                (ie, '+' and '%xx' are decoded) but note that it is *not* utf8-decoded
+                yet.
+
+            **kwargs (dict[unicode, unicode]): the dict mapping keys to path
+                components as specified in the path match regexp.
+
+        Returns:
+            Deferred[(int, object)|None]: either (response code, response object) to
+                 return a JSON response, or None if the request has already been handled.
+
+        Raises:
+            SynapseError: to return an error code
+
+            Exception: other exceptions will be caught, logged, and a 500 will be
+                returned.
+    """
     REQUIRE_AUTH = True
 
     def __init__(self, handler, authenticator, ratelimiter, server_name):
@@ -204,6 +239,18 @@ def _wrap(self, func):
         @defer.inlineCallbacks
         @functools.wraps(func)
         def new_func(request, *args, **kwargs):
+            """ A callback which can be passed to HttpServer.RegisterPaths
+
+            Args:
+                request (twisted.web.http.Request):
+                *args: unused?
+                **kwargs (dict[unicode, unicode]): the dict mapping keys to path
+                    components as specified in the path match regexp.
+
+            Returns:
+                Deferred[(int, object)|None]: (response code, response object) as returned
+                    by the callback method. None if the request has already been handled.
+            """
             content = None
             if request.method in ["PUT", "POST"]:
                 # TODO: Handle other method types? other content types?

From 0ca459ea334ff86016bda241c0c823178789c215 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 25 Jul 2018 22:10:39 +0100
Subject: [PATCH 257/830] Basic support for room versioning

This is the first tranche of support for room versioning. It includes:
 * setting the default room version in the config file
 * new room_version param on the createRoom API
 * storing the version of newly-created rooms in the m.room.create event
 * fishing the version of existing rooms out of the m.room.create event
---
 synapse/api/constants.py                    |  6 ++++
 synapse/api/errors.py                       |  2 ++
 synapse/config/server.py                    | 14 +++++++++
 synapse/handlers/room.py                    | 27 ++++++++++++++++-
 synapse/replication/slave/storage/events.py |  2 +-
 synapse/storage/state.py                    | 33 +++++++++++++++++++--
 tests/utils.py                              |  4 +++
 7 files changed, 83 insertions(+), 5 deletions(-)

diff --git a/synapse/api/constants.py b/synapse/api/constants.py
index 4df930c8d1b1..a27bf3b32d92 100644
--- a/synapse/api/constants.py
+++ b/synapse/api/constants.py
@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2017 Vector Creations Ltd
+# Copyright 2018 New Vector Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -94,3 +95,8 @@ class RoomCreationPreset(object):
 class ThirdPartyEntityKind(object):
     USER = "user"
     LOCATION = "location"
+
+
+# vdh-test-version is a placeholder to get room versioning support working and tested
+# until we have a working v2.
+KNOWN_ROOM_VERSIONS = {"1", "vdh-test-version"}
diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index b41d595059a3..477ca07a242b 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2018 New Vector Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -56,6 +57,7 @@ class Codes(object):
     CONSENT_NOT_GIVEN = "M_CONSENT_NOT_GIVEN"
     CANNOT_LEAVE_SERVER_NOTICE_ROOM = "M_CANNOT_LEAVE_SERVER_NOTICE_ROOM"
     MAU_LIMIT_EXCEEDED = "M_MAU_LIMIT_EXCEEDED"
+    UNSUPPORTED_ROOM_VERSION = "M_UNSUPPORTED_ROOM_VERSION"
 
 
 class CodeMessageException(RuntimeError):
diff --git a/synapse/config/server.py b/synapse/config/server.py
index 6a471a0a5ec7..68ef2789d3c9 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -16,6 +16,7 @@
 
 import logging
 
+from synapse.api.constants import KNOWN_ROOM_VERSIONS
 from synapse.http.endpoint import parse_and_validate_server_name
 
 from ._base import Config, ConfigError
@@ -75,6 +76,16 @@ def read_config(self, config):
             )
         else:
             self.max_mau_value = 0
+
+        # the version of rooms created by default on this server
+        self.default_room_version = str(config.get(
+            "default_room_version", "1",
+        ))
+        if self.default_room_version not in KNOWN_ROOM_VERSIONS:
+            raise ConfigError("Unrecognised value '%s' for default_room_version" % (
+                self.default_room_version,
+            ))
+
         # FIXME: federation_domain_whitelist needs sytests
         self.federation_domain_whitelist = None
         federation_domain_whitelist = config.get(
@@ -249,6 +260,9 @@ def default_config(self, server_name, **kwargs):
         # (except those sent by local server admins). The default is False.
         # block_non_admin_invites: True
 
+        # The room_version of rooms which are created by default by this server.
+        # default_room_version: 1
+
         # Restrict federation to the following whitelist of domains.
         # N.B. we recommend also firewalling your federation listener to limit
         # inbound federation traffic as early as possible, rather than relying
diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index 7b7804d9b2cc..a526b684e9dd 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -21,9 +21,16 @@
 import string
 from collections import OrderedDict
 
+from six import string_types
+
 from twisted.internet import defer
 
-from synapse.api.constants import EventTypes, JoinRules, RoomCreationPreset
+from synapse.api.constants import (
+    KNOWN_ROOM_VERSIONS,
+    EventTypes,
+    JoinRules,
+    RoomCreationPreset,
+)
 from synapse.api.errors import AuthError, Codes, StoreError, SynapseError
 from synapse.types import RoomAlias, RoomID, RoomStreamToken, StreamToken, UserID
 from synapse.util import stringutils
@@ -99,6 +106,21 @@ def create_room(self, requester, config, ratelimit=True,
         if ratelimit:
             yield self.ratelimit(requester)
 
+        room_version = config.get("room_version", self.hs.config.default_room_version)
+        if not isinstance(room_version, string_types):
+            raise SynapseError(
+                400,
+                "room_version must be a string",
+                Codes.BAD_JSON,
+            )
+
+        if room_version not in KNOWN_ROOM_VERSIONS:
+            raise SynapseError(
+                400,
+                "Your homeserver does not support this room version",
+                Codes.UNSUPPORTED_ROOM_VERSION,
+            )
+
         if "room_alias_name" in config:
             for wchar in string.whitespace:
                 if wchar in config["room_alias_name"]:
@@ -184,6 +206,9 @@ def create_room(self, requester, config, ratelimit=True,
 
         creation_content = config.get("creation_content", {})
 
+        # override any attempt to set room versions via the creation_content
+        creation_content["room_version"] = room_version
+
         room_member_handler = self.hs.get_room_member_handler()
 
         yield self._send_events_for_new_room(
diff --git a/synapse/replication/slave/storage/events.py b/synapse/replication/slave/storage/events.py
index bdb5eee4afc5..4830c68f3575 100644
--- a/synapse/replication/slave/storage/events.py
+++ b/synapse/replication/slave/storage/events.py
@@ -44,8 +44,8 @@ class SlavedEventStore(EventFederationWorkerStore,
                        RoomMemberWorkerStore,
                        EventPushActionsWorkerStore,
                        StreamWorkerStore,
-                       EventsWorkerStore,
                        StateGroupWorkerStore,
+                       EventsWorkerStore,
                        SignatureWorkerStore,
                        UserErasureWorkerStore,
                        BaseSlavedStore):
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index b27b3ae14458..17b14d464baf 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -21,15 +21,17 @@
 
 from twisted.internet import defer
 
+from synapse.api.constants import EventTypes
+from synapse.api.errors import NotFoundError
+from synapse.storage._base import SQLBaseStore
 from synapse.storage.background_updates import BackgroundUpdateStore
 from synapse.storage.engines import PostgresEngine
+from synapse.storage.events_worker import EventsWorkerStore
 from synapse.util.caches import get_cache_factor_for, intern_string
 from synapse.util.caches.descriptors import cached, cachedList
 from synapse.util.caches.dictionary_cache import DictionaryCache
 from synapse.util.stringutils import to_ascii
 
-from ._base import SQLBaseStore
-
 logger = logging.getLogger(__name__)
 
 
@@ -46,7 +48,8 @@ def __len__(self):
         return len(self.delta_ids) if self.delta_ids else 0
 
 
-class StateGroupWorkerStore(SQLBaseStore):
+# this inherits from EventsWorkerStore because it calls self.get_events
+class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore):
     """The parts of StateGroupStore that can be called from workers.
     """
 
@@ -61,6 +64,30 @@ def __init__(self, db_conn, hs):
             "*stateGroupCache*", 500000 * get_cache_factor_for("stateGroupCache")
         )
 
+    @defer.inlineCallbacks
+    def get_room_version(self, room_id):
+        """Get the room_version of a given room
+
+        Args:
+            room_id (str)
+
+        Returns:
+            Deferred[str]
+
+        Raises:
+            NotFoundError if the room is unknown
+        """
+        # for now we do this by looking at the create event. We may want to cache this
+        # more intelligently in future.
+        state_ids = yield self.get_current_state_ids(room_id)
+        create_id = state_ids.get((EventTypes.Create, ""))
+
+        if not create_id:
+            raise NotFoundError("Unknown room")
+
+        create_event = yield self.get_event(create_id)
+        defer.returnValue(create_event.content.get("room_version", "1"))
+
     @cached(max_entries=100000, iterable=True)
     def get_current_state_ids(self, room_id):
         """Get the current state event ids for a room based on the
diff --git a/tests/utils.py b/tests/utils.py
index 9bff3ff3b9b5..9e188a8ed4e9 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -74,6 +74,10 @@ def setup_test_homeserver(name="test", datastore=None, config=None, reactor=None
         config.media_storage_providers = []
         config.auto_join_rooms = []
 
+        # we need a sane default_room_version, otherwise attempts to create rooms will
+        # fail.
+        config.default_room_version = "1"
+
         # disable user directory updates, because they get done in the
         # background, which upsets the test runner.
         config.update_user_directory = False

From 0d63d93ca834771324f0c5b9340346ad2113a4b1 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 25 Jul 2018 22:25:41 +0100
Subject: [PATCH 258/830] Enforce compatibility when processing make_join
 requests

Reject make_join requests from servers which do not support the room version.

Also include the room version in the response.
---
 synapse/api/errors.py                   | 22 ++++++++++++++++++++++
 synapse/federation/federation_server.py | 21 ++++++++++++++++++---
 synapse/federation/transport/server.py  | 24 +++++++++++++++++++++++-
 3 files changed, 63 insertions(+), 4 deletions(-)

diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index 477ca07a242b..70400347bc08 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -58,6 +58,7 @@ class Codes(object):
     CANNOT_LEAVE_SERVER_NOTICE_ROOM = "M_CANNOT_LEAVE_SERVER_NOTICE_ROOM"
     MAU_LIMIT_EXCEEDED = "M_MAU_LIMIT_EXCEEDED"
     UNSUPPORTED_ROOM_VERSION = "M_UNSUPPORTED_ROOM_VERSION"
+    INCOMPATIBLE_ROOM_VERSION = "M_INCOMPATIBLE_ROOM_VERSION"
 
 
 class CodeMessageException(RuntimeError):
@@ -287,6 +288,27 @@ def error_dict(self):
         )
 
 
+class IncompatibleRoomVersionError(SynapseError):
+    """A server is trying to join a room whose version it does not support."""
+
+    def __init__(self, room_version):
+        super(IncompatibleRoomVersionError, self).__init__(
+            code=400,
+            msg="Your homeserver does not support the features required to "
+                "join this room",
+            errcode=Codes.INCOMPATIBLE_ROOM_VERSION,
+        )
+
+        self._room_version = room_version
+
+    def error_dict(self):
+        return cs_error(
+            self.msg,
+            self.errcode,
+            room_version=self._room_version,
+        )
+
+
 def cs_error(msg, code=Codes.UNKNOWN, **kwargs):
     """ Utility method for constructing an error response for client-server
     interactions.
diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index bf89d568afed..2b62f687b683 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -27,7 +27,13 @@
 from twisted.python import failure
 
 from synapse.api.constants import EventTypes
-from synapse.api.errors import AuthError, FederationError, NotFoundError, SynapseError
+from synapse.api.errors import (
+    AuthError,
+    FederationError,
+    IncompatibleRoomVersionError,
+    NotFoundError,
+    SynapseError,
+)
 from synapse.crypto.event_signing import compute_event_signature
 from synapse.federation.federation_base import FederationBase, event_from_pdu_json
 from synapse.federation.persistence import TransactionActions
@@ -323,12 +329,21 @@ def on_query_request(self, query_type, args):
         defer.returnValue((200, resp))
 
     @defer.inlineCallbacks
-    def on_make_join_request(self, origin, room_id, user_id):
+    def on_make_join_request(self, origin, room_id, user_id, supported_versions):
         origin_host, _ = parse_server_name(origin)
         yield self.check_server_matches_acl(origin_host, room_id)
+
+        room_version = yield self.store.get_room_version(room_id)
+        if room_version not in supported_versions:
+            logger.warn("Room version %s not in %s", room_version, supported_versions)
+            raise IncompatibleRoomVersionError(room_version=room_version)
+
         pdu = yield self.handler.on_make_join_request(room_id, user_id)
         time_now = self._clock.time_msec()
-        defer.returnValue({"event": pdu.get_pdu_json(time_now)})
+        defer.returnValue({
+            "event": pdu.get_pdu_json(time_now),
+            "room_version": room_version,
+        })
 
     @defer.inlineCallbacks
     def on_invite_request(self, origin, content):
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 93bd899b865c..77969a4f3804 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -431,9 +431,31 @@ class FederationMakeJoinServlet(BaseFederationServlet):
     PATH = "/make_join/(?P<context>[^/]*)/(?P<user_id>[^/]*)"
 
     @defer.inlineCallbacks
-    def on_GET(self, origin, content, query, context, user_id):
+    def on_GET(self, origin, _content, query, context, user_id):
+        """
+        Args:
+            origin (unicode): The authenticated server_name of the calling server
+
+            _content (None): (GETs don't have bodies)
+
+            query (dict[bytes, list[bytes]]): Query params from the request.
+
+            **kwargs (dict[unicode, unicode]): the dict mapping keys to path
+                components as specified in the path match regexp.
+
+        Returns:
+            Deferred[(int, object)|None]: either (response code, response object) to
+                 return a JSON response, or None if the request has already been handled.
+        """
+        versions = query.get(b'ver')
+        if versions is not None:
+            supported_versions = [v.decode("utf-8") for v in versions]
+        else:
+            supported_versions = ["1"]
+
         content = yield self.handler.on_make_join_request(
             origin, context, user_id,
+            supported_versions=supported_versions,
         )
         defer.returnValue((200, content))
 

From 3777fa26aaf36245168980054c4eebf96169dd13 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 1 Aug 2018 15:35:29 +0100
Subject: [PATCH 259/830] sanity check response from make_join

---
 synapse/federation/federation_client.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index 7550e11b6ec2..de4b813a1572 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -561,7 +561,9 @@ def send_request(destination):
                 destination, room_id, user_id, membership
             )
 
-            pdu_dict = ret["event"]
+            pdu_dict = ret.get("event", None)
+            if not isinstance(pdu_dict, dict):
+                raise InvalidResponseError("Bad 'event' field in response")
 
             logger.debug("Got response to make_%s: %s", membership, pdu_dict)
 

From e10830e9765eb3897da5af6ffb4809badb8e3009 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Fri, 3 Aug 2018 17:55:50 +0100
Subject: [PATCH 260/830] wip commit - tests failing

---
 synapse/api/auth.py                        |  6 +-
 synapse/storage/client_ips.py              | 21 +------
 synapse/storage/monthly_active_users.py    | 66 ++++++++++++++++------
 tests/storage/test_client_ips.py           | 12 ++--
 tests/storage/test_monthly_active_users.py | 14 ++---
 5 files changed, 66 insertions(+), 53 deletions(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 943a488339bb..d8ebbbc6e8de 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -775,7 +775,7 @@ def check_in_room_or_world_readable(self, room_id, user_id):
             )
 
     @defer.inlineCallbacks
-    def check_auth_blocking(self, error):
+    def check_auth_blocking(self):
         """Checks if the user should be rejected for some external reason,
         such as monthly active user limiting or global disable flag
         Args:
@@ -785,4 +785,6 @@ def check_auth_blocking(self, error):
         if self.hs.config.limit_usage_by_mau is True:
             current_mau = yield self.store.get_monthly_active_count()
             if current_mau >= self.hs.config.max_mau_value:
-                raise error
+                raise AuthError(
+                    403, "MAU Limit Exceeded", errcode=Codes.MAU_LIMIT_EXCEEDED
+                )
diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py
index 83d64d1563c2..2489527f2cc8 100644
--- a/synapse/storage/client_ips.py
+++ b/synapse/storage/client_ips.py
@@ -86,7 +86,7 @@ def insert_client_ip(self, user_id, access_token, ip, user_agent, device_id,
             last_seen = self.client_ip_last_seen.get(key)
         except KeyError:
             last_seen = None
-        yield self._populate_monthly_active_users(user_id)
+        yield self.populate_monthly_active_users(user_id)
         # Rate-limited inserts
         if last_seen is not None and (now - last_seen) < LAST_SEEN_GRANULARITY:
             return
@@ -95,25 +95,6 @@ def insert_client_ip(self, user_id, access_token, ip, user_agent, device_id,
 
         self._batch_row_update[key] = (user_agent, device_id, now)
 
-    @defer.inlineCallbacks
-    def _populate_monthly_active_users(self, user_id):
-        """Checks on the state of monthly active user limits and optionally
-        add the user to the monthly active tables
-
-        Args:
-            user_id(str): the user_id to query
-        """
-
-        store = self.hs.get_datastore()
-        if self.hs.config.limit_usage_by_mau:
-            is_user_monthly_active = yield store.is_user_monthly_active(user_id)
-            if is_user_monthly_active:
-                yield store.upsert_monthly_active_user(user_id)
-            else:
-                count = yield store.get_monthly_active_count()
-                if count < self.hs.config.max_mau_value:
-                    yield store.upsert_monthly_active_user(user_id)
-
     def _update_client_ips_batch(self):
         def update():
             to_update = self._batch_row_update
diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py
index d06c90cbcc83..6def6830d002 100644
--- a/synapse/storage/monthly_active_users.py
+++ b/synapse/storage/monthly_active_users.py
@@ -19,6 +19,10 @@
 
 from ._base import SQLBaseStore
 
+# Number of msec of granularity to store the monthly_active_user timestamp
+# This means it is not necessary to update the table on every request
+LAST_SEEN_GRANULARITY = 60 * 60 * 1000
+
 
 class MonthlyActiveUsersStore(SQLBaseStore):
     def __init__(self, dbconn, hs):
@@ -30,8 +34,9 @@ def reap_monthly_active_users(self):
         """
         Cleans out monthly active user table to ensure that no stale
         entries exist.
-        Return:
-            Defered()
+
+        Returns:
+            Deferred[]
         """
         def _reap_users(txn):
 
@@ -49,7 +54,7 @@ def _reap_users(txn):
                 """
             txn.execute(sql, (self.hs.config.max_mau_value,))
 
-        res = self.runInteraction("reap_monthly_active_users", _reap_users)
+        res = yield self.runInteraction("reap_monthly_active_users", _reap_users)
         # It seems poor to invalidate the whole cache, Postgres supports
         # 'Returning' which would allow me to invalidate only the
         # specific users, but sqlite has no way to do this and instead
@@ -57,16 +62,16 @@ def _reap_users(txn):
         # is racy.
         # Have resolved to invalidate the whole cache for now and do
         # something about it if and when the perf becomes significant
-        self.is_user_monthly_active.invalidate_all()
+        self._user_last_seen_monthly_active.invalidate_all()
         self.get_monthly_active_count.invalidate_all()
         return res
 
     @cached(num_args=0)
     def get_monthly_active_count(self):
-        """
-            Generates current count of monthly active users.abs
-            Return:
-                Defered(int): Number of current monthly active users
+        """Generates current count of monthly active users.abs
+
+        Returns:
+            Defered[int]: Number of current monthly active users
         """
 
         def _count_users(txn):
@@ -82,10 +87,10 @@ def upsert_monthly_active_user(self, user_id):
             Updates or inserts monthly active user member
             Arguments:
                 user_id (str): user to add/update
-            Deferred(bool): True if a new entry was created, False if an
+            Deferred[bool]: True if a new entry was created, False if an
                 existing one was updated.
         """
-        self._simple_upsert(
+        is_insert = self._simple_upsert(
             desc="upsert_monthly_active_user",
             table="monthly_active_users",
             keyvalues={
@@ -96,24 +101,49 @@ def upsert_monthly_active_user(self, user_id):
             },
             lock=False,
         )
-        self.is_user_monthly_active.invalidate((user_id,))
-        self.get_monthly_active_count.invalidate(())
+        if is_insert:
+            self._user_last_seen_monthly_active.invalidate((user_id,))
+            self.get_monthly_active_count.invalidate(())
 
     @cachedInlineCallbacks(num_args=1)
-    def is_user_monthly_active(self, user_id):
+    def _user_last_seen_monthly_active(self, user_id):
         """
             Checks if a given user is part of the monthly active user group
             Arguments:
                 user_id (str): user to add/update
             Return:
-                bool : True if user part of group, False otherwise
+                int : timestamp since last seen, None if never seen
+
         """
-        user_present = yield self._simple_select_onecol(
+        result = yield self._simple_select_onecol(
             table="monthly_active_users",
             keyvalues={
                 "user_id": user_id,
             },
-            retcol="user_id",
-            desc="is_user_monthly_active",
+            retcol="timestamp",
+            desc="_user_last_seen_monthly_active",
         )
-        defer.returnValue(bool(user_present))
+        timestamp = None
+        if len(result) > 0:
+            timestamp = result[0]
+        defer.returnValue(timestamp)
+
+    @defer.inlineCallbacks
+    def populate_monthly_active_users(self, user_id):
+        """Checks on the state of monthly active user limits and optionally
+        add the user to the monthly active tables
+
+        Args:
+            user_id(str): the user_id to query
+        """
+
+        if self.hs.config.limit_usage_by_mau:
+            last_seen_timestamp = yield self._user_last_seen_monthly_active(user_id)
+            now = self.hs.get_clock().time_msec()
+
+            if last_seen_timestamp is None:
+                count = yield self.get_monthly_active_count()
+                if count < self.hs.config.max_mau_value:
+                    yield self.upsert_monthly_active_user(user_id)
+            elif now - last_seen_timestamp > LAST_SEEN_GRANULARITY:
+                yield self.upsert_monthly_active_user(user_id)
diff --git a/tests/storage/test_client_ips.py b/tests/storage/test_client_ips.py
index e1552510cc50..7a58c6eb24e5 100644
--- a/tests/storage/test_client_ips.py
+++ b/tests/storage/test_client_ips.py
@@ -64,7 +64,7 @@ def test_disabled_monthly_active_user(self):
         yield self.store.insert_client_ip(
             user_id, "access_token", "ip", "user_agent", "device_id",
         )
-        active = yield self.store.is_user_monthly_active(user_id)
+        active = yield self.store._user_last_seen_monthly_active(user_id)
         self.assertFalse(active)
 
     @defer.inlineCallbacks
@@ -80,7 +80,7 @@ def test_adding_monthly_active_user_when_full(self):
         yield self.store.insert_client_ip(
             user_id, "access_token", "ip", "user_agent", "device_id",
         )
-        active = yield self.store.is_user_monthly_active(user_id)
+        active = yield self.store._user_last_seen_monthly_active(user_id)
         self.assertFalse(active)
 
     @defer.inlineCallbacks
@@ -88,13 +88,13 @@ def test_adding_monthly_active_user_when_space(self):
         self.hs.config.limit_usage_by_mau = True
         self.hs.config.max_mau_value = 50
         user_id = "@user:server"
-        active = yield self.store.is_user_monthly_active(user_id)
+        active = yield self.store._user_last_seen_monthly_active(user_id)
         self.assertFalse(active)
 
         yield self.store.insert_client_ip(
             user_id, "access_token", "ip", "user_agent", "device_id",
         )
-        active = yield self.store.is_user_monthly_active(user_id)
+        active = yield self.store._user_last_seen_monthly_active(user_id)
         self.assertTrue(active)
 
     @defer.inlineCallbacks
@@ -103,7 +103,7 @@ def test_updating_monthly_active_user_when_space(self):
         self.hs.config.max_mau_value = 50
         user_id = "@user:server"
 
-        active = yield self.store.is_user_monthly_active(user_id)
+        active = yield self.store._user_last_seen_monthly_active(user_id)
         self.assertFalse(active)
 
         yield self.store.insert_client_ip(
@@ -112,5 +112,5 @@ def test_updating_monthly_active_user_when_space(self):
         yield self.store.insert_client_ip(
             user_id, "access_token", "ip", "user_agent", "device_id",
         )
-        active = yield self.store.is_user_monthly_active(user_id)
+        active = yield self.store._user_last_seen_monthly_active(user_id)
         self.assertTrue(active)
diff --git a/tests/storage/test_monthly_active_users.py b/tests/storage/test_monthly_active_users.py
index c32109ecc5d9..0bfd24a7fb3d 100644
--- a/tests/storage/test_monthly_active_users.py
+++ b/tests/storage/test_monthly_active_users.py
@@ -40,18 +40,18 @@ def test_can_insert_and_count_mau(self):
         self.assertEqual(1, count)
 
     @defer.inlineCallbacks
-    def test_is_user_monthly_active(self):
+    def test__user_last_seen_monthly_active(self):
         user_id1 = "@user1:server"
         user_id2 = "@user2:server"
         user_id3 = "@user3:server"
-        result = yield self.store.is_user_monthly_active(user_id1)
-        self.assertFalse(result)
+        result = yield self.store._user_last_seen_monthly_active(user_id1)
+        self.assertFalse(result == 0)
         yield self.store.upsert_monthly_active_user(user_id1)
         yield self.store.upsert_monthly_active_user(user_id2)
-        result = yield self.store.is_user_monthly_active(user_id1)
-        self.assertTrue(result)
-        result = yield self.store.is_user_monthly_active(user_id3)
-        self.assertFalse(result)
+        result = yield self.store._user_last_seen_monthly_active(user_id1)
+        self.assertTrue(result > 0)
+        result = yield self.store._user_last_seen_monthly_active(user_id3)
+        self.assertFalse(result == 0)
 
     @defer.inlineCallbacks
     def test_reap_monthly_active_users(self):

From 16d970189299c2ef83df4107e0cf1054cfb9da42 Mon Sep 17 00:00:00 2001
From: Will Hunt <will@half-shot.uk>
Date: Fri, 3 Aug 2018 19:08:05 +0100
Subject: [PATCH 261/830] Return M_NOT_FOUND when a profile could not be found.
 (#3596)

---
 changelog.d/3585.bugfix     |  1 +
 synapse/handlers/profile.py | 89 ++++++++++++++++++++++++-------------
 2 files changed, 59 insertions(+), 31 deletions(-)
 create mode 100644 changelog.d/3585.bugfix

diff --git a/changelog.d/3585.bugfix b/changelog.d/3585.bugfix
new file mode 100644
index 000000000000..e8ae1d8cb4c9
--- /dev/null
+++ b/changelog.d/3585.bugfix
@@ -0,0 +1 @@
+Respond with M_NOT_FOUND when profiles are not found locally or over federation. Fixes #3585
diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py
index cb5c6d587e0f..9af2e8f86973 100644
--- a/synapse/handlers/profile.py
+++ b/synapse/handlers/profile.py
@@ -17,7 +17,13 @@
 
 from twisted.internet import defer
 
-from synapse.api.errors import AuthError, CodeMessageException, SynapseError
+from synapse.api.errors import (
+    AuthError,
+    CodeMessageException,
+    Codes,
+    StoreError,
+    SynapseError,
+)
 from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.types import UserID, get_domain_from_id
 
@@ -49,12 +55,17 @@ def __init__(self, hs):
     def get_profile(self, user_id):
         target_user = UserID.from_string(user_id)
         if self.hs.is_mine(target_user):
-            displayname = yield self.store.get_profile_displayname(
-                target_user.localpart
-            )
-            avatar_url = yield self.store.get_profile_avatar_url(
-                target_user.localpart
-            )
+            try:
+                displayname = yield self.store.get_profile_displayname(
+                    target_user.localpart
+                )
+                avatar_url = yield self.store.get_profile_avatar_url(
+                    target_user.localpart
+                )
+            except StoreError as e:
+                if e.code == 404:
+                    raise SynapseError(404, "Profile was not found", Codes.NOT_FOUND)
+                raise
 
             defer.returnValue({
                 "displayname": displayname,
@@ -74,7 +85,6 @@ def get_profile(self, user_id):
             except CodeMessageException as e:
                 if e.code != 404:
                     logger.exception("Failed to get displayname")
-
                 raise
 
     @defer.inlineCallbacks
@@ -85,12 +95,17 @@ def get_profile_from_cache(self, user_id):
         """
         target_user = UserID.from_string(user_id)
         if self.hs.is_mine(target_user):
-            displayname = yield self.store.get_profile_displayname(
-                target_user.localpart
-            )
-            avatar_url = yield self.store.get_profile_avatar_url(
-                target_user.localpart
-            )
+            try:
+                displayname = yield self.store.get_profile_displayname(
+                    target_user.localpart
+                )
+                avatar_url = yield self.store.get_profile_avatar_url(
+                    target_user.localpart
+                )
+            except StoreError as e:
+                if e.code == 404:
+                    raise SynapseError(404, "Profile was not found", Codes.NOT_FOUND)
+                raise
 
             defer.returnValue({
                 "displayname": displayname,
@@ -103,9 +118,14 @@ def get_profile_from_cache(self, user_id):
     @defer.inlineCallbacks
     def get_displayname(self, target_user):
         if self.hs.is_mine(target_user):
-            displayname = yield self.store.get_profile_displayname(
-                target_user.localpart
-            )
+            try:
+                displayname = yield self.store.get_profile_displayname(
+                    target_user.localpart
+                )
+            except StoreError as e:
+                if e.code == 404:
+                    raise SynapseError(404, "Profile was not found", Codes.NOT_FOUND)
+                raise
 
             defer.returnValue(displayname)
         else:
@@ -122,7 +142,6 @@ def get_displayname(self, target_user):
             except CodeMessageException as e:
                 if e.code != 404:
                     logger.exception("Failed to get displayname")
-
                 raise
             except Exception:
                 logger.exception("Failed to get displayname")
@@ -157,10 +176,14 @@ def set_displayname(self, target_user, requester, new_displayname, by_admin=Fals
     @defer.inlineCallbacks
     def get_avatar_url(self, target_user):
         if self.hs.is_mine(target_user):
-            avatar_url = yield self.store.get_profile_avatar_url(
-                target_user.localpart
-            )
-
+            try:
+                avatar_url = yield self.store.get_profile_avatar_url(
+                    target_user.localpart
+                )
+            except StoreError as e:
+                if e.code == 404:
+                    raise SynapseError(404, "Profile was not found", Codes.NOT_FOUND)
+                raise
             defer.returnValue(avatar_url)
         else:
             try:
@@ -213,16 +236,20 @@ def on_profile_query(self, args):
         just_field = args.get("field", None)
 
         response = {}
+        try:
+            if just_field is None or just_field == "displayname":
+                response["displayname"] = yield self.store.get_profile_displayname(
+                    user.localpart
+                )
 
-        if just_field is None or just_field == "displayname":
-            response["displayname"] = yield self.store.get_profile_displayname(
-                user.localpart
-            )
-
-        if just_field is None or just_field == "avatar_url":
-            response["avatar_url"] = yield self.store.get_profile_avatar_url(
-                user.localpart
-            )
+            if just_field is None or just_field == "avatar_url":
+                response["avatar_url"] = yield self.store.get_profile_avatar_url(
+                    user.localpart
+                )
+        except StoreError as e:
+            if e.code == 404:
+                raise SynapseError(404, "Profile was not found", Codes.NOT_FOUND)
+            raise
 
         defer.returnValue(response)
 

From 886be75ad1bc60e016611b453b9644e8db17a9f1 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Fri, 3 Aug 2018 22:29:03 +0100
Subject: [PATCH 262/830] bug fixes

---
 synapse/handlers/auth.py                | 15 ++-------------
 synapse/handlers/register.py            |  8 ++++----
 synapse/storage/monthly_active_users.py |  3 +--
 tests/api/test_auth.py                  | 10 +++-------
 tests/handlers/test_register.py         |  1 -
 5 files changed, 10 insertions(+), 27 deletions(-)

diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py
index 8f9cff92e84b..7ea8ce9f9479 100644
--- a/synapse/handlers/auth.py
+++ b/synapse/handlers/auth.py
@@ -520,7 +520,7 @@ def get_access_token_for_user_id(self, user_id, device_id=None):
         """
         logger.info("Logging in user %s on device %s", user_id, device_id)
         access_token = yield self.issue_access_token(user_id, device_id)
-        yield self._check_mau_limits()
+        yield self.auth.check_auth_blocking()
 
         # the device *should* have been registered before we got here; however,
         # it's possible we raced against a DELETE operation. The thing we
@@ -734,7 +734,7 @@ def issue_access_token(self, user_id, device_id=None):
 
     @defer.inlineCallbacks
     def validate_short_term_login_token_and_get_user_id(self, login_token):
-        yield self._check_mau_limits()
+        yield self.auth.check_auth_blocking()
         auth_api = self.hs.get_auth()
         user_id = None
         try:
@@ -907,17 +907,6 @@ def _do_validate_hash():
         else:
             return defer.succeed(False)
 
-    @defer.inlineCallbacks
-    def _check_mau_limits(self):
-        """
-        Ensure that if mau blocking is enabled that invalid users cannot
-        log in.
-        """
-        error = AuthError(
-            403, "Monthly Active User limits exceeded", errcode=Codes.MAU_LIMIT_EXCEEDED
-        )
-        yield self.auth.check_auth_blocking(error)
-
 
 @attr.s
 class MacaroonGenerator(object):
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index 706ed8c292db..8cf0a36a8f8e 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -540,7 +540,7 @@ def _check_mau_limits(self):
         Do not accept registrations if monthly active user limits exceeded
          and limiting is enabled
         """
-        error = RegistrationError(
-            403, "Monthly Active User limits exceeded", errcode=Codes.MAU_LIMIT_EXCEEDED
-        )
-        yield self.auth.check_auth_blocking(error)
+        try:
+            yield self.auth.check_auth_blocking()
+        except AuthError as e:
+            raise RegistrationError(e.code, e.message, e.errcode)
diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py
index 6def6830d002..135837507a14 100644
--- a/synapse/storage/monthly_active_users.py
+++ b/synapse/storage/monthly_active_users.py
@@ -54,7 +54,7 @@ def _reap_users(txn):
                 """
             txn.execute(sql, (self.hs.config.max_mau_value,))
 
-        res = yield self.runInteraction("reap_monthly_active_users", _reap_users)
+        yield self.runInteraction("reap_monthly_active_users", _reap_users)
         # It seems poor to invalidate the whole cache, Postgres supports
         # 'Returning' which would allow me to invalidate only the
         # specific users, but sqlite has no way to do this and instead
@@ -64,7 +64,6 @@ def _reap_users(txn):
         # something about it if and when the perf becomes significant
         self._user_last_seen_monthly_active.invalidate_all()
         self.get_monthly_active_count.invalidate_all()
-        return res
 
     @cached(num_args=0)
     def get_monthly_active_count(self):
diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py
index 54bdf28663f9..e963963c730a 100644
--- a/tests/api/test_auth.py
+++ b/tests/api/test_auth.py
@@ -452,12 +452,8 @@ def test_blocking_mau(self):
         lots_of_users = 100
         small_number_of_users = 1
 
-        error = AuthError(
-            403, "MAU Limit Exceeded", errcode=Codes.MAU_LIMIT_EXCEEDED
-        )
-
         # Ensure no error thrown
-        yield self.auth.check_auth_blocking(error)
+        yield self.auth.check_auth_blocking()
 
         self.hs.config.limit_usage_by_mau = True
 
@@ -466,10 +462,10 @@ def test_blocking_mau(self):
         )
 
         with self.assertRaises(AuthError):
-            yield self.auth.check_auth_blocking(error)
+            yield self.auth.check_auth_blocking()
 
         # Ensure does not throw an error
         self.store.get_monthly_active_count = Mock(
             return_value=defer.succeed(small_number_of_users)
         )
-        yield self.auth.check_auth_blocking(error)
+        yield self.auth.check_auth_blocking()
diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py
index 6b5b8b3772a4..4ea59a58de26 100644
--- a/tests/handlers/test_register.py
+++ b/tests/handlers/test_register.py
@@ -104,7 +104,6 @@ def test_get_or_create_user_mau_blocked(self):
         self.store.get_monthly_active_count = Mock(
             return_value=defer.succeed(self.lots_of_users)
         )
-
         with self.assertRaises(RegistrationError):
             yield self.handler.get_or_create_user("requester", 'b', "display_name")
 

From d08296f9f278f39372bb5db99266d627614ecc96 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Fri, 3 Aug 2018 23:00:16 +0100
Subject: [PATCH 263/830] remove unused import

---
 tests/api/test_auth.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py
index e963963c730a..5dc33983003d 100644
--- a/tests/api/test_auth.py
+++ b/tests/api/test_auth.py
@@ -21,7 +21,7 @@
 
 import synapse.handlers.auth
 from synapse.api.auth import Auth
-from synapse.api.errors import AuthError, Codes
+from synapse.api.errors import AuthError
 from synapse.types import UserID
 
 from tests import unittest

From e40a510fbff931b5e6b295351847b95fb8c69e71 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Fri, 3 Aug 2018 23:19:13 +0100
Subject: [PATCH 264/830] py3 fix

---
 synapse/handlers/register.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index 8cf0a36a8f8e..0e16bbe0ee7a 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -543,4 +543,4 @@ def _check_mau_limits(self):
         try:
             yield self.auth.check_auth_blocking()
         except AuthError as e:
-            raise RegistrationError(e.code, e.message, e.errcode)
+            raise RegistrationError(e.code, str(e), e.errcode)

From 42c6823827a12cdb4374bc6e64d32d4ce4f526ec Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Sat, 4 Aug 2018 22:07:04 +0100
Subject: [PATCH 265/830] disable HS from config

---
 synapse/api/auth.py      |  4 ++++
 synapse/api/errors.py    |  1 +
 synapse/config/server.py |  4 ++++
 tests/api/test_auth.py   | 11 ++++++++++-
 tests/utils.py           |  2 ++
 5 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index d8ebbbc6e8de..089f166152d2 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -782,6 +782,10 @@ def check_auth_blocking(self):
             error (Error): The error that should be raised if user is to be
             blocked
             """
+        if self.hs.config.hs_disabled:
+            raise AuthError(
+                403, self.hs.config.hs_disabled_message, errcode=Codes.HS_DISABLED
+            )
         if self.hs.config.limit_usage_by_mau is True:
             current_mau = yield self.store.get_monthly_active_count()
             if current_mau >= self.hs.config.max_mau_value:
diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index b41d595059a3..466240248aeb 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -56,6 +56,7 @@ class Codes(object):
     CONSENT_NOT_GIVEN = "M_CONSENT_NOT_GIVEN"
     CANNOT_LEAVE_SERVER_NOTICE_ROOM = "M_CANNOT_LEAVE_SERVER_NOTICE_ROOM"
     MAU_LIMIT_EXCEEDED = "M_MAU_LIMIT_EXCEEDED"
+    HS_DISABLED = "M_HS_DISABLED"
 
 
 class CodeMessageException(RuntimeError):
diff --git a/synapse/config/server.py b/synapse/config/server.py
index 8fd23197590c..2e1e2f5961c4 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -75,6 +75,10 @@ def read_config(self, config):
                 "max_mau_value", 0,
             )
 
+        # Options to disable HS
+        self.hs_disabled = config.get("hs_disabled", False)
+        self.hs_disabled_message = config.get("hs_disabled_message", "")
+
         # FIXME: federation_domain_whitelist needs sytests
         self.federation_domain_whitelist = None
         federation_domain_whitelist = config.get(
diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py
index 5dc33983003d..fbb96361a8bc 100644
--- a/tests/api/test_auth.py
+++ b/tests/api/test_auth.py
@@ -21,7 +21,7 @@
 
 import synapse.handlers.auth
 from synapse.api.auth import Auth
-from synapse.api.errors import AuthError
+from synapse.api.errors import AuthError, Codes
 from synapse.types import UserID
 
 from tests import unittest
@@ -469,3 +469,12 @@ def test_blocking_mau(self):
             return_value=defer.succeed(small_number_of_users)
         )
         yield self.auth.check_auth_blocking()
+
+    @defer.inlineCallbacks
+    def test_hs_disabled(self):
+        self.hs.config.hs_disabled = True
+        self.hs.config.hs_disabled_message = "Reason for being disabled"
+        with self.assertRaises(AuthError) as e:
+            yield self.auth.check_auth_blocking()
+        self.assertEquals(e.exception.errcode, Codes.HS_DISABLED)
+        self.assertEquals(e.exception.code, 403)
diff --git a/tests/utils.py b/tests/utils.py
index ec40428e7438..a0aa38d26488 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -74,6 +74,8 @@ def setup_test_homeserver(name="test", datastore=None, config=None, reactor=None
         config.media_storage_providers = []
         config.auto_join_rooms = []
         config.limit_usage_by_mau = False
+        config.hs_disabled = False
+        config.hs_disabled_message = ""
 
         # disable user directory updates, because they get done in the
         # background, which upsets the test runner.

From f900d508244b4277065d34dd9a05224fd60d5221 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 6 Aug 2018 13:45:37 +0100
Subject: [PATCH 266/830] include known room versions in outgoing make_joins

---
 synapse/federation/federation_client.py |  8 +++++---
 synapse/federation/transport/client.py  |  5 ++++-
 synapse/handlers/federation.py          | 13 +++++++++++--
 synapse/http/matrixfederationclient.py  |  7 +++++--
 4 files changed, 25 insertions(+), 8 deletions(-)

diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index de4b813a1572..7ec1d7a88984 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -521,7 +521,7 @@ def _try_destination_list(self, description, destinations, callback):
         raise RuntimeError("Failed to %s via any server", description)
 
     def make_membership_event(self, destinations, room_id, user_id, membership,
-                              content={},):
+                              content, params):
         """
         Creates an m.room.member event, with context, without participating in the room.
 
@@ -537,8 +537,10 @@ def make_membership_event(self, destinations, room_id, user_id, membership,
             user_id (str): The user whose membership is being evented.
             membership (str): The "membership" property of the event. Must be
                 one of "join" or "leave".
-            content (object): Any additional data to put into the content field
+            content (dict): Any additional data to put into the content field
                 of the event.
+            params (dict[str, str|Iterable[str]]): Query parameters to include in the
+                request.
         Return:
             Deferred: resolves to a tuple of (origin (str), event (object))
             where origin is the remote homeserver which generated the event.
@@ -558,7 +560,7 @@ def make_membership_event(self, destinations, room_id, user_id, membership,
         @defer.inlineCallbacks
         def send_request(destination):
             ret = yield self.transport_layer.make_membership_event(
-                destination, room_id, user_id, membership
+                destination, room_id, user_id, membership, params,
             )
 
             pdu_dict = ret.get("event", None)
diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index 4529d454af46..b4fbe2c9d582 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -195,7 +195,7 @@ def make_query(self, destination, query_type, args, retry_on_dns_fail,
 
     @defer.inlineCallbacks
     @log_function
-    def make_membership_event(self, destination, room_id, user_id, membership):
+    def make_membership_event(self, destination, room_id, user_id, membership, params):
         """Asks a remote server to build and sign us a membership event
 
         Note that this does not append any events to any graphs.
@@ -205,6 +205,8 @@ def make_membership_event(self, destination, room_id, user_id, membership):
             room_id (str): room to join/leave
             user_id (str): user to be joined/left
             membership (str): one of join/leave
+            params (dict[str, str|Iterable[str]]): Query parameters to include in the
+                request.
 
         Returns:
             Deferred: Succeeds when we get a 2xx HTTP response. The result
@@ -241,6 +243,7 @@ def make_membership_event(self, destination, room_id, user_id, membership):
         content = yield self.client.get_json(
             destination=destination,
             path=path,
+            args=params,
             retry_on_dns_fail=retry_on_dns_fail,
             timeout=20000,
             ignore_backoff=ignore_backoff,
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 533b82c78361..0dffd44e22e0 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -30,7 +30,12 @@
 
 from twisted.internet import defer
 
-from synapse.api.constants import EventTypes, Membership, RejectedReason
+from synapse.api.constants import (
+    KNOWN_ROOM_VERSIONS,
+    EventTypes,
+    Membership,
+    RejectedReason,
+)
 from synapse.api.errors import (
     AuthError,
     CodeMessageException,
@@ -922,6 +927,9 @@ def do_invite_join(self, target_hosts, room_id, joinee, content):
             joinee,
             "join",
             content,
+            params={
+                "ver": KNOWN_ROOM_VERSIONS,
+            },
         )
 
         # This shouldn't happen, because the RoomMemberHandler has a
@@ -1187,13 +1195,14 @@ def do_remotely_reject_invite(self, target_hosts, room_id, user_id):
 
     @defer.inlineCallbacks
     def _make_and_verify_event(self, target_hosts, room_id, user_id, membership,
-                               content={},):
+                               content={}, params=None):
         origin, pdu = yield self.federation_client.make_membership_event(
             target_hosts,
             room_id,
             user_id,
             membership,
             content,
+            params=params,
         )
 
         logger.debug("Got response to make_%s: %s", membership, pdu)
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index bf1aa2950271..b3f5415aa660 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -439,7 +439,7 @@ def body_callback(method, url_bytes, headers_dict):
         defer.returnValue(json.loads(body))
 
     @defer.inlineCallbacks
-    def get_json(self, destination, path, args={}, retry_on_dns_fail=True,
+    def get_json(self, destination, path, args=None, retry_on_dns_fail=True,
                  timeout=None, ignore_backoff=False):
         """ GETs some json from the given host homeserver and path
 
@@ -447,7 +447,7 @@ def get_json(self, destination, path, args={}, retry_on_dns_fail=True,
             destination (str): The remote server to send the HTTP request
                 to.
             path (str): The HTTP path.
-            args (dict): A dictionary used to create query strings, defaults to
+            args (dict|None): A dictionary used to create query strings, defaults to
                 None.
             timeout (int): How long to try (in ms) the destination for before
                 giving up. None indicates no timeout and that the request will
@@ -702,6 +702,9 @@ def check_content_type_is_json(headers):
 
 
 def encode_query_args(args):
+    if args is None:
+        return b""
+
     encoded_args = {}
     for k, vs in args.items():
         if isinstance(vs, string_types):

From 051a99c4006a7deb1f9256df0a25c702dcdb451d Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 6 Aug 2018 14:29:31 +0100
Subject: [PATCH 267/830] Fix isort

---
 synapse/replication/http/_base.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/synapse/replication/http/_base.py b/synapse/replication/http/_base.py
index a7d1b2dabe59..4de3825fdab5 100644
--- a/synapse/replication/http/_base.py
+++ b/synapse/replication/http/_base.py
@@ -21,10 +21,7 @@
 
 from twisted.internet import defer
 
-from synapse.api.errors import (
-    CodeMessageException,
-    HttpResponseException,
-)
+from synapse.api.errors import CodeMessageException, HttpResponseException
 from synapse.util.caches.response_cache import ResponseCache
 from synapse.util.stringutils import random_string
 

From e26dbd82ef5f1d755be9a62165556ebce041af10 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 25 Jul 2018 16:32:05 +0100
Subject: [PATCH 268/830] Add replication APIs for persisting federation events

---
 synapse/app/federation_reader.py       |   8 +
 synapse/handlers/federation.py         |  44 ++++-
 synapse/replication/http/__init__.py   |   3 +-
 synapse/replication/http/federation.py | 245 +++++++++++++++++++++++++
 4 files changed, 290 insertions(+), 10 deletions(-)
 create mode 100644 synapse/replication/http/federation.py

diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py
index 7af00b8bcfaa..c512b4be877e 100644
--- a/synapse/app/federation_reader.py
+++ b/synapse/app/federation_reader.py
@@ -32,9 +32,13 @@
 from synapse.metrics import RegistryProxy
 from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.replication.slave.storage._base import BaseSlavedStore
+from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore
 from synapse.replication.slave.storage.directory import DirectoryStore
 from synapse.replication.slave.storage.events import SlavedEventStore
 from synapse.replication.slave.storage.keys import SlavedKeyStore
+from synapse.replication.slave.storage.push_rule import SlavedPushRuleStore
+from synapse.replication.slave.storage.pushers import SlavedPusherStore
+from synapse.replication.slave.storage.receipts import SlavedReceiptsStore
 from synapse.replication.slave.storage.room import RoomStore
 from synapse.replication.slave.storage.transactions import TransactionStore
 from synapse.replication.tcp.client import ReplicationClientHandler
@@ -49,6 +53,10 @@
 
 
 class FederationReaderSlavedStore(
+    SlavedApplicationServiceStore,
+    SlavedPusherStore,
+    SlavedPushRuleStore,
+    SlavedReceiptsStore,
     SlavedEventStore,
     SlavedKeyStore,
     RoomStore,
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 533b82c78361..0524dec942d1 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -44,6 +44,8 @@
     compute_event_signature,
 )
 from synapse.events.validator import EventValidator
+from synapse.replication.http.federation import send_federation_events_to_master
+from synapse.replication.http.membership import notify_user_membership_change
 from synapse.state import resolve_events_with_factory
 from synapse.types import UserID, get_domain_from_id
 from synapse.util import logcontext, unwrapFirstError
@@ -86,6 +88,8 @@ def __init__(self, hs):
         self.spam_checker = hs.get_spam_checker()
         self.event_creation_handler = hs.get_event_creation_handler()
         self._server_notices_mxid = hs.config.server_notices_mxid
+        self.config = hs.config
+        self.http_client = hs.get_simple_http_client()
 
         # When joining a room we need to queue any events for that room up
         self.room_queues = {}
@@ -2288,7 +2292,7 @@ def _check_key_revocation(self, public_key, url):
                 for revocation.
         """
         try:
-            response = yield self.hs.get_simple_http_client().get_json(
+            response = yield self.http_client.get_json(
                 url,
                 {"public_key": public_key}
             )
@@ -2313,14 +2317,25 @@ def _persist_events(self, event_and_contexts, backfilled=False):
         Returns:
             Deferred
         """
-        max_stream_id = yield self.store.persist_events(
-            event_and_contexts,
-            backfilled=backfilled,
-        )
+        if self.config.worker_app:
+            yield send_federation_events_to_master(
+                clock=self.hs.get_clock(),
+                store=self.store,
+                client=self.http_client,
+                host=self.config.worker_replication_host,
+                port=self.config.worker_replication_http_port,
+                event_and_contexts=event_and_contexts,
+                backfilled=backfilled
+            )
+        else:
+            max_stream_id = yield self.store.persist_events(
+                event_and_contexts,
+                backfilled=backfilled,
+            )
 
-        if not backfilled:  # Never notify for backfilled events
-            for event, _ in event_and_contexts:
-                self._notify_persisted_event(event, max_stream_id)
+            if not backfilled:  # Never notify for backfilled events
+                for event, _ in event_and_contexts:
+                    self._notify_persisted_event(event, max_stream_id)
 
     def _notify_persisted_event(self, event, max_stream_id):
         """Checks to see if notifier/pushers should be notified about the
@@ -2359,9 +2374,20 @@ def _notify_persisted_event(self, event, max_stream_id):
         )
 
     def _clean_room_for_join(self, room_id):
+        # TODO move this out to master
         return self.store.clean_room_for_join(room_id)
 
     def user_joined_room(self, user, room_id):
         """Called when a new user has joined the room
         """
-        return user_joined_room(self.distributor, user, room_id)
+        if self.config.worker_app:
+            return notify_user_membership_change(
+                client=self.http_client,
+                host=self.config.worker_replication_host,
+                port=self.config.worker_replication_http_port,
+                room_id=room_id,
+                user_id=user.to_string(),
+                change="joined",
+            )
+        else:
+            return user_joined_room(self.distributor, user, room_id)
diff --git a/synapse/replication/http/__init__.py b/synapse/replication/http/__init__.py
index 589ee94c66a0..19f214281ef6 100644
--- a/synapse/replication/http/__init__.py
+++ b/synapse/replication/http/__init__.py
@@ -14,7 +14,7 @@
 # limitations under the License.
 
 from synapse.http.server import JsonResource
-from synapse.replication.http import membership, send_event
+from synapse.replication.http import federation, membership, send_event
 
 REPLICATION_PREFIX = "/_synapse/replication"
 
@@ -27,3 +27,4 @@ def __init__(self, hs):
     def register_servlets(self, hs):
         send_event.register_servlets(hs, self)
         membership.register_servlets(hs, self)
+        federation.register_servlets(hs, self)
diff --git a/synapse/replication/http/federation.py b/synapse/replication/http/federation.py
new file mode 100644
index 000000000000..f39aaa89be57
--- /dev/null
+++ b/synapse/replication/http/federation.py
@@ -0,0 +1,245 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from twisted.internet import defer
+
+from synapse.api.constants import EventTypes, Membership
+from synapse.events import FrozenEvent
+from synapse.events.snapshot import EventContext
+from synapse.http.servlet import parse_json_object_from_request
+from synapse.replication.http._base import ReplicationEndpoint
+from synapse.types import UserID
+from synapse.util.logcontext import run_in_background
+from synapse.util.metrics import Measure
+
+logger = logging.getLogger(__name__)
+
+
+class ReplicationFederationSendEventsRestServlet(ReplicationEndpoint):
+    """Handles events newly received from federation, including persisting and
+    notifying.
+
+    The API looks like:
+
+        POST /_synapse/replication/fed_send_events/:txn_id
+
+        {
+            "events": [{
+                "event": { .. serialized event .. },
+                "internal_metadata": { .. serialized internal_metadata .. },
+                "rejected_reason": ..,   // The event.rejected_reason field
+                "context": { .. serialized event context .. },
+            }],
+            "backfilled": false
+    """
+
+    NAME = "fed_send_events"
+    PATH_ARGS = ()
+
+    def __init__(self, hs):
+        super(ReplicationFederationSendEventsRestServlet, self).__init__(hs)
+
+        self.store = hs.get_datastore()
+        self.clock = hs.get_clock()
+        self.is_mine_id = hs.is_mine_id
+        self.notifier = hs.get_notifier()
+        self.pusher_pool = hs.get_pusherpool()
+
+    @defer.inlineCallbacks
+    @staticmethod
+    def _serialize_payload(store, event_and_contexts, backfilled):
+        """
+        Args:
+            store
+            event_and_contexts (list[tuple[FrozenEvent, EventContext]])
+            backfilled (bool): Whether or not the events are the result of
+                backfilling
+        """
+        event_payloads = []
+        for event, context in event_and_contexts:
+            serialized_context = yield context.serialize(event, store)
+
+            event_payloads.append({
+                "event": event.get_pdu_json(),
+                "internal_metadata": event.internal_metadata.get_dict(),
+                "rejected_reason": event.rejected_reason,
+                "context": serialized_context,
+            })
+
+        payload = {
+            "events": event_payloads,
+            "backfilled": backfilled,
+        }
+
+        defer.returnValue(payload)
+
+    @defer.inlineCallbacks
+    def _handle_request(self, request):
+        with Measure(self.clock, "repl_fed_send_events_parse"):
+            content = parse_json_object_from_request(request)
+
+            backfilled = content["backfilled"]
+
+            event_payloads = content["events"]
+
+            event_and_contexts = []
+            for event_payload in event_payloads:
+                event_dict = event_payload["event"]
+                internal_metadata = event_payload["internal_metadata"]
+                rejected_reason = event_payload["rejected_reason"]
+                event = FrozenEvent(event_dict, internal_metadata, rejected_reason)
+
+                context = yield EventContext.deserialize(
+                    self.store, event_payload["context"],
+                )
+
+                event_and_contexts.append((event, context))
+
+        logger.info(
+            "Got %d events from federation",
+            len(event_and_contexts),
+        )
+
+        max_stream_id = yield self.store.persist_events(
+            event_and_contexts,
+            backfilled=backfilled
+        )
+
+        if not backfilled:
+            for event, _ in event_and_contexts:
+                self._notify_persisted_event(event, max_stream_id)
+
+        defer.returnValue((200, {}))
+
+    def _notify_persisted_event(self, event, max_stream_id):
+        extra_users = []
+        if event.type == EventTypes.Member:
+            target_user_id = event.state_key
+
+            # We notify for memberships if its an invite for one of our
+            # users
+            if event.internal_metadata.is_outlier():
+                if event.membership != Membership.INVITE:
+                    if not self.is_mine_id(target_user_id):
+                        return
+
+            target_user = UserID.from_string(target_user_id)
+            extra_users.append(target_user)
+        elif event.internal_metadata.is_outlier():
+            return
+
+        event_stream_id = event.internal_metadata.stream_ordering
+        self.notifier.on_new_room_event(
+            event, event_stream_id, max_stream_id,
+            extra_users=extra_users
+        )
+
+        run_in_background(
+            self.pusher_pool.on_new_notifications,
+            event_stream_id, max_stream_id,
+        )
+
+
+class ReplicationFederationSendEduRestServlet(ReplicationEndpoint):
+    """Handles EDUs newly received from federation, including persisting and
+    notifying.
+    """
+
+    NAME = "fed_send_edu"
+    PATH_ARGS = ("edu_type",)
+
+    def __init__(self, hs):
+        super(ReplicationFederationSendEduRestServlet, self).__init__(hs)
+
+        self.store = hs.get_datastore()
+        self.clock = hs.get_clock()
+        self.registry = hs.get_federation_registry()
+
+    @staticmethod
+    def _serialize_payload(edu_type, origin, content):
+        return {
+            "origin": origin,
+            "content": content,
+        }
+
+    @defer.inlineCallbacks
+    def _handle_request(self, request, edu_type):
+        with Measure(self.clock, "repl_fed_send_edu_parse"):
+            content = parse_json_object_from_request(request)
+
+            origin = content["origin"]
+            edu_content = content["content"]
+
+        logger.info(
+            "Got %r edu from $s",
+            edu_type, origin,
+        )
+
+        result = yield self.registry.on_edu(edu_type, origin, edu_content)
+
+        defer.returnValue((200, result))
+
+
+class ReplicationGetQueryRestServlet(ReplicationEndpoint):
+    """Handle responding to queries from federation.
+    """
+
+    NAME = "fed_query"
+    PATH_ARGS = ("query_type",)
+
+    # This is a query, so let's not bother caching
+    CACHE = False
+
+    def __init__(self, hs):
+        super(ReplicationGetQueryRestServlet, self).__init__(hs)
+
+        self.store = hs.get_datastore()
+        self.clock = hs.get_clock()
+        self.registry = hs.get_federation_registry()
+
+    @staticmethod
+    def _serialize_payload(query_type, args):
+        """
+        Args:
+            query_type (str)
+            args (dict): The arguments received for the given query type
+        """
+        return {
+            "args": args,
+        }
+
+    @defer.inlineCallbacks
+    def _handle_request(self, request, query_type):
+        with Measure(self.clock, "repl_fed_query_parse"):
+            content = parse_json_object_from_request(request)
+
+            args = content["args"]
+
+        logger.info(
+            "Got %r query",
+            query_type,
+        )
+
+        result = yield self.registry.on_query(query_type, args)
+
+        defer.returnValue((200, result))
+
+
+def register_servlets(hs, http_server):
+    ReplicationFederationSendEventsRestServlet(hs).register(http_server)
+    ReplicationFederationSendEduRestServlet(hs).register(http_server)
+    ReplicationGetQueryRestServlet(hs).register(http_server)

From a3f5bf79a0fc0ea6d59069945f53717a3e9c6581 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 26 Jul 2018 11:44:22 +0100
Subject: [PATCH 269/830] Add EDU/query handling over replication

---
 synapse/federation/federation_server.py | 43 +++++++++++++++++++++++++
 synapse/handlers/federation.py          | 24 +++++++-------
 synapse/replication/http/federation.py  |  2 +-
 synapse/server.py                       |  6 +++-
 4 files changed, 62 insertions(+), 13 deletions(-)

diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index bf89d568afed..941e30a59632 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -33,6 +33,10 @@
 from synapse.federation.persistence import TransactionActions
 from synapse.federation.units import Edu, Transaction
 from synapse.http.endpoint import parse_server_name
+from synapse.replication.http.federation import (
+    ReplicationFederationSendEduRestServlet,
+    ReplicationGetQueryRestServlet,
+)
 from synapse.types import get_domain_from_id
 from synapse.util import async
 from synapse.util.caches.response_cache import ResponseCache
@@ -745,6 +749,8 @@ def register_edu_handler(self, edu_type, handler):
         if edu_type in self.edu_handlers:
             raise KeyError("Already have an EDU handler for %s" % (edu_type,))
 
+        logger.info("Registering federation EDU handler for %r", edu_type)
+
         self.edu_handlers[edu_type] = handler
 
     def register_query_handler(self, query_type, handler):
@@ -763,6 +769,8 @@ def register_query_handler(self, query_type, handler):
                 "Already have a Query handler for %s" % (query_type,)
             )
 
+        logger.info("Registering federation query handler for %r", query_type)
+
         self.query_handlers[query_type] = handler
 
     @defer.inlineCallbacks
@@ -785,3 +793,38 @@ def on_query(self, query_type, args):
             raise NotFoundError("No handler for Query type '%s'" % (query_type,))
 
         return handler(args)
+
+
+class ReplicationFederationHandlerRegistry(FederationHandlerRegistry):
+    def __init__(self, hs):
+        self.config = hs.config
+        self.http_client = hs.get_simple_http_client()
+        self.clock = hs.get_clock()
+
+        self._get_query_client = ReplicationGetQueryRestServlet.make_client(hs)
+        self._send_edu = ReplicationFederationSendEduRestServlet.make_client(hs)
+
+        super(ReplicationFederationHandlerRegistry, self).__init__()
+
+    def on_edu(self, edu_type, origin, content):
+        handler = self.edu_handlers.get(edu_type)
+        if handler:
+            return super(ReplicationFederationHandlerRegistry, self).on_edu(
+                edu_type, origin, content,
+            )
+
+        return self._send_edu(
+                edu_type=edu_type,
+                origin=origin,
+                content=content,
+        )
+
+    def on_query(self, query_type, args):
+        handler = self.query_handlers.get(query_type)
+        if handler:
+            return handler(args)
+
+        return self._get_query_client(
+                query_type=query_type,
+                args=args,
+        )
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 0524dec942d1..d2cbb12df3e2 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -44,8 +44,10 @@
     compute_event_signature,
 )
 from synapse.events.validator import EventValidator
-from synapse.replication.http.federation import send_federation_events_to_master
-from synapse.replication.http.membership import notify_user_membership_change
+from synapse.replication.http.federation import (
+    ReplicationFederationSendEventsRestServlet,
+)
+from synapse.replication.http.membership import ReplicationUserJoinedLeftRoomRestServlet
 from synapse.state import resolve_events_with_factory
 from synapse.types import UserID, get_domain_from_id
 from synapse.util import logcontext, unwrapFirstError
@@ -91,6 +93,13 @@ def __init__(self, hs):
         self.config = hs.config
         self.http_client = hs.get_simple_http_client()
 
+        self._send_events_to_master = (
+            ReplicationFederationSendEventsRestServlet.make_client(hs)
+        )
+        self._notify_user_membership_change = (
+            ReplicationUserJoinedLeftRoomRestServlet.make_client(hs)
+        )
+
         # When joining a room we need to queue any events for that room up
         self.room_queues = {}
         self._room_pdu_linearizer = Linearizer("fed_room_pdu")
@@ -2318,12 +2327,8 @@ def _persist_events(self, event_and_contexts, backfilled=False):
             Deferred
         """
         if self.config.worker_app:
-            yield send_federation_events_to_master(
-                clock=self.hs.get_clock(),
+            yield self._send_events_to_master(
                 store=self.store,
-                client=self.http_client,
-                host=self.config.worker_replication_host,
-                port=self.config.worker_replication_http_port,
                 event_and_contexts=event_and_contexts,
                 backfilled=backfilled
             )
@@ -2381,10 +2386,7 @@ def user_joined_room(self, user, room_id):
         """Called when a new user has joined the room
         """
         if self.config.worker_app:
-            return notify_user_membership_change(
-                client=self.http_client,
-                host=self.config.worker_replication_host,
-                port=self.config.worker_replication_http_port,
+            return self._notify_user_membership_change(
                 room_id=room_id,
                 user_id=user.to_string(),
                 change="joined",
diff --git a/synapse/replication/http/federation.py b/synapse/replication/http/federation.py
index f39aaa89be57..3fa7bd64c7b5 100644
--- a/synapse/replication/http/federation.py
+++ b/synapse/replication/http/federation.py
@@ -59,8 +59,8 @@ def __init__(self, hs):
         self.notifier = hs.get_notifier()
         self.pusher_pool = hs.get_pusherpool()
 
-    @defer.inlineCallbacks
     @staticmethod
+    @defer.inlineCallbacks
     def _serialize_payload(store, event_and_contexts, backfilled):
         """
         Args:
diff --git a/synapse/server.py b/synapse/server.py
index 140be9ebe863..26228d8c72b4 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -36,6 +36,7 @@
 from synapse.federation.federation_server import (
     FederationHandlerRegistry,
     FederationServer,
+    ReplicationFederationHandlerRegistry,
 )
 from synapse.federation.send_queue import FederationRemoteSendQueue
 from synapse.federation.transaction_queue import TransactionQueue
@@ -423,7 +424,10 @@ def build_room_member_handler(self):
         return RoomMemberMasterHandler(self)
 
     def build_federation_registry(self):
-        return FederationHandlerRegistry()
+        if self.config.worker_app:
+            return ReplicationFederationHandlerRegistry(self)
+        else:
+            return FederationHandlerRegistry()
 
     def build_server_notices_manager(self):
         if self.config.worker_app:

From 1e2bed9656a4826ed7fdc82bc096df775bc38baf Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 26 Jul 2018 11:45:12 +0100
Subject: [PATCH 270/830] Import all functions from TransactionStore

---
 synapse/replication/slave/storage/transactions.py | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/synapse/replication/slave/storage/transactions.py b/synapse/replication/slave/storage/transactions.py
index 9c9a5eadd906..cd6416c47c8a 100644
--- a/synapse/replication/slave/storage/transactions.py
+++ b/synapse/replication/slave/storage/transactions.py
@@ -13,19 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from synapse.storage import DataStore
 from synapse.storage.transactions import TransactionStore
 
 from ._base import BaseSlavedStore
 
 
-class TransactionStore(BaseSlavedStore):
-    get_destination_retry_timings = TransactionStore.__dict__[
-        "get_destination_retry_timings"
-    ]
-    _get_destination_retry_timings = DataStore._get_destination_retry_timings.__func__
-    set_destination_retry_timings = DataStore.set_destination_retry_timings.__func__
-    _set_destination_retry_timings = DataStore._set_destination_retry_timings.__func__
-
-    prep_send_transaction = DataStore.prep_send_transaction.__func__
-    delivered_txn = DataStore.delivered_txn.__func__
+class TransactionStore(TransactionStore, BaseSlavedStore):
+    pass

From 62ace05c4521caeed023e5b9dadd993afe5c154f Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 26 Jul 2018 13:31:59 +0100
Subject: [PATCH 271/830] Move necessary storage functions to worker classes

---
 synapse/storage/events.py        | 82 -------------------------------
 synapse/storage/events_worker.py | 84 ++++++++++++++++++++++++++++++++
 synapse/storage/room.py          | 32 ++++++------
 3 files changed, 100 insertions(+), 98 deletions(-)

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index e8e5a0fe443c..5374796fd730 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -1430,88 +1430,6 @@ def _store_redaction(self, txn, event):
             (event.event_id, event.redacts)
         )
 
-    @defer.inlineCallbacks
-    def have_events_in_timeline(self, event_ids):
-        """Given a list of event ids, check if we have already processed and
-        stored them as non outliers.
-        """
-        rows = yield self._simple_select_many_batch(
-            table="events",
-            retcols=("event_id",),
-            column="event_id",
-            iterable=list(event_ids),
-            keyvalues={"outlier": False},
-            desc="have_events_in_timeline",
-        )
-
-        defer.returnValue(set(r["event_id"] for r in rows))
-
-    @defer.inlineCallbacks
-    def have_seen_events(self, event_ids):
-        """Given a list of event ids, check if we have already processed them.
-
-        Args:
-            event_ids (iterable[str]):
-
-        Returns:
-            Deferred[set[str]]: The events we have already seen.
-        """
-        results = set()
-
-        def have_seen_events_txn(txn, chunk):
-            sql = (
-                "SELECT event_id FROM events as e WHERE e.event_id IN (%s)"
-                % (",".join("?" * len(chunk)), )
-            )
-            txn.execute(sql, chunk)
-            for (event_id, ) in txn:
-                results.add(event_id)
-
-        # break the input up into chunks of 100
-        input_iterator = iter(event_ids)
-        for chunk in iter(lambda: list(itertools.islice(input_iterator, 100)),
-                          []):
-            yield self.runInteraction(
-                "have_seen_events",
-                have_seen_events_txn,
-                chunk,
-            )
-        defer.returnValue(results)
-
-    def get_seen_events_with_rejections(self, event_ids):
-        """Given a list of event ids, check if we rejected them.
-
-        Args:
-            event_ids (list[str])
-
-        Returns:
-            Deferred[dict[str, str|None):
-                Has an entry for each event id we already have seen. Maps to
-                the rejected reason string if we rejected the event, else maps
-                to None.
-        """
-        if not event_ids:
-            return defer.succeed({})
-
-        def f(txn):
-            sql = (
-                "SELECT e.event_id, reason FROM events as e "
-                "LEFT JOIN rejections as r ON e.event_id = r.event_id "
-                "WHERE e.event_id = ?"
-            )
-
-            res = {}
-            for event_id in event_ids:
-                txn.execute(sql, (event_id,))
-                row = txn.fetchone()
-                if row:
-                    _, rejected = row
-                    res[event_id] = rejected
-
-            return res
-
-        return self.runInteraction("get_rejection_reasons", f)
-
     @defer.inlineCallbacks
     def count_daily_messages(self):
         """
diff --git a/synapse/storage/events_worker.py b/synapse/storage/events_worker.py
index 9b4cfeb89922..e1dc2c62fd6f 100644
--- a/synapse/storage/events_worker.py
+++ b/synapse/storage/events_worker.py
@@ -12,7 +12,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import itertools
 import logging
+
 from collections import namedtuple
 
 from canonicaljson import json
@@ -442,3 +444,85 @@ def _get_event_from_row(self, internal_metadata, js, redacted,
             self._get_event_cache.prefill((original_ev.event_id,), cache_entry)
 
         defer.returnValue(cache_entry)
+
+    @defer.inlineCallbacks
+    def have_events_in_timeline(self, event_ids):
+        """Given a list of event ids, check if we have already processed and
+        stored them as non outliers.
+        """
+        rows = yield self._simple_select_many_batch(
+            table="events",
+            retcols=("event_id",),
+            column="event_id",
+            iterable=list(event_ids),
+            keyvalues={"outlier": False},
+            desc="have_events_in_timeline",
+        )
+
+        defer.returnValue(set(r["event_id"] for r in rows))
+
+    @defer.inlineCallbacks
+    def have_seen_events(self, event_ids):
+        """Given a list of event ids, check if we have already processed them.
+
+        Args:
+            event_ids (iterable[str]):
+
+        Returns:
+            Deferred[set[str]]: The events we have already seen.
+        """
+        results = set()
+
+        def have_seen_events_txn(txn, chunk):
+            sql = (
+                "SELECT event_id FROM events as e WHERE e.event_id IN (%s)"
+                % (",".join("?" * len(chunk)), )
+            )
+            txn.execute(sql, chunk)
+            for (event_id, ) in txn:
+                results.add(event_id)
+
+        # break the input up into chunks of 100
+        input_iterator = iter(event_ids)
+        for chunk in iter(lambda: list(itertools.islice(input_iterator, 100)),
+                          []):
+            yield self.runInteraction(
+                "have_seen_events",
+                have_seen_events_txn,
+                chunk,
+            )
+        defer.returnValue(results)
+
+    def get_seen_events_with_rejections(self, event_ids):
+        """Given a list of event ids, check if we rejected them.
+
+        Args:
+            event_ids (list[str])
+
+        Returns:
+            Deferred[dict[str, str|None):
+                Has an entry for each event id we already have seen. Maps to
+                the rejected reason string if we rejected the event, else maps
+                to None.
+        """
+        if not event_ids:
+            return defer.succeed({})
+
+        def f(txn):
+            sql = (
+                "SELECT e.event_id, reason FROM events as e "
+                "LEFT JOIN rejections as r ON e.event_id = r.event_id "
+                "WHERE e.event_id = ?"
+            )
+
+            res = {}
+            for event_id in event_ids:
+                txn.execute(sql, (event_id,))
+                row = txn.fetchone()
+                if row:
+                    _, rejected = row
+                    res[event_id] = rejected
+
+            return res
+
+        return self.runInteraction("get_rejection_reasons", f)
diff --git a/synapse/storage/room.py b/synapse/storage/room.py
index 3147fb682775..3378fc77d1c4 100644
--- a/synapse/storage/room.py
+++ b/synapse/storage/room.py
@@ -41,6 +41,22 @@
 
 
 class RoomWorkerStore(SQLBaseStore):
+    def get_room(self, room_id):
+        """Retrieve a room.
+
+        Args:
+            room_id (str): The ID of the room to retrieve.
+        Returns:
+            A namedtuple containing the room information, or an empty list.
+        """
+        return self._simple_select_one(
+            table="rooms",
+            keyvalues={"room_id": room_id},
+            retcols=("room_id", "is_public", "creator"),
+            desc="get_room",
+            allow_none=True,
+        )
+
     def get_public_room_ids(self):
         return self._simple_select_onecol(
             table="rooms",
@@ -215,22 +231,6 @@ def store_room_txn(txn, next_id):
             logger.error("store_room with room_id=%s failed: %s", room_id, e)
             raise StoreError(500, "Problem creating room.")
 
-    def get_room(self, room_id):
-        """Retrieve a room.
-
-        Args:
-            room_id (str): The ID of the room to retrieve.
-        Returns:
-            A namedtuple containing the room information, or an empty list.
-        """
-        return self._simple_select_one(
-            table="rooms",
-            keyvalues={"room_id": room_id},
-            retcols=("room_id", "is_public", "creator"),
-            desc="get_room",
-            allow_none=True,
-        )
-
     @defer.inlineCallbacks
     def set_room_is_public(self, room_id, is_public):
         def set_room_is_public_txn(txn, next_id):

From 96a9a296455e2c08be4e0375cf784e4113fa51e1 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 26 Jul 2018 13:35:36 +0100
Subject: [PATCH 272/830] Pull in necessary stores in federation_reader

---
 synapse/app/federation_reader.py | 2 ++
 synapse/storage/events_worker.py | 1 -
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py
index c512b4be877e..0c557a824227 100644
--- a/synapse/app/federation_reader.py
+++ b/synapse/app/federation_reader.py
@@ -36,6 +36,7 @@
 from synapse.replication.slave.storage.directory import DirectoryStore
 from synapse.replication.slave.storage.events import SlavedEventStore
 from synapse.replication.slave.storage.keys import SlavedKeyStore
+from synapse.replication.slave.storage.profile import SlavedProfileStore
 from synapse.replication.slave.storage.push_rule import SlavedPushRuleStore
 from synapse.replication.slave.storage.pushers import SlavedPusherStore
 from synapse.replication.slave.storage.receipts import SlavedReceiptsStore
@@ -53,6 +54,7 @@
 
 
 class FederationReaderSlavedStore(
+    SlavedProfileStore,
     SlavedApplicationServiceStore,
     SlavedPusherStore,
     SlavedPushRuleStore,
diff --git a/synapse/storage/events_worker.py b/synapse/storage/events_worker.py
index e1dc2c62fd6f..59822178ff02 100644
--- a/synapse/storage/events_worker.py
+++ b/synapse/storage/events_worker.py
@@ -14,7 +14,6 @@
 # limitations under the License.
 import itertools
 import logging
-
 from collections import namedtuple
 
 from canonicaljson import json

From 19a17068f1bc98a1556ff618b544b5fbf57eeba0 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 6 Aug 2018 15:15:19 +0100
Subject: [PATCH 273/830] Check m.room.create for sane room_versions

---
 synapse/event_auth.py                   | 10 +++++++++-
 synapse/federation/federation_client.py | 26 ++++++++++++++++++++++---
 2 files changed, 32 insertions(+), 4 deletions(-)

diff --git a/synapse/event_auth.py b/synapse/event_auth.py
index b32f64e72976..6baeccca38ae 100644
--- a/synapse/event_auth.py
+++ b/synapse/event_auth.py
@@ -20,7 +20,7 @@
 from signedjson.sign import SignatureVerifyException, verify_signed_json
 from unpaddedbase64 import decode_base64
 
-from synapse.api.constants import EventTypes, JoinRules, Membership
+from synapse.api.constants import KNOWN_ROOM_VERSIONS, EventTypes, JoinRules, Membership
 from synapse.api.errors import AuthError, EventSizeError, SynapseError
 from synapse.types import UserID, get_domain_from_id
 
@@ -83,6 +83,14 @@ def check(event, auth_events, do_sig_check=True, do_size_check=True):
                 403,
                 "Creation event's room_id domain does not match sender's"
             )
+
+        room_version = event.content.get("room_version", "1")
+        if room_version not in KNOWN_ROOM_VERSIONS:
+            raise AuthError(
+                403,
+                "room appears to have unsupported version %s" % (
+                    room_version,
+                ))
         # FIXME
         logger.debug("Allowing! %s", event)
         return
diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index 7ec1d7a88984..c9f3c2d35262 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -25,7 +25,7 @@
 
 from twisted.internet import defer
 
-from synapse.api.constants import Membership
+from synapse.api.constants import KNOWN_ROOM_VERSIONS, EventTypes, Membership
 from synapse.api.errors import (
     CodeMessageException,
     FederationDeniedError,
@@ -518,7 +518,7 @@ def _try_destination_list(self, description, destinations, callback):
                     description, destination, exc_info=1,
                 )
 
-        raise RuntimeError("Failed to %s via any server", description)
+        raise RuntimeError("Failed to %s via any server" % (description, ))
 
     def make_membership_event(self, destinations, room_id, user_id, membership,
                               content, params):
@@ -609,6 +609,26 @@ def send_join(self, destinations, pdu):
             Fails with a ``RuntimeError`` if no servers were reachable.
         """
 
+        def check_authchain_validity(signed_auth_chain):
+            for e in signed_auth_chain:
+                if e.type == EventTypes.Create:
+                    create_event = e
+                    break
+            else:
+                raise InvalidResponseError(
+                    "no %s in auth chain" % (EventTypes.Create,),
+                )
+
+            # the room version should be sane.
+            room_version = create_event.content.get("room_version", "1")
+            if room_version not in KNOWN_ROOM_VERSIONS:
+                # This shouldn't be possible, because the remote server should have
+                # rejected the join attempt during make_join.
+                raise InvalidResponseError(
+                    "room appears to have unsupported version %s" % (
+                        room_version,
+                    ))
+
         @defer.inlineCallbacks
         def send_request(destination):
             time_now = self._clock.time_msec()
@@ -665,7 +685,7 @@ def send_request(destination):
             for s in signed_state:
                 s.internal_metadata = copy.deepcopy(s.internal_metadata)
 
-            auth_chain.sort(key=lambda e: e.depth)
+            check_authchain_validity(signed_auth)
 
             defer.returnValue({
                 "state": signed_state,

From 7f3f1085612a3c46a5e3c2a6a376b42024f6b6cb Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 6 Aug 2018 16:30:52 +0100
Subject: [PATCH 274/830] changelog

---
 changelog.d/3654.feature | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3654.feature

diff --git a/changelog.d/3654.feature b/changelog.d/3654.feature
new file mode 100644
index 000000000000..35c95580bc45
--- /dev/null
+++ b/changelog.d/3654.feature
@@ -0,0 +1 @@
+Basic support for room versioning

From 16d78be315fd71d8bdb39af53317b3f7dd1dbb32 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Mon, 6 Aug 2018 17:51:15 +0100
Subject: [PATCH 275/830] make use of _simple_select_one_onecol, improved
 comments

---
 synapse/storage/monthly_active_users.py       | 19 +++++++++++--------
 .../schema/delta/51/monthly_active_users.sql  |  4 ++++
 2 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py
index 135837507a14..c28c698c6c19 100644
--- a/synapse/storage/monthly_active_users.py
+++ b/synapse/storage/monthly_active_users.py
@@ -15,7 +15,7 @@
 
 from twisted.internet import defer
 
-from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
+from synapse.util.caches.descriptors import cached
 
 from ._base import SQLBaseStore
 
@@ -104,7 +104,7 @@ def upsert_monthly_active_user(self, user_id):
             self._user_last_seen_monthly_active.invalidate((user_id,))
             self.get_monthly_active_count.invalidate(())
 
-    @cachedInlineCallbacks(num_args=1)
+    @cached(num_args=1)
     def _user_last_seen_monthly_active(self, user_id):
         """
             Checks if a given user is part of the monthly active user group
@@ -114,18 +114,16 @@ def _user_last_seen_monthly_active(self, user_id):
                 int : timestamp since last seen, None if never seen
 
         """
-        result = yield self._simple_select_onecol(
+
+        return(self._simple_select_one_onecol(
             table="monthly_active_users",
             keyvalues={
                 "user_id": user_id,
             },
             retcol="timestamp",
+            allow_none=True,
             desc="_user_last_seen_monthly_active",
-        )
-        timestamp = None
-        if len(result) > 0:
-            timestamp = result[0]
-        defer.returnValue(timestamp)
+        ))
 
     @defer.inlineCallbacks
     def populate_monthly_active_users(self, user_id):
@@ -140,6 +138,11 @@ def populate_monthly_active_users(self, user_id):
             last_seen_timestamp = yield self._user_last_seen_monthly_active(user_id)
             now = self.hs.get_clock().time_msec()
 
+            # We want to reduce to the total number of db writes, and are happy
+            # to trade accuracy of timestamp in order to lighten load. This means
+            # We always insert new users (where MAU threshold has not been reached),
+            # but only update if we have not previously seen the user for
+            # LAST_SEEN_GRANULARITY ms
             if last_seen_timestamp is None:
                 count = yield self.get_monthly_active_count()
                 if count < self.hs.config.max_mau_value:
diff --git a/synapse/storage/schema/delta/51/monthly_active_users.sql b/synapse/storage/schema/delta/51/monthly_active_users.sql
index f2b6d3e31e4e..10aac90ce137 100644
--- a/synapse/storage/schema/delta/51/monthly_active_users.sql
+++ b/synapse/storage/schema/delta/51/monthly_active_users.sql
@@ -16,6 +16,10 @@
 -- a table of monthly active users, for use where blocking based on mau limits
 CREATE TABLE monthly_active_users (
     user_id TEXT NOT NULL,
+    -- Last time we saw the user. Not guaranteed to be accurate due to rate limiting
+    -- on updates, Granularity of updates governed by
+    -- syanpse.storage.monthly_active_users.LAST_SEEN_GRANULARITY
+    -- Measured in ms since epoch.
     timestamp BIGINT NOT NULL
 );
 

From 1911c037cbace498c0827eb5abe6ed3f64acb671 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Mon, 6 Aug 2018 18:01:46 +0100
Subject: [PATCH 276/830] update comments to reflect new sig

---
 synapse/api/auth.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index d8ebbbc6e8de..91b23ff1d720 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -778,10 +778,7 @@ def check_in_room_or_world_readable(self, room_id, user_id):
     def check_auth_blocking(self):
         """Checks if the user should be rejected for some external reason,
         such as monthly active user limiting or global disable flag
-        Args:
-            error (Error): The error that should be raised if user is to be
-            blocked
-            """
+        """
         if self.hs.config.limit_usage_by_mau is True:
             current_mau = yield self.store.get_monthly_active_count()
             if current_mau >= self.hs.config.max_mau_value:

From e54794f5b658992627eab5400b13199128eec0a1 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Mon, 6 Aug 2018 21:55:54 +0100
Subject: [PATCH 277/830] Fix postgres compatibility bug

---
 synapse/storage/monthly_active_users.py | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py
index c28c698c6c19..abe1e6bb99ad 100644
--- a/synapse/storage/monthly_active_users.py
+++ b/synapse/storage/monthly_active_users.py
@@ -43,14 +43,25 @@ def _reap_users(txn):
             thirty_days_ago = (
                 int(self._clock.time_msec()) - (1000 * 60 * 60 * 24 * 30)
             )
-
+            # Purge stale users
             sql = "DELETE FROM monthly_active_users WHERE timestamp < ?"
-
             txn.execute(sql, (thirty_days_ago,))
+
+            # If MAU user count still exceeds the MAU threshold, then delete on
+            # a least recently active basis.
+            # Note it is not possible to write this query using OFFSET due to
+            # incompatibilities in how sqlite an postgres support the feature.
+            # sqlite requires 'LIMIT -1 OFFSET ?', the LIMIT must be present
+            # While Postgres does not require 'LIMIT', but also does not support
+            # negative LIMIT values. So there is no way to write it that both can
+            # support
             sql = """
                 DELETE FROM monthly_active_users
-                ORDER BY timestamp desc
-                LIMIT -1 OFFSET ?
+                WHERE user_id NOT IN (
+                    SELECT user_id FROM monthly_active_users
+                    ORDER BY timestamp DESC
+                    LIMIT ?
+                    )
                 """
             txn.execute(sql, (self.hs.config.max_mau_value,))
 

From 7daa8a78c5183a46ae462c0718939ca52fa0130f Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Mon, 6 Aug 2018 22:55:05 +0100
Subject: [PATCH 278/830] load mau limit threepids

---
 synapse/config/server.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/synapse/config/server.py b/synapse/config/server.py
index 8fd23197590c..cacac253ab0f 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -74,6 +74,9 @@ def read_config(self, config):
             self.max_mau_value = config.get(
                 "max_mau_value", 0,
             )
+        self.mau_limits_reserved_threepids = config.get(
+            "mau_limit_reserved_threepid", []
+        )
 
         # FIXME: federation_domain_whitelist needs sytests
         self.federation_domain_whitelist = None

From fbe255f9a4c46fe82a394b9efab6ad8811ae71e0 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Mon, 6 Aug 2018 23:24:54 +0100
Subject: [PATCH 279/830] add default mau_limits_reserved_threepids

---
 tests/utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/utils.py b/tests/utils.py
index ec40428e7438..baeed5d60047 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -74,6 +74,7 @@ def setup_test_homeserver(name="test", datastore=None, config=None, reactor=None
         config.media_storage_providers = []
         config.auto_join_rooms = []
         config.limit_usage_by_mau = False
+        config.mau_limits_reserved_threepids = []
 
         # disable user directory updates, because they get done in the
         # background, which upsets the test runner.

From a74b25faaaf47b4696cb30207ef2eae6a3a5d8c7 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Mon, 6 Aug 2018 23:25:25 +0100
Subject: [PATCH 280/830] WIP building out mau reserved users

---
 synapse/storage/monthly_active_users.py | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py
index abe1e6bb99ad..6a37d6fc22af 100644
--- a/synapse/storage/monthly_active_users.py
+++ b/synapse/storage/monthly_active_users.py
@@ -19,16 +19,30 @@
 
 from ._base import SQLBaseStore
 
+
 # Number of msec of granularity to store the monthly_active_user timestamp
 # This means it is not necessary to update the table on every request
 LAST_SEEN_GRANULARITY = 60 * 60 * 1000
 
 
 class MonthlyActiveUsersStore(SQLBaseStore):
+    @defer.inlineCallbacks
     def __init__(self, dbconn, hs):
         super(MonthlyActiveUsersStore, self).__init__(None, hs)
         self._clock = hs.get_clock()
         self.hs = hs
+        threepids = self.hs.config.mau_limits_reserved_threepids
+        self.reserved_user_ids = set()
+        for tp in threepids:
+            user_id = yield hs.get_datastore().get_user_id_by_threepid(
+                tp["medium"], tp["address"]
+            )
+            if user_id:
+                self.reserved_user_ids.add(user_id)
+            else:
+                logger.warning(
+                    "mau limit reserved threepid %s not found in db" % tp
+                )
 
     def reap_monthly_active_users(self):
         """
@@ -78,7 +92,7 @@ def _reap_users(txn):
 
     @cached(num_args=0)
     def get_monthly_active_count(self):
-        """Generates current count of monthly active users.abs
+        """Generates current count of monthly active users
 
         Returns:
             Defered[int]: Number of current monthly active users

From ca9bc1f4feaaabce48595ee2e387529f5bd365e6 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 7 Aug 2018 10:00:03 +0100
Subject: [PATCH 281/830] Fix occasional glitches in the
 synapse_event_persisted_position metric

Every so often this metric glitched to a negative number. I'm assuming it was
due to backfilled events.
---
 synapse/storage/events.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index e8e5a0fe443c..ce32e8fefd23 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -485,9 +485,14 @@ def _persist_events(self, events_and_contexts, backfilled=False,
                     new_forward_extremeties=new_forward_extremeties,
                 )
                 persist_event_counter.inc(len(chunk))
-                synapse.metrics.event_persisted_position.set(
-                    chunk[-1][0].internal_metadata.stream_ordering,
-                )
+
+                if not backfilled:
+                    # backfilled events have negative stream orderings, so we don't
+                    # want to set the event_persisted_position to that.
+                    synapse.metrics.event_persisted_position.set(
+                        chunk[-1][0].internal_metadata.stream_ordering,
+                    )
+
                 for event, context in chunk:
                     if context.app_service:
                         origin_type = "local"

From 865d07cd381214c38628cc4ad4200a5cdd45f5d3 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 7 Aug 2018 10:02:01 +0100
Subject: [PATCH 282/830] changelog

---
 changelog.d/3658.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3658.bugfix

diff --git a/changelog.d/3658.bugfix b/changelog.d/3658.bugfix
new file mode 100644
index 000000000000..556011a15060
--- /dev/null
+++ b/changelog.d/3658.bugfix
@@ -0,0 +1 @@
+Fix occasional glitches in the synapse_event_persisted_position metric

From 495cb100d127212d55a46c177706d732950e70be Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 6 Aug 2018 14:46:17 +0100
Subject: [PATCH 283/830] Allow profile changes to happen on workers

---
 synapse/app/event_creator.py         |  8 ++++
 synapse/handlers/profile.py          | 26 ++++++++--
 synapse/replication/http/__init__.py |  3 +-
 synapse/replication/http/profile.py  | 71 ++++++++++++++++++++++++++++
 synapse/storage/profile.py           |  6 ++-
 5 files changed, 106 insertions(+), 8 deletions(-)
 create mode 100644 synapse/replication/http/profile.py

diff --git a/synapse/app/event_creator.py b/synapse/app/event_creator.py
index 374f1156449c..0c56fc777bf7 100644
--- a/synapse/app/event_creator.py
+++ b/synapse/app/event_creator.py
@@ -45,6 +45,11 @@
 from synapse.replication.slave.storage.room import RoomStore
 from synapse.replication.slave.storage.transactions import TransactionStore
 from synapse.replication.tcp.client import ReplicationClientHandler
+from synapse.rest.client.v1.profile import (
+    ProfileAvatarURLRestServlet,
+    ProfileDisplaynameRestServlet,
+    ProfileRestServlet,
+)
 from synapse.rest.client.v1.room import (
     JoinRoomAliasServlet,
     RoomMembershipRestServlet,
@@ -101,6 +106,9 @@ def _listen_http(self, listener_config):
                     RoomMembershipRestServlet(self).register(resource)
                     RoomStateEventRestServlet(self).register(resource)
                     JoinRoomAliasServlet(self).register(resource)
+                    ProfileAvatarURLRestServlet(self).register(resource)
+                    ProfileDisplaynameRestServlet(self).register(resource)
+                    ProfileRestServlet(self).register(resource)
                     resources.update({
                         "/_matrix/client/r0": resource,
                         "/_matrix/client/unstable": resource,
diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py
index cb5c6d587e0f..a3bdb1830f7d 100644
--- a/synapse/handlers/profile.py
+++ b/synapse/handlers/profile.py
@@ -19,6 +19,7 @@
 
 from synapse.api.errors import AuthError, CodeMessageException, SynapseError
 from synapse.metrics.background_process_metrics import run_as_background_process
+from synapse.replication.http.profile import ReplicationHandleProfileChangeRestServlet
 from synapse.types import UserID, get_domain_from_id
 
 from ._base import BaseHandler
@@ -45,6 +46,10 @@ def __init__(self, hs):
                 self._start_update_remote_profile_cache, self.PROFILE_UPDATE_MS,
             )
 
+        self._notify_master_profile_change = (
+            ReplicationHandleProfileChangeRestServlet.make_client(hs)
+        )
+
     @defer.inlineCallbacks
     def get_profile(self, user_id):
         target_user = UserID.from_string(user_id)
@@ -147,10 +152,16 @@ def set_displayname(self, target_user, requester, new_displayname, by_admin=Fals
         )
 
         if self.hs.config.user_directory_search_all_users:
-            profile = yield self.store.get_profileinfo(target_user.localpart)
-            yield self.user_directory_handler.handle_local_profile_change(
-                target_user.to_string(), profile
-            )
+            if self.hs.config.worker_app is None:
+                profile = yield self.store.get_profileinfo(target_user.localpart)
+                yield self.user_directory_handler.handle_local_profile_change(
+                    target_user.to_string(), profile
+                )
+            else:
+                yield self._notify_master_profile_change(
+                    requester=requester,
+                    user_id=target_user.to_string(),
+                )
 
         yield self._update_join_states(requester, target_user)
 
@@ -196,11 +207,16 @@ def set_avatar_url(self, target_user, requester, new_avatar_url, by_admin=False)
             target_user.localpart, new_avatar_url
         )
 
-        if self.hs.config.user_directory_search_all_users:
+        if self.hs.config.worker_app is None:
             profile = yield self.store.get_profileinfo(target_user.localpart)
             yield self.user_directory_handler.handle_local_profile_change(
                 target_user.to_string(), profile
             )
+        else:
+            yield self._notify_master_profile_change(
+                requester=requester,
+                user_id=target_user.to_string(),
+            )
 
         yield self._update_join_states(requester, target_user)
 
diff --git a/synapse/replication/http/__init__.py b/synapse/replication/http/__init__.py
index 589ee94c66a0..1fbbdf53e9c3 100644
--- a/synapse/replication/http/__init__.py
+++ b/synapse/replication/http/__init__.py
@@ -14,7 +14,7 @@
 # limitations under the License.
 
 from synapse.http.server import JsonResource
-from synapse.replication.http import membership, send_event
+from synapse.replication.http import membership, profile, send_event
 
 REPLICATION_PREFIX = "/_synapse/replication"
 
@@ -27,3 +27,4 @@ def __init__(self, hs):
     def register_servlets(self, hs):
         send_event.register_servlets(hs, self)
         membership.register_servlets(hs, self)
+        profile.register_servlets(hs, self)
diff --git a/synapse/replication/http/profile.py b/synapse/replication/http/profile.py
new file mode 100644
index 000000000000..c4d54c936fe4
--- /dev/null
+++ b/synapse/replication/http/profile.py
@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from twisted.internet import defer
+
+from synapse.http.servlet import parse_json_object_from_request
+from synapse.replication.http._base import ReplicationEndpoint
+from synapse.types import Requester, UserID
+
+logger = logging.getLogger(__name__)
+
+
+class ReplicationHandleProfileChangeRestServlet(ReplicationEndpoint):
+    NAME = "profile_changed"
+    PATH_ARGS = ("user_id",)
+    POST = True
+
+    def __init__(self, hs):
+        super(ReplicationHandleProfileChangeRestServlet, self).__init__(hs)
+
+        self.user_directory_handler = hs.get_user_directory_handler()
+        self.store = hs.get_datastore()
+        self.clock = hs.get_clock()
+
+    @staticmethod
+    def _serialize_payload(requester, user_id):
+        """
+        Args:
+            requester (Requester)
+            user_id (str)
+        """
+
+        return {
+            "requester": requester.serialize(),
+        }
+
+    @defer.inlineCallbacks
+    def _handle_request(self, request, user_id):
+        content = parse_json_object_from_request(request)
+
+        requester = Requester.deserialize(self.store, content["requester"])
+
+        if requester.user:
+            request.authenticated_entity = requester.user.to_string()
+
+        target_user = UserID.from_string(user_id)
+
+        profile = yield self.store.get_profileinfo(target_user.localpart)
+        yield self.user_directory_handler.handle_local_profile_change(
+            user_id, profile
+        )
+
+        defer.returnValue((200, {}))
+
+
+def register_servlets(hs, http_server):
+    ReplicationHandleProfileChangeRestServlet(hs).register(http_server)
diff --git a/synapse/storage/profile.py b/synapse/storage/profile.py
index 60295da2547b..246ab836bd2e 100644
--- a/synapse/storage/profile.py
+++ b/synapse/storage/profile.py
@@ -71,8 +71,6 @@ def get_from_remote_profile_cache(self, user_id):
             desc="get_from_remote_profile_cache",
         )
 
-
-class ProfileStore(ProfileWorkerStore):
     def create_profile(self, user_localpart):
         return self._simple_insert(
             table="profiles",
@@ -182,3 +180,7 @@ def is_subscribed_remote_profile_for_user(self, user_id):
 
         if res:
             defer.returnValue(True)
+
+
+class ProfileStore(ProfileWorkerStore):
+    pass

From cd9765805e2792ad7d6b0b014447f51d2b9add8c Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 7 Aug 2018 10:48:31 +0100
Subject: [PATCH 284/830] Allow ratelimiting on workers

---
 synapse/storage/room.py | 58 ++++++++++++++++++++---------------------
 1 file changed, 29 insertions(+), 29 deletions(-)

diff --git a/synapse/storage/room.py b/synapse/storage/room.py
index 3147fb682775..1e473cc1883c 100644
--- a/synapse/storage/room.py
+++ b/synapse/storage/room.py
@@ -170,6 +170,35 @@ def is_room_blocked(self, room_id):
             desc="is_room_blocked",
         )
 
+    @cachedInlineCallbacks(max_entries=10000)
+    def get_ratelimit_for_user(self, user_id):
+        """Check if there are any overrides for ratelimiting for the given
+        user
+
+        Args:
+            user_id (str)
+
+        Returns:
+            RatelimitOverride if there is an override, else None. If the contents
+            of RatelimitOverride are None or 0 then ratelimitng has been
+            disabled for that user entirely.
+        """
+        row = yield self._simple_select_one(
+            table="ratelimit_override",
+            keyvalues={"user_id": user_id},
+            retcols=("messages_per_second", "burst_count"),
+            allow_none=True,
+            desc="get_ratelimit_for_user",
+        )
+
+        if row:
+            defer.returnValue(RatelimitOverride(
+                messages_per_second=row["messages_per_second"],
+                burst_count=row["burst_count"],
+            ))
+        else:
+            defer.returnValue(None)
+
 
 class RoomStore(RoomWorkerStore, SearchStore):
 
@@ -469,35 +498,6 @@ def get_all_new_public_rooms(txn):
             "get_all_new_public_rooms", get_all_new_public_rooms
         )
 
-    @cachedInlineCallbacks(max_entries=10000)
-    def get_ratelimit_for_user(self, user_id):
-        """Check if there are any overrides for ratelimiting for the given
-        user
-
-        Args:
-            user_id (str)
-
-        Returns:
-            RatelimitOverride if there is an override, else None. If the contents
-            of RatelimitOverride are None or 0 then ratelimitng has been
-            disabled for that user entirely.
-        """
-        row = yield self._simple_select_one(
-            table="ratelimit_override",
-            keyvalues={"user_id": user_id},
-            retcols=("messages_per_second", "burst_count"),
-            allow_none=True,
-            desc="get_ratelimit_for_user",
-        )
-
-        if row:
-            defer.returnValue(RatelimitOverride(
-                messages_per_second=row["messages_per_second"],
-                burst_count=row["burst_count"],
-            ))
-        else:
-            defer.returnValue(None)
-
     @defer.inlineCallbacks
     def block_room(self, room_id, user_id):
         yield self._simple_insert(

From f81f4210869a65d68b256a82171ba8709649b61e Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 7 Aug 2018 10:51:35 +0100
Subject: [PATCH 285/830] Update workers.rst with new paths

---
 docs/workers.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/workers.rst b/docs/workers.rst
index c5b37c3ded73..72c0f289fb9f 100644
--- a/docs/workers.rst
+++ b/docs/workers.rst
@@ -244,6 +244,7 @@ Handles some event creation. It can handle REST endpoints matching::
     ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/send
     ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)$
     ^/_matrix/client/(api/v1|r0|unstable)/join/
+    ^/_matrix/client/(api/v1|r0|unstable)/profile/
 
 It will create events locally and then send them on to the main synapse
 instance to be persisted and handled.

From 3b9662339b77b7aeafa9cf6282adfbcc13b10fed Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Tue, 7 Aug 2018 21:10:52 +1000
Subject: [PATCH 286/830] version

---
 synapse/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/__init__.py b/synapse/__init__.py
index 1810cb6fcd3f..a14d578e36a7 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -17,4 +17,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.33.1"
+__version__ = "0.33.2"

From cf8ef11f357ec520751ba021326c666683f9e6bd Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Tue, 7 Aug 2018 21:11:24 +1000
Subject: [PATCH 287/830] changelog

---
 CHANGES.md               | 64 ++++++++++++++++++++++++++++++++++++++++
 changelog.d/2952.bugfix  |  1 -
 changelog.d/2970.feature |  1 -
 changelog.d/3331.feature |  1 -
 changelog.d/3350.misc    |  1 -
 changelog.d/3367.misc    |  1 -
 changelog.d/3384.misc    |  1 -
 changelog.d/3391.bugfix  |  1 -
 changelog.d/3415.misc    |  0
 changelog.d/3460.misc    |  0
 changelog.d/3514.bugfix  |  1 -
 changelog.d/3520.bugfix  |  1 -
 changelog.d/3543.misc    |  1 -
 changelog.d/3548.bugfix  |  1 -
 changelog.d/3552.misc    |  1 -
 changelog.d/3553.feature |  1 -
 changelog.d/3554.feature |  1 -
 changelog.d/3555.feature |  1 -
 changelog.d/3556.feature |  1 -
 changelog.d/3559.misc    |  1 -
 changelog.d/3562.misc    |  0
 changelog.d/3567.feature |  1 -
 changelog.d/3569.bugfix  |  1 -
 changelog.d/3570.bugfix  |  1 -
 changelog.d/3571.misc    |  1 -
 changelog.d/3572.misc    |  1 -
 changelog.d/3577.misc    |  0
 changelog.d/3579.misc    |  1 -
 changelog.d/3581.misc    |  1 -
 changelog.d/3582.misc    |  1 -
 changelog.d/3584.misc    |  1 -
 changelog.d/3585.bugfix  |  1 -
 changelog.d/3586.misc    |  1 -
 changelog.d/3587.misc    |  1 -
 changelog.d/3590.misc    |  1 -
 changelog.d/3591.misc    |  1 -
 changelog.d/3592.misc    |  1 -
 changelog.d/3595.misc    |  1 -
 changelog.d/3597.feature |  1 -
 changelog.d/3601.bugfix  |  1 -
 changelog.d/3604.feature |  1 -
 changelog.d/3605.bugfix  |  1 -
 changelog.d/3606.misc    |  1 -
 changelog.d/3607.bugfix  |  1 -
 changelog.d/3609.misc    |  1 -
 changelog.d/3610.feature |  1 -
 changelog.d/3612.misc    |  1 -
 changelog.d/3613.misc    |  1 -
 changelog.d/3614.misc    |  1 -
 changelog.d/3616.misc    |  1 -
 changelog.d/3621.misc    |  1 -
 changelog.d/3626.bugfix  |  1 -
 changelog.d/3628.misc    |  1 -
 changelog.d/3630.feature |  1 -
 changelog.d/3634.misc    |  1 -
 changelog.d/3638.misc    |  1 -
 changelog.d/3639.feature |  1 -
 changelog.d/3644.misc    |  1 -
 changelog.d/3645.misc    |  1 -
 59 files changed, 64 insertions(+), 54 deletions(-)
 delete mode 100644 changelog.d/2952.bugfix
 delete mode 100644 changelog.d/2970.feature
 delete mode 100644 changelog.d/3331.feature
 delete mode 100644 changelog.d/3350.misc
 delete mode 100644 changelog.d/3367.misc
 delete mode 100644 changelog.d/3384.misc
 delete mode 100644 changelog.d/3391.bugfix
 delete mode 100644 changelog.d/3415.misc
 delete mode 100644 changelog.d/3460.misc
 delete mode 100644 changelog.d/3514.bugfix
 delete mode 100644 changelog.d/3520.bugfix
 delete mode 100644 changelog.d/3543.misc
 delete mode 100644 changelog.d/3548.bugfix
 delete mode 100644 changelog.d/3552.misc
 delete mode 100644 changelog.d/3553.feature
 delete mode 100644 changelog.d/3554.feature
 delete mode 100644 changelog.d/3555.feature
 delete mode 100644 changelog.d/3556.feature
 delete mode 100644 changelog.d/3559.misc
 delete mode 100644 changelog.d/3562.misc
 delete mode 100644 changelog.d/3567.feature
 delete mode 100644 changelog.d/3569.bugfix
 delete mode 100644 changelog.d/3570.bugfix
 delete mode 100644 changelog.d/3571.misc
 delete mode 100644 changelog.d/3572.misc
 delete mode 100644 changelog.d/3577.misc
 delete mode 100644 changelog.d/3579.misc
 delete mode 100644 changelog.d/3581.misc
 delete mode 100644 changelog.d/3582.misc
 delete mode 100644 changelog.d/3584.misc
 delete mode 100644 changelog.d/3585.bugfix
 delete mode 100644 changelog.d/3586.misc
 delete mode 100644 changelog.d/3587.misc
 delete mode 100644 changelog.d/3590.misc
 delete mode 100644 changelog.d/3591.misc
 delete mode 100644 changelog.d/3592.misc
 delete mode 100644 changelog.d/3595.misc
 delete mode 100644 changelog.d/3597.feature
 delete mode 100644 changelog.d/3601.bugfix
 delete mode 100644 changelog.d/3604.feature
 delete mode 100644 changelog.d/3605.bugfix
 delete mode 100644 changelog.d/3606.misc
 delete mode 100644 changelog.d/3607.bugfix
 delete mode 100644 changelog.d/3609.misc
 delete mode 100644 changelog.d/3610.feature
 delete mode 100644 changelog.d/3612.misc
 delete mode 100644 changelog.d/3613.misc
 delete mode 100644 changelog.d/3614.misc
 delete mode 100644 changelog.d/3616.misc
 delete mode 100644 changelog.d/3621.misc
 delete mode 100644 changelog.d/3626.bugfix
 delete mode 100644 changelog.d/3628.misc
 delete mode 100644 changelog.d/3630.feature
 delete mode 100644 changelog.d/3634.misc
 delete mode 100644 changelog.d/3638.misc
 delete mode 100644 changelog.d/3639.feature
 delete mode 100644 changelog.d/3644.misc
 delete mode 100644 changelog.d/3645.misc

diff --git a/CHANGES.md b/CHANGES.md
index a4c3ce31ae19..83a48c8b2621 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,67 @@
+Synapse 0.33.2 (2018-08-07)
+===========================
+
+Features
+--------
+
+- add support for the lazy_loaded_members filter as per MSC1227 ([\#2970](https://github.com/matrix-org/synapse/issues/2970>))
+- add support for the include_redundant_members filter param as per MSC1227 ([\#3331](https://github.com/matrix-org/synapse/issues/3331>))
+- Add metrics to track resource usage by background processes ([\#3553](https://github.com/matrix-org/synapse/issues/3553>), [\#3556](https://github.com/matrix-org/synapse/issues/3556>), [\#3604](https://github.com/matrix-org/synapse/issues/3604>), [\#3610](https://github.com/matrix-org/synapse/issues/3610>))
+- Add `code` label to `synapse_http_server_response_time_seconds` prometheus metric ([\#3554](https://github.com/matrix-org/synapse/issues/3554>))
+- Add support for client_reader to handle more APIs ([\#3555](https://github.com/matrix-org/synapse/issues/3555>), [\#3597](https://github.com/matrix-org/synapse/issues/3597>))
+- make the /context API filter & lazy-load aware as per MSC1227 ([\#3567](https://github.com/matrix-org/synapse/issues/3567>))
+- Add ability to limit number of monthly active users on the server ([\#3630](https://github.com/matrix-org/synapse/issues/3630>))
+- When we fail to join a room over federation, pass the error code back to the client. ([\#3639](https://github.com/matrix-org/synapse/issues/3639>))
+
+
+Bugfixes
+--------
+
+- Make /directory/list API return 404 for room not found instead of 400 ([\#2952](https://github.com/matrix-org/synapse/issues/2952>))
+- Default inviter_display_name to mxid for email invites ([\#3391](https://github.com/matrix-org/synapse/issues/3391>))
+- Don't generate TURN credentials if no TURN config options are set ([\#3514](https://github.com/matrix-org/synapse/issues/3514>))
+- Correctly announce deleted devices over federation ([\#3520](https://github.com/matrix-org/synapse/issues/3520>))
+- Catch failures saving metrics captured by Measure, and instead log the faulty metrics information for further analysis. ([\#3548](https://github.com/matrix-org/synapse/issues/3548>))
+- Unicode passwords are now normalised before hashing, preventing the instance where two different devices or browsers might send a different UTF-8 sequence for the password. ([\#3569](https://github.com/matrix-org/synapse/issues/3569>))
+- Fix potential stack overflow and deadlock under heavy load ([\#3570](https://github.com/matrix-org/synapse/issues/3570>))
+- Respond with M_NOT_FOUND when profiles are not found locally or over federation. Fixes #3585 ([\#3585](https://github.com/matrix-org/synapse/issues/3585>))
+- Fix failure to persist events over federation under load ([\#3601](https://github.com/matrix-org/synapse/issues/3601>))
+- Fix updating of cached remote profiles ([\#3605](https://github.com/matrix-org/synapse/issues/3605>))
+- Fix 'tuple index out of range' error ([\#3607](https://github.com/matrix-org/synapse/issues/3607>))
+- Only import secrets when available (fix for py < 3.6) ([\#3626](https://github.com/matrix-org/synapse/issues/3626>))
+
+
+Internal Changes
+----------------
+
+- Remove redundant checks on who_forgot_in_room ([\#3350](https://github.com/matrix-org/synapse/issues/3350>))
+- Remove unnecessary event re-signing hacks ([\#3367](https://github.com/matrix-org/synapse/issues/3367>))
+- Rewrite cache list decorator ([\#3384](https://github.com/matrix-org/synapse/issues/3384>))
+-  ([\#3415](https://github.com/matrix-org/synapse/issues/3415>), [\#3460](https://github.com/matrix-org/synapse/issues/3460>), [\#3562](https://github.com/matrix-org/synapse/issues/3562>), [\#3577](https://github.com/matrix-org/synapse/issues/3577>))
+- Improve Dockerfile and docker-compose instructions ([\#3543](https://github.com/matrix-org/synapse/issues/3543>))
+- Release notes are now in the Markdown format. ([\#3552](https://github.com/matrix-org/synapse/issues/3552>))
+- add config for pep8 ([\#3559](https://github.com/matrix-org/synapse/issues/3559>))
+- Merge Linearizer and Limiter ([\#3571](https://github.com/matrix-org/synapse/issues/3571>), [\#3572](https://github.com/matrix-org/synapse/issues/3572>))
+- Lazily load state on master process when using workers to reduce DB consumption ([\#3579](https://github.com/matrix-org/synapse/issues/3579>), [\#3581](https://github.com/matrix-org/synapse/issues/3581>), [\#3582](https://github.com/matrix-org/synapse/issues/3582>), [\#3584](https://github.com/matrix-org/synapse/issues/3584>))
+- Fixes and optimisations for resolve_state_groups ([\#3586](https://github.com/matrix-org/synapse/issues/3586>))
+- Improve logging for exceptions when handling PDUs ([\#3587](https://github.com/matrix-org/synapse/issues/3587>))
+- Add some measure blocks to persist_events ([\#3590](https://github.com/matrix-org/synapse/issues/3590>))
+- Fix some random logcontext leaks. ([\#3591](https://github.com/matrix-org/synapse/issues/3591>), [\#3606](https://github.com/matrix-org/synapse/issues/3606>))
+- Speed up calculating state deltas in persist_event loop ([\#3592](https://github.com/matrix-org/synapse/issues/3592>))
+- Attempt to reduce amount of state pulled out of DB during persist_events ([\#3595](https://github.com/matrix-org/synapse/issues/3595>))
+- Fix a documentation typo in on_make_leave_request ([\#3609](https://github.com/matrix-org/synapse/issues/3609>))
+- Make EventStore inherit from EventFederationStore ([\#3612](https://github.com/matrix-org/synapse/issues/3612>))
+- Remove some redundant joins on event_edges.room_id ([\#3613](https://github.com/matrix-org/synapse/issues/3613>))
+- Stop populating events.content ([\#3614](https://github.com/matrix-org/synapse/issues/3614>))
+- Update the /send_leave path registration to use event_id rather than a transaction ID. ([\#3616](https://github.com/matrix-org/synapse/issues/3616>))
+- Refactor FederationHandler to move DB writes into separate functions ([\#3621](https://github.com/matrix-org/synapse/issues/3621>))
+- Remove unused field "pdu_failures" from transactions. ([\#3628](https://github.com/matrix-org/synapse/issues/3628>))
+- rename replication_layer to federation_client ([\#3634](https://github.com/matrix-org/synapse/issues/3634>))
+- Factor out exception handling in federation_client ([\#3638](https://github.com/matrix-org/synapse/issues/3638>))
+- Refactor location of docker build script. ([\#3644](https://github.com/matrix-org/synapse/issues/3644>))
+- Update CONTRIBUTING to mention newsfragments. ([\#3645](https://github.com/matrix-org/synapse/issues/3645>))
+
+
 Synapse 0.33.1 (2018-08-02)
 ===========================
 
diff --git a/changelog.d/2952.bugfix b/changelog.d/2952.bugfix
deleted file mode 100644
index 07a3e483048d..000000000000
--- a/changelog.d/2952.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Make /directory/list API return 404 for room not found instead of 400
diff --git a/changelog.d/2970.feature b/changelog.d/2970.feature
deleted file mode 100644
index 5eb928563fea..000000000000
--- a/changelog.d/2970.feature
+++ /dev/null
@@ -1 +0,0 @@
-add support for the lazy_loaded_members filter as per MSC1227
diff --git a/changelog.d/3331.feature b/changelog.d/3331.feature
deleted file mode 100644
index e574b9bcc3af..000000000000
--- a/changelog.d/3331.feature
+++ /dev/null
@@ -1 +0,0 @@
-add support for the include_redundant_members filter param as per MSC1227
diff --git a/changelog.d/3350.misc b/changelog.d/3350.misc
deleted file mode 100644
index 3713cd6d635e..000000000000
--- a/changelog.d/3350.misc
+++ /dev/null
@@ -1 +0,0 @@
-Remove redundant checks on who_forgot_in_room
\ No newline at end of file
diff --git a/changelog.d/3367.misc b/changelog.d/3367.misc
deleted file mode 100644
index 1f21ddea488a..000000000000
--- a/changelog.d/3367.misc
+++ /dev/null
@@ -1 +0,0 @@
-Remove unnecessary event re-signing hacks
\ No newline at end of file
diff --git a/changelog.d/3384.misc b/changelog.d/3384.misc
deleted file mode 100644
index 5d56c876d90b..000000000000
--- a/changelog.d/3384.misc
+++ /dev/null
@@ -1 +0,0 @@
-Rewrite cache list decorator
diff --git a/changelog.d/3391.bugfix b/changelog.d/3391.bugfix
deleted file mode 100644
index 88eeb50df233..000000000000
--- a/changelog.d/3391.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Default inviter_display_name to mxid for email invites
\ No newline at end of file
diff --git a/changelog.d/3415.misc b/changelog.d/3415.misc
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/changelog.d/3460.misc b/changelog.d/3460.misc
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/changelog.d/3514.bugfix b/changelog.d/3514.bugfix
deleted file mode 100644
index 460fe24ac9e2..000000000000
--- a/changelog.d/3514.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Don't generate TURN credentials if no TURN config options are set
diff --git a/changelog.d/3520.bugfix b/changelog.d/3520.bugfix
deleted file mode 100644
index 9278cb3708db..000000000000
--- a/changelog.d/3520.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Correctly announce deleted devices over federation
diff --git a/changelog.d/3543.misc b/changelog.d/3543.misc
deleted file mode 100644
index d231d17749b5..000000000000
--- a/changelog.d/3543.misc
+++ /dev/null
@@ -1 +0,0 @@
-Improve Dockerfile and docker-compose instructions
diff --git a/changelog.d/3548.bugfix b/changelog.d/3548.bugfix
deleted file mode 100644
index 38dc3b1232a4..000000000000
--- a/changelog.d/3548.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Catch failures saving metrics captured by Measure, and instead log the faulty metrics information for further analysis.
diff --git a/changelog.d/3552.misc b/changelog.d/3552.misc
deleted file mode 100644
index 709c3282b4c3..000000000000
--- a/changelog.d/3552.misc
+++ /dev/null
@@ -1 +0,0 @@
-Release notes are now in the Markdown format.
diff --git a/changelog.d/3553.feature b/changelog.d/3553.feature
deleted file mode 100644
index 77a294cb9f42..000000000000
--- a/changelog.d/3553.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add metrics to track resource usage by background processes
diff --git a/changelog.d/3554.feature b/changelog.d/3554.feature
deleted file mode 100644
index b00397872c4c..000000000000
--- a/changelog.d/3554.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add `code` label to `synapse_http_server_response_time_seconds` prometheus metric
diff --git a/changelog.d/3555.feature b/changelog.d/3555.feature
deleted file mode 100644
index ea4a85e0aee7..000000000000
--- a/changelog.d/3555.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add support for client_reader to handle more APIs
diff --git a/changelog.d/3556.feature b/changelog.d/3556.feature
deleted file mode 100644
index 77a294cb9f42..000000000000
--- a/changelog.d/3556.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add metrics to track resource usage by background processes
diff --git a/changelog.d/3559.misc b/changelog.d/3559.misc
deleted file mode 100644
index 26df859e4563..000000000000
--- a/changelog.d/3559.misc
+++ /dev/null
@@ -1 +0,0 @@
-add config for pep8
diff --git a/changelog.d/3562.misc b/changelog.d/3562.misc
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/changelog.d/3567.feature b/changelog.d/3567.feature
deleted file mode 100644
index c74c1f57a90a..000000000000
--- a/changelog.d/3567.feature
+++ /dev/null
@@ -1 +0,0 @@
-make the /context API filter & lazy-load aware as per MSC1227
diff --git a/changelog.d/3569.bugfix b/changelog.d/3569.bugfix
deleted file mode 100644
index d77f035ee0e1..000000000000
--- a/changelog.d/3569.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Unicode passwords are now normalised before hashing, preventing the instance where two different devices or browsers might send a different UTF-8 sequence for the password.
diff --git a/changelog.d/3570.bugfix b/changelog.d/3570.bugfix
deleted file mode 100644
index cec5158a9902..000000000000
--- a/changelog.d/3570.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix potential stack overflow and deadlock under heavy load
\ No newline at end of file
diff --git a/changelog.d/3571.misc b/changelog.d/3571.misc
deleted file mode 100644
index 8908324e6803..000000000000
--- a/changelog.d/3571.misc
+++ /dev/null
@@ -1 +0,0 @@
-Merge Linearizer and Limiter
diff --git a/changelog.d/3572.misc b/changelog.d/3572.misc
deleted file mode 100644
index 8908324e6803..000000000000
--- a/changelog.d/3572.misc
+++ /dev/null
@@ -1 +0,0 @@
-Merge Linearizer and Limiter
diff --git a/changelog.d/3577.misc b/changelog.d/3577.misc
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/changelog.d/3579.misc b/changelog.d/3579.misc
deleted file mode 100644
index 2374dc0c44d5..000000000000
--- a/changelog.d/3579.misc
+++ /dev/null
@@ -1 +0,0 @@
-Lazily load state on master process when using workers to reduce DB consumption
diff --git a/changelog.d/3581.misc b/changelog.d/3581.misc
deleted file mode 100644
index 2374dc0c44d5..000000000000
--- a/changelog.d/3581.misc
+++ /dev/null
@@ -1 +0,0 @@
-Lazily load state on master process when using workers to reduce DB consumption
diff --git a/changelog.d/3582.misc b/changelog.d/3582.misc
deleted file mode 100644
index 2374dc0c44d5..000000000000
--- a/changelog.d/3582.misc
+++ /dev/null
@@ -1 +0,0 @@
-Lazily load state on master process when using workers to reduce DB consumption
diff --git a/changelog.d/3584.misc b/changelog.d/3584.misc
deleted file mode 100644
index 2374dc0c44d5..000000000000
--- a/changelog.d/3584.misc
+++ /dev/null
@@ -1 +0,0 @@
-Lazily load state on master process when using workers to reduce DB consumption
diff --git a/changelog.d/3585.bugfix b/changelog.d/3585.bugfix
deleted file mode 100644
index e8ae1d8cb4c9..000000000000
--- a/changelog.d/3585.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Respond with M_NOT_FOUND when profiles are not found locally or over federation. Fixes #3585
diff --git a/changelog.d/3586.misc b/changelog.d/3586.misc
deleted file mode 100644
index e853e2481bcd..000000000000
--- a/changelog.d/3586.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fixes and optimisations for resolve_state_groups
diff --git a/changelog.d/3587.misc b/changelog.d/3587.misc
deleted file mode 100644
index 75a3479910b8..000000000000
--- a/changelog.d/3587.misc
+++ /dev/null
@@ -1 +0,0 @@
-Improve logging for exceptions when handling PDUs
\ No newline at end of file
diff --git a/changelog.d/3590.misc b/changelog.d/3590.misc
deleted file mode 100644
index 0f1688fd0f73..000000000000
--- a/changelog.d/3590.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add some measure blocks to persist_events
diff --git a/changelog.d/3591.misc b/changelog.d/3591.misc
deleted file mode 100644
index f0137766a066..000000000000
--- a/changelog.d/3591.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix some random logcontext leaks.
\ No newline at end of file
diff --git a/changelog.d/3592.misc b/changelog.d/3592.misc
deleted file mode 100644
index 60129569c278..000000000000
--- a/changelog.d/3592.misc
+++ /dev/null
@@ -1 +0,0 @@
-Speed up calculating state deltas in persist_event loop
diff --git a/changelog.d/3595.misc b/changelog.d/3595.misc
deleted file mode 100644
index 85903504cc2f..000000000000
--- a/changelog.d/3595.misc
+++ /dev/null
@@ -1 +0,0 @@
-Attempt to reduce amount of state pulled out of DB during persist_events
diff --git a/changelog.d/3597.feature b/changelog.d/3597.feature
deleted file mode 100644
index ea4a85e0aee7..000000000000
--- a/changelog.d/3597.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add support for client_reader to handle more APIs
diff --git a/changelog.d/3601.bugfix b/changelog.d/3601.bugfix
deleted file mode 100644
index 1678b261d049..000000000000
--- a/changelog.d/3601.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix failure to persist events over federation under load
diff --git a/changelog.d/3604.feature b/changelog.d/3604.feature
deleted file mode 100644
index 77a294cb9f42..000000000000
--- a/changelog.d/3604.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add metrics to track resource usage by background processes
diff --git a/changelog.d/3605.bugfix b/changelog.d/3605.bugfix
deleted file mode 100644
index 786da546ebe9..000000000000
--- a/changelog.d/3605.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix updating of cached remote profiles
diff --git a/changelog.d/3606.misc b/changelog.d/3606.misc
deleted file mode 100644
index f0137766a066..000000000000
--- a/changelog.d/3606.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix some random logcontext leaks.
\ No newline at end of file
diff --git a/changelog.d/3607.bugfix b/changelog.d/3607.bugfix
deleted file mode 100644
index 7ad64593b8f2..000000000000
--- a/changelog.d/3607.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix 'tuple index out of range' error
\ No newline at end of file
diff --git a/changelog.d/3609.misc b/changelog.d/3609.misc
deleted file mode 100644
index 5b9566d0761f..000000000000
--- a/changelog.d/3609.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix a documentation typo in on_make_leave_request
diff --git a/changelog.d/3610.feature b/changelog.d/3610.feature
deleted file mode 100644
index 77a294cb9f42..000000000000
--- a/changelog.d/3610.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add metrics to track resource usage by background processes
diff --git a/changelog.d/3612.misc b/changelog.d/3612.misc
deleted file mode 100644
index f90d2f2ff518..000000000000
--- a/changelog.d/3612.misc
+++ /dev/null
@@ -1 +0,0 @@
-Make EventStore inherit from EventFederationStore
diff --git a/changelog.d/3613.misc b/changelog.d/3613.misc
deleted file mode 100644
index d9378f6b49a3..000000000000
--- a/changelog.d/3613.misc
+++ /dev/null
@@ -1 +0,0 @@
-Remove some redundant joins on event_edges.room_id
diff --git a/changelog.d/3614.misc b/changelog.d/3614.misc
deleted file mode 100644
index 356f28471eaa..000000000000
--- a/changelog.d/3614.misc
+++ /dev/null
@@ -1 +0,0 @@
-Stop populating events.content
diff --git a/changelog.d/3616.misc b/changelog.d/3616.misc
deleted file mode 100644
index 04629faa367b..000000000000
--- a/changelog.d/3616.misc
+++ /dev/null
@@ -1 +0,0 @@
-Update the /send_leave path registration to use event_id rather than a transaction ID.
diff --git a/changelog.d/3621.misc b/changelog.d/3621.misc
deleted file mode 100644
index 83e5fc8aa1ac..000000000000
--- a/changelog.d/3621.misc
+++ /dev/null
@@ -1 +0,0 @@
-Refactor FederationHandler to move DB writes into separate functions
diff --git a/changelog.d/3626.bugfix b/changelog.d/3626.bugfix
deleted file mode 100644
index 9a4b87898615..000000000000
--- a/changelog.d/3626.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Only import secrets when available (fix for py < 3.6)
diff --git a/changelog.d/3628.misc b/changelog.d/3628.misc
deleted file mode 100644
index 1aebefbe186e..000000000000
--- a/changelog.d/3628.misc
+++ /dev/null
@@ -1 +0,0 @@
-Remove unused field "pdu_failures" from transactions.
diff --git a/changelog.d/3630.feature b/changelog.d/3630.feature
deleted file mode 100644
index 8007a048405b..000000000000
--- a/changelog.d/3630.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add ability to limit number of monthly active users on the server
diff --git a/changelog.d/3634.misc b/changelog.d/3634.misc
deleted file mode 100644
index 2cd6af91ff16..000000000000
--- a/changelog.d/3634.misc
+++ /dev/null
@@ -1 +0,0 @@
-rename replication_layer to federation_client
diff --git a/changelog.d/3638.misc b/changelog.d/3638.misc
deleted file mode 100644
index 9faab15cf265..000000000000
--- a/changelog.d/3638.misc
+++ /dev/null
@@ -1 +0,0 @@
-Factor out exception handling in federation_client
diff --git a/changelog.d/3639.feature b/changelog.d/3639.feature
deleted file mode 100644
index c8c387e219c1..000000000000
--- a/changelog.d/3639.feature
+++ /dev/null
@@ -1 +0,0 @@
-When we fail to join a room over federation, pass the error code back to the client.
\ No newline at end of file
diff --git a/changelog.d/3644.misc b/changelog.d/3644.misc
deleted file mode 100644
index 2347fc850047..000000000000
--- a/changelog.d/3644.misc
+++ /dev/null
@@ -1 +0,0 @@
-Refactor location of docker build script.
diff --git a/changelog.d/3645.misc b/changelog.d/3645.misc
deleted file mode 100644
index 0fe6b28da173..000000000000
--- a/changelog.d/3645.misc
+++ /dev/null
@@ -1 +0,0 @@
-Update CONTRIBUTING to mention newsfragments.

From 848431be1d586a6c91a9defcc09e30e817961768 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Tue, 7 Aug 2018 21:13:22 +1000
Subject: [PATCH 288/830] fix for rc1

---
 CHANGES.md          | 4 ++--
 synapse/__init__.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 83a48c8b2621..b9caf34c0dba 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,5 @@
-Synapse 0.33.2 (2018-08-07)
-===========================
+Synapse 0.33.2rc1 (2018-08-07)
+==============================
 
 Features
 --------
diff --git a/synapse/__init__.py b/synapse/__init__.py
index a14d578e36a7..807d78d79f91 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -17,4 +17,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.33.2"
+__version__ = "0.33.2rc1"

From 23cd86554ecde6fc85ef842c3768eacfb5cd6f91 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Tue, 7 Aug 2018 21:20:42 +1000
Subject: [PATCH 289/830] fix changelog

---
 CHANGES.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/CHANGES.md b/CHANGES.md
index b9caf34c0dba..299780e9ec84 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -12,6 +12,7 @@ Features
 - make the /context API filter & lazy-load aware as per MSC1227 ([\#3567](https://github.com/matrix-org/synapse/issues/3567>))
 - Add ability to limit number of monthly active users on the server ([\#3630](https://github.com/matrix-org/synapse/issues/3630>))
 - When we fail to join a room over federation, pass the error code back to the client. ([\#3639](https://github.com/matrix-org/synapse/issues/3639>))
+- Add a new /admin/register API for non-interactively creating users. ([\#3415](https://github.com/matrix-org/synapse/issues/3415>))
 
 
 Bugfixes
@@ -37,7 +38,9 @@ Internal Changes
 - Remove redundant checks on who_forgot_in_room ([\#3350](https://github.com/matrix-org/synapse/issues/3350>))
 - Remove unnecessary event re-signing hacks ([\#3367](https://github.com/matrix-org/synapse/issues/3367>))
 - Rewrite cache list decorator ([\#3384](https://github.com/matrix-org/synapse/issues/3384>))
--  ([\#3415](https://github.com/matrix-org/synapse/issues/3415>), [\#3460](https://github.com/matrix-org/synapse/issues/3460>), [\#3562](https://github.com/matrix-org/synapse/issues/3562>), [\#3577](https://github.com/matrix-org/synapse/issues/3577>))
+- Move v1-only REST APIs into their own module. ([\#3460](https://github.com/matrix-org/synapse/issues/3460>))
+- Replace more instances of Python 2-only iteritems and itervalues uses. ([\#3562](https://github.com/matrix-org/synapse/issues/3562>))
+- Refactor EventContext to accept state during init ([\#3577](https://github.com/matrix-org/synapse/issues/3577>))
 - Improve Dockerfile and docker-compose instructions ([\#3543](https://github.com/matrix-org/synapse/issues/3543>))
 - Release notes are now in the Markdown format. ([\#3552](https://github.com/matrix-org/synapse/issues/3552>))
 - add config for pep8 ([\#3559](https://github.com/matrix-org/synapse/issues/3559>))

From 9b92720d88f27c2961d4c0e078cfb2205f248dec Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 7 Aug 2018 00:41:23 +0100
Subject: [PATCH 290/830] fix event lag graph

---
 contrib/grafana/synapse.json | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/contrib/grafana/synapse.json b/contrib/grafana/synapse.json
index 94a1de58f4e3..c58612594ab8 100644
--- a/contrib/grafana/synapse.json
+++ b/contrib/grafana/synapse.json
@@ -54,7 +54,7 @@
   "gnetId": null,
   "graphTooltip": 0,
   "id": null,
-  "iteration": 1533026624326,
+  "iteration": 1533598785368,
   "links": [
     {
       "asDropdown": true,
@@ -4629,7 +4629,7 @@
             "h": 9,
             "w": 12,
             "x": 0,
-            "y": 11
+            "y": 29
           },
           "id": 67,
           "legend": {
@@ -4655,11 +4655,11 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": " synapse_event_persisted_position{instance=\"$instance\"} - ignoring(index, job, name) group_right(instance) synapse_event_processing_positions{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+              "expr": " synapse_event_persisted_position{instance=\"$instance\",job=\"synapse\"}  - ignoring(index, job, name) group_right() synapse_event_processing_positions{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{job}}-{{index}}",
+              "legendFormat": "{{job}}-{{index}} ",
               "refId": "A"
             }
           ],
@@ -4697,7 +4697,11 @@
               "min": null,
               "show": true
             }
-          ]
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
         },
         {
           "aliasColors": {},
@@ -4710,7 +4714,7 @@
             "h": 9,
             "w": 12,
             "x": 12,
-            "y": 11
+            "y": 29
           },
           "id": 71,
           "legend": {
@@ -4778,7 +4782,11 @@
               "min": null,
               "show": true
             }
-          ]
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
         }
       ],
       "title": "Event processing loop positions",
@@ -4957,5 +4965,5 @@
   "timezone": "",
   "title": "Synapse",
   "uid": "000000012",
-  "version": 125
+  "version": 127
 }
\ No newline at end of file

From 3523f5432a79659ac365dc2ff1212aa1a3707d8e Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 7 Aug 2018 12:51:57 +0100
Subject: [PATCH 291/830] Don't expose default_room_version as config opt

---
 synapse/api/constants.py |  3 +++
 synapse/config/server.py | 14 --------------
 synapse/handlers/room.py |  3 ++-
 3 files changed, 5 insertions(+), 15 deletions(-)

diff --git a/synapse/api/constants.py b/synapse/api/constants.py
index a27bf3b32d92..b0da506f6d64 100644
--- a/synapse/api/constants.py
+++ b/synapse/api/constants.py
@@ -97,6 +97,9 @@ class ThirdPartyEntityKind(object):
     LOCATION = "location"
 
 
+# the version we will give rooms which are created on this server
+DEFAULT_ROOM_VERSION = "1"
+
 # vdh-test-version is a placeholder to get room versioning support working and tested
 # until we have a working v2.
 KNOWN_ROOM_VERSIONS = {"1", "vdh-test-version"}
diff --git a/synapse/config/server.py b/synapse/config/server.py
index 68ef2789d3c9..6a471a0a5ec7 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -16,7 +16,6 @@
 
 import logging
 
-from synapse.api.constants import KNOWN_ROOM_VERSIONS
 from synapse.http.endpoint import parse_and_validate_server_name
 
 from ._base import Config, ConfigError
@@ -76,16 +75,6 @@ def read_config(self, config):
             )
         else:
             self.max_mau_value = 0
-
-        # the version of rooms created by default on this server
-        self.default_room_version = str(config.get(
-            "default_room_version", "1",
-        ))
-        if self.default_room_version not in KNOWN_ROOM_VERSIONS:
-            raise ConfigError("Unrecognised value '%s' for default_room_version" % (
-                self.default_room_version,
-            ))
-
         # FIXME: federation_domain_whitelist needs sytests
         self.federation_domain_whitelist = None
         federation_domain_whitelist = config.get(
@@ -260,9 +249,6 @@ def default_config(self, server_name, **kwargs):
         # (except those sent by local server admins). The default is False.
         # block_non_admin_invites: True
 
-        # The room_version of rooms which are created by default by this server.
-        # default_room_version: 1
-
         # Restrict federation to the following whitelist of domains.
         # N.B. we recommend also firewalling your federation listener to limit
         # inbound federation traffic as early as possible, rather than relying
diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index a526b684e9dd..6a17c42238ec 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -26,6 +26,7 @@
 from twisted.internet import defer
 
 from synapse.api.constants import (
+    DEFAULT_ROOM_VERSION,
     KNOWN_ROOM_VERSIONS,
     EventTypes,
     JoinRules,
@@ -106,7 +107,7 @@ def create_room(self, requester, config, ratelimit=True,
         if ratelimit:
             yield self.ratelimit(requester)
 
-        room_version = config.get("room_version", self.hs.config.default_room_version)
+        room_version = config.get("room_version", DEFAULT_ROOM_VERSION)
         if not isinstance(room_version, string_types):
             raise SynapseError(
                 400,

From e8eba2b4e3a99d35f08c96205ebb18211adddcb9 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Tue, 7 Aug 2018 17:49:43 +0100
Subject: [PATCH 292/830] implement reserved users for mau limits

---
 synapse/app/homeserver.py                  |  6 +++
 synapse/config/server.py                   |  2 +-
 synapse/storage/monthly_active_users.py    | 45 +++++++++++++----
 tests/storage/test_monthly_active_users.py | 59 +++++++++++++++++++++-
 4 files changed, 99 insertions(+), 13 deletions(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 3a67db8b30db..a4a65e728660 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -518,6 +518,8 @@ def generate_user_daily_visit_stats():
     # If you increase the loop period, the accuracy of user_daily_visits
     # table will decrease
     clock.looping_call(generate_user_daily_visit_stats, 5 * 60 * 1000)
+
+    # monthly active user limiting functionality
     clock.looping_call(
         hs.get_datastore().reap_monthly_active_users, 1000 * 60 * 60
     )
@@ -530,9 +532,13 @@ def generate_monthly_active_users():
         current_mau_gauge.set(float(count))
         max_mau_value_gauge.set(float(hs.config.max_mau_value))
 
+    hs.get_datastore().initialise_reserved_users(
+        hs.config.mau_limits_reserved_threepids
+    )
     generate_monthly_active_users()
     if hs.config.limit_usage_by_mau:
         clock.looping_call(generate_monthly_active_users, 5 * 60 * 1000)
+    # End of monthly active user settings
 
     if hs.config.report_stats:
         logger.info("Scheduling stats reporting for 3 hour intervals")
diff --git a/synapse/config/server.py b/synapse/config/server.py
index cacac253ab0f..114d7a981595 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -75,7 +75,7 @@ def read_config(self, config):
                 "max_mau_value", 0,
             )
         self.mau_limits_reserved_threepids = config.get(
-            "mau_limit_reserved_threepid", []
+            "mau_limit_reserved_threepids", []
         )
 
         # FIXME: federation_domain_whitelist needs sytests
diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py
index 6a37d6fc22af..168f564ed5d1 100644
--- a/synapse/storage/monthly_active_users.py
+++ b/synapse/storage/monthly_active_users.py
@@ -12,6 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import logging
 
 from twisted.internet import defer
 
@@ -19,6 +20,7 @@
 
 from ._base import SQLBaseStore
 
+logger = logging.getLogger(__name__)
 
 # Number of msec of granularity to store the monthly_active_user timestamp
 # This means it is not necessary to update the table on every request
@@ -26,24 +28,31 @@
 
 
 class MonthlyActiveUsersStore(SQLBaseStore):
-    @defer.inlineCallbacks
     def __init__(self, dbconn, hs):
         super(MonthlyActiveUsersStore, self).__init__(None, hs)
         self._clock = hs.get_clock()
         self.hs = hs
-        threepids = self.hs.config.mau_limits_reserved_threepids
-        self.reserved_user_ids = set()
+        self.reserved_users = ()
+
+    @defer.inlineCallbacks
+    def initialise_reserved_users(self, threepids):
+        # TODO Why can't I do this in init?
+        store = self.hs.get_datastore()
+        reserved_user_list = []
         for tp in threepids:
-            user_id = yield hs.get_datastore().get_user_id_by_threepid(
+            user_id = yield store.get_user_id_by_threepid(
                 tp["medium"], tp["address"]
             )
             if user_id:
-                self.reserved_user_ids.add(user_id)
+                self.upsert_monthly_active_user(user_id)
+                reserved_user_list.append(user_id)
             else:
                 logger.warning(
                     "mau limit reserved threepid %s not found in db" % tp
                 )
+        self.reserved_users = tuple(reserved_user_list)
 
+    @defer.inlineCallbacks
     def reap_monthly_active_users(self):
         """
         Cleans out monthly active user table to ensure that no stale
@@ -58,8 +67,20 @@ def _reap_users(txn):
                 int(self._clock.time_msec()) - (1000 * 60 * 60 * 24 * 30)
             )
             # Purge stale users
-            sql = "DELETE FROM monthly_active_users WHERE timestamp < ?"
-            txn.execute(sql, (thirty_days_ago,))
+
+            # questionmarks is a hack to overcome sqlite not supporting
+            # tuples in 'WHERE IN %s'
+            questionmarks = '?' * len(self.reserved_users)
+            query_args = [thirty_days_ago]
+            query_args.extend(self.reserved_users)
+
+            sql = """
+                DELETE FROM monthly_active_users
+                WHERE timestamp < ?
+                AND user_id NOT IN ({})
+                """.format(','.join(questionmarks))
+
+            txn.execute(sql, query_args)
 
             # If MAU user count still exceeds the MAU threshold, then delete on
             # a least recently active basis.
@@ -69,6 +90,8 @@ def _reap_users(txn):
             # While Postgres does not require 'LIMIT', but also does not support
             # negative LIMIT values. So there is no way to write it that both can
             # support
+            query_args = [self.hs.config.max_mau_value]
+            query_args.extend(self.reserved_users)
             sql = """
                 DELETE FROM monthly_active_users
                 WHERE user_id NOT IN (
@@ -76,8 +99,9 @@ def _reap_users(txn):
                     ORDER BY timestamp DESC
                     LIMIT ?
                     )
-                """
-            txn.execute(sql, (self.hs.config.max_mau_value,))
+                AND user_id NOT IN ({})
+                """.format(','.join(questionmarks))
+            txn.execute(sql, query_args)
 
         yield self.runInteraction("reap_monthly_active_users", _reap_users)
         # It seems poor to invalidate the whole cache, Postgres supports
@@ -136,7 +160,7 @@ def _user_last_seen_monthly_active(self, user_id):
             Arguments:
                 user_id (str): user to add/update
             Return:
-                int : timestamp since last seen, None if never seen
+                Deferred[int] : timestamp since last seen, None if never seen
 
         """
 
@@ -158,7 +182,6 @@ def populate_monthly_active_users(self, user_id):
         Args:
             user_id(str): the user_id to query
         """
-
         if self.hs.config.limit_usage_by_mau:
             last_seen_timestamp = yield self._user_last_seen_monthly_active(user_id)
             now = self.hs.get_clock().time_msec()
diff --git a/tests/storage/test_monthly_active_users.py b/tests/storage/test_monthly_active_users.py
index 0bfd24a7fb3d..cbd480cd4215 100644
--- a/tests/storage/test_monthly_active_users.py
+++ b/tests/storage/test_monthly_active_users.py
@@ -19,6 +19,8 @@
 import tests.utils
 from tests.utils import setup_test_homeserver
 
+FORTY_DAYS = 40 * 24 * 60 * 60
+
 
 class MonthlyActiveUsersTestCase(tests.unittest.TestCase):
     def __init__(self, *args, **kwargs):
@@ -29,6 +31,56 @@ def setUp(self):
         self.hs = yield setup_test_homeserver()
         self.store = self.hs.get_datastore()
 
+    @defer.inlineCallbacks
+    def test_initialise_reserved_users(self):
+
+        user1 = "@user1:server"
+        user1_email = "user1@matrix.org"
+        user2 = "@user2:server"
+        user2_email = "user2@matrix.org"
+        threepids = [
+            {'medium': 'email', 'address': user1_email},
+            {'medium': 'email', 'address': user2_email}
+        ]
+        user_num = len(threepids)
+
+        yield self.store.register(
+            user_id=user1,
+            token="123",
+            password_hash=None)
+
+        yield self.store.register(
+            user_id=user2,
+            token="456",
+            password_hash=None)
+
+        now = int(self.hs.get_clock().time_msec())
+        yield self.store.user_add_threepid(user1, "email", user1_email, now, now)
+        yield self.store.user_add_threepid(user2, "email", user2_email, now, now)
+        yield self.store.initialise_reserved_users(threepids)
+
+        active_count = yield self.store.get_monthly_active_count()
+
+        # Test total counts
+        self.assertEquals(active_count, user_num)
+
+        # Test user is marked as active
+
+        timestamp = yield self.store._user_last_seen_monthly_active(user1)
+        self.assertTrue(timestamp)
+        timestamp = yield self.store._user_last_seen_monthly_active(user2)
+        self.assertTrue(timestamp)
+
+        # Test that users are never removed from the db.
+        self.hs.config.max_mau_value = 0
+
+        self.hs.get_clock().advance_time(FORTY_DAYS)
+
+        yield self.store.reap_monthly_active_users()
+
+        active_count = yield self.store.get_monthly_active_count()
+        self.assertEquals(active_count, user_num)
+
     @defer.inlineCallbacks
     def test_can_insert_and_count_mau(self):
         count = yield self.store.get_monthly_active_count()
@@ -63,4 +115,9 @@ def test_reap_monthly_active_users(self):
         self.assertTrue(count, initial_users)
         yield self.store.reap_monthly_active_users()
         count = yield self.store.get_monthly_active_count()
-        self.assertTrue(count, initial_users - self.hs.config.max_mau_value)
+        self.assertEquals(count, initial_users - self.hs.config.max_mau_value)
+
+        self.hs.get_clock().advance_time(FORTY_DAYS)
+        yield self.store.reap_monthly_active_users()
+        count = yield self.store.get_monthly_active_count()
+        self.assertEquals(count, 0)

From ef3589063adddd1eee89f136e6a54250cce414a1 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Tue, 7 Aug 2018 17:59:27 +0100
Subject: [PATCH 293/830] prevent total number of reserved users being too
 large

---
 synapse/storage/monthly_active_users.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py
index 168f564ed5d1..54de5a8686fa 100644
--- a/synapse/storage/monthly_active_users.py
+++ b/synapse/storage/monthly_active_users.py
@@ -39,7 +39,9 @@ def initialise_reserved_users(self, threepids):
         # TODO Why can't I do this in init?
         store = self.hs.get_datastore()
         reserved_user_list = []
-        for tp in threepids:
+
+        # Do not add more reserved users than the total allowable number
+        for tp in threepids[:self.hs.config.max_mau_value]:
             user_id = yield store.get_user_id_by_threepid(
                 tp["medium"], tp["address"]
             )

From 53bca4690b5c94fb1506dbc628ce2ef0f770745f Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 7 Aug 2018 19:09:48 +0100
Subject: [PATCH 294/830] more metrics for the federation and appservice
 senders

---
 synapse/federation/transaction_queue.py | 10 +++++++++-
 synapse/handlers/appservice.py          | 10 ++++++++++
 synapse/metrics/__init__.py             | 13 +++++++++++++
 3 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py
index 78f9d40a3ad1..f603c8a368aa 100644
--- a/synapse/federation/transaction_queue.py
+++ b/synapse/federation/transaction_queue.py
@@ -26,6 +26,8 @@
 from synapse.handlers.presence import format_user_presence_state, get_interested_remotes
 from synapse.metrics import (
     LaterGauge,
+    event_processing_loop_counter,
+    event_processing_loop_room_count,
     events_processed_counter,
     sent_edus_counter,
     sent_transactions_counter,
@@ -253,7 +255,13 @@ def handle_room_events(events):
                     synapse.metrics.event_processing_last_ts.labels(
                         "federation_sender").set(ts)
 
-                events_processed_counter.inc(len(events))
+                    events_processed_counter.inc(len(events))
+
+                    event_processing_loop_room_count.labels(
+                        "federation_sender"
+                    ).inc(len(events_by_room))
+
+                event_processing_loop_counter.labels("federation_sender").inc()
 
                 synapse.metrics.event_processing_positions.labels(
                     "federation_sender").set(next_token)
diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py
index ee41aed69e46..f0f89af7dcda 100644
--- a/synapse/handlers/appservice.py
+++ b/synapse/handlers/appservice.py
@@ -23,6 +23,10 @@
 
 import synapse
 from synapse.api.constants import EventTypes
+from synapse.metrics import (
+    event_processing_loop_counter,
+    event_processing_loop_room_count,
+)
 from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.util.logcontext import make_deferred_yieldable, run_in_background
 from synapse.util.metrics import Measure
@@ -136,6 +140,12 @@ def handle_room_events(events):
 
                     events_processed_counter.inc(len(events))
 
+                    event_processing_loop_room_count.labels(
+                        "appservice_sender"
+                    ).inc(len(events_by_room))
+
+                    event_processing_loop_counter.labels("appservice_sender").inc()
+
                     synapse.metrics.event_processing_lag.labels(
                         "appservice_sender").set(now - ts)
                     synapse.metrics.event_processing_last_ts.labels(
diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index a9158fc0669a..b7cb3a730a77 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -174,6 +174,19 @@ def collect(self):
 
 events_processed_counter = Counter("synapse_federation_client_events_processed", "")
 
+event_processing_loop_counter = Counter(
+    "synapse_event_processing_loop",
+    "Event processing loop iterations",
+    ["name"],
+)
+
+event_processing_loop_room_count = Counter(
+    "synapse_event_processing_loop_room_count",
+    "Rooms seen per event processing loop iteration",
+    ["name"],
+)
+
+
 # Used to track where various components have processed in the event stream,
 # e.g. federation sending, appservice sending, etc.
 event_processing_positions = Gauge("synapse_event_processing_positions", "", ["name"])

From bab94da79cb71cc483184c2732707e40f7a708ce Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 7 Aug 2018 22:11:45 +0100
Subject: [PATCH 295/830] fix metric name

---
 synapse/metrics/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index b7cb3a730a77..550f8443f78a 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -175,7 +175,7 @@ def collect(self):
 events_processed_counter = Counter("synapse_federation_client_events_processed", "")
 
 event_processing_loop_counter = Counter(
-    "synapse_event_processing_loop",
+    "synapse_event_processing_loop_count",
     "Event processing loop iterations",
     ["name"],
 )

From e6d73b858231567abfe47b05d3e04376c9016b1a Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 7 Aug 2018 22:14:35 +0100
Subject: [PATCH 296/830] changelog

---
 changelog.d/3664.feature | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3664.feature

diff --git a/changelog.d/3664.feature b/changelog.d/3664.feature
new file mode 100644
index 000000000000..184dde99393d
--- /dev/null
+++ b/changelog.d/3664.feature
@@ -0,0 +1 @@
+Add some metrics for the appservice and federation event sending loops

From 501141763245647f41636621867ad1bacc47e6b5 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 8 Aug 2018 10:29:58 +0100
Subject: [PATCH 297/830] Fixup logging and docstrings

---
 synapse/replication/http/_base.py      |  6 +++--
 synapse/replication/http/membership.py | 36 ++++++++++++++++++++++++++
 2 files changed, 40 insertions(+), 2 deletions(-)

diff --git a/synapse/replication/http/_base.py b/synapse/replication/http/_base.py
index 4de3825fdab5..619ddab54081 100644
--- a/synapse/replication/http/_base.py
+++ b/synapse/replication/http/_base.py
@@ -151,7 +151,7 @@ def send_request(**kwargs):
                         if e.code != 504 or not cls.RETRY_ON_TIMEOUT:
                             raise
 
-                    logger.warn("send_federation_events_to_master request timed out")
+                    logger.warn("%s request timed out", cls.NAME)
 
                     # If we timed out we probably don't need to worry about backing
                     # off too much, but lets just wait a little anyway.
@@ -190,7 +190,9 @@ def register(self, http_server):
         http_server.register_paths(method, [pattern], handler)
 
     def _cached_handler(self, request, txn_id, **kwargs):
-        """Wraps `_handle_request` the responses should be cached.
+        """Called on new incoming requests when caching is enabled. Checks
+        if their is a cached response for the request and returns that,
+        otherwise calls `_handle_request` and caches its response.
         """
         # We just use the txn_id here, but we probably also want to use the
         # other PATH_ARGS as well.
diff --git a/synapse/replication/http/membership.py b/synapse/replication/http/membership.py
index 8ad83e8421c6..e58bebf12ad9 100644
--- a/synapse/replication/http/membership.py
+++ b/synapse/replication/http/membership.py
@@ -27,6 +27,16 @@
 
 class ReplicationRemoteJoinRestServlet(ReplicationEndpoint):
     """Does a remote join for the given user to the given room
+
+    Request format:
+
+        POST /_synapse/replication/remote_join/:room_id/:user_id
+
+        {
+            "requester": ...,
+            "remote_room_hosts": [...],
+            "content": { ... }
+        }
     """
 
     NAME = "remote_join"
@@ -85,6 +95,15 @@ def _handle_request(self, request, room_id, user_id):
 
 class ReplicationRemoteRejectInviteRestServlet(ReplicationEndpoint):
     """Rejects the invite for the user and room.
+
+    Request format:
+
+        POST /_synapse/replication/remote_reject_invite/:room_id/:user_id
+
+        {
+            "requester": ...,
+            "remote_room_hosts": [...],
+        }
     """
 
     NAME = "remote_reject_invite"
@@ -153,6 +172,17 @@ def _handle_request(self, request, room_id, user_id):
 
 class ReplicationRegister3PIDGuestRestServlet(ReplicationEndpoint):
     """Gets/creates a guest account for given 3PID.
+
+    Request format:
+
+        POST /_synapse/replication/get_or_register_3pid_guest/
+
+        {
+            "requester": ...,
+            "medium": ...,
+            "address": ...,
+            "inviter_user_id": ...
+        }
     """
 
     NAME = "get_or_register_3pid_guest"
@@ -206,6 +236,12 @@ def _handle_request(self, request):
 
 class ReplicationUserJoinedLeftRoomRestServlet(ReplicationEndpoint):
     """Notifies that a user has joined or left the room
+
+    Request format:
+
+        POST /_synapse/replication/membership_change/:room_id/:user_id/:change
+
+        {}
     """
 
     NAME = "membership_change"

From bebe325e6cdd83f7bacd8ad71f6cdd23273c88db Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 8 Aug 2018 10:35:47 +0100
Subject: [PATCH 298/830] Rename POST param to METHOD

---
 synapse/replication/http/_base.py      | 34 +++++++++++++++++---------
 synapse/replication/http/send_event.py |  1 -
 2 files changed, 22 insertions(+), 13 deletions(-)

diff --git a/synapse/replication/http/_base.py b/synapse/replication/http/_base.py
index 619ddab54081..53a0fd459a82 100644
--- a/synapse/replication/http/_base.py
+++ b/synapse/replication/http/_base.py
@@ -40,8 +40,8 @@ class ReplicationEndpoint(object):
 
         /_synapse/replication/send_event/:event_id/:txn_id
 
-    For POST requests the payload is serialized to json and sent as the body,
-    while for GET requests the payload is added as query parameters. See
+    For POST/PUT requests the payload is serialized to json and sent as the
+    body, while for GET requests the payload is added as query parameters. See
     `_serialize_payload` for details.
 
     Incoming requests are handled by overriding `_handle_request`. Servers
@@ -55,8 +55,9 @@ class ReplicationEndpoint(object):
         PATH_ARGS (tuple[str]): A list of parameters to be added to the path.
             Adding parameters to the path (rather than payload) can make it
             easier to follow along in the log files.
-        POST (bool): True to use POST request with JSON body, or false to use
-            GET requests with query params.
+        METHOD (str): The method of the HTTP request, defaults to POST. Can be
+            one of POST, PUT or GET. If GET then the payload is sent as query
+            parameters rather than a JSON body.
         CACHE (bool): Whether server should cache the result of the request/
             If true then transparently adds a txn_id to all requests, and
             `_handle_request` must return a Deferred.
@@ -69,7 +70,7 @@ class ReplicationEndpoint(object):
     NAME = abc.abstractproperty()
     PATH_ARGS = abc.abstractproperty()
 
-    POST = True
+    METHOD = "POST"
     CACHE = True
     RETRY_ON_TIMEOUT = True
 
@@ -80,6 +81,8 @@ def __init__(self, hs):
                 timeout_ms=30 * 60 * 1000,
             )
 
+        assert self.METHOD in ("PUT", "POST", "GET")
+
     @abc.abstractmethod
     def _serialize_payload(**kwargs):
         """Static method that is called when creating a request.
@@ -90,9 +93,9 @@ def _serialize_payload(**kwargs):
         argument list.
 
         Returns:
-            Deferred[dict]|dict: If POST request then dictionary must be JSON
-            serialisable, otherwise must be appropriate for adding as query
-            args.
+            Deferred[dict]|dict: If POST/PUT request then dictionary must be
+            JSON serialisable, otherwise must be appropriate for adding as
+            query args.
         """
         return {}
 
@@ -130,10 +133,18 @@ def send_request(**kwargs):
                 txn_id = random_string(10)
                 url_args.append(txn_id)
 
-            if cls.POST:
+            if cls.METHOD == "POST":
                 request_func = client.post_json_get_json
-            else:
+            elif cls.METHOD == "PUT":
+                request_func = client.put_json
+            elif cls.METHOD == "GET":
                 request_func = client.get_json
+            else:
+                # We have already asserted in the constructor that a
+                # compatible was picked, but lets be paranoid.
+                raise Exception(
+                    "Unknown METHOD on %s replication endpoint" % (cls.NAME,)
+                )
 
             uri = "http://%s:%s/_synapse/replication/%s/%s" % (
                 host, port, cls.NAME, "/".join(url_args)
@@ -174,8 +185,7 @@ def register(self, http_server):
         url_args = list(self.PATH_ARGS)
         method = "GET"
         handler = self._handle_request
-        if self.POST:
-            method = "POST"
+        method = self.METHOD
 
         if self.CACHE:
             handler = self._cached_handler
diff --git a/synapse/replication/http/send_event.py b/synapse/replication/http/send_event.py
index 50810d94cb64..5b52c91650a8 100644
--- a/synapse/replication/http/send_event.py
+++ b/synapse/replication/http/send_event.py
@@ -47,7 +47,6 @@ class ReplicationSendEventRestServlet(ReplicationEndpoint):
     """
     NAME = "send_event"
     PATH_ARGS = ("event_id",)
-    POST = True
 
     def __init__(self, hs):
         super(ReplicationSendEventRestServlet, self).__init__(hs)

From 7f3d897e7af7c5227fc27355ead082055a8c0ecf Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 8 Aug 2018 11:46:23 +0100
Subject: [PATCH 299/830] mock config.max_mau_value

---
 tests/utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/utils.py b/tests/utils.py
index baeed5d60047..e7894819c025 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -74,6 +74,7 @@ def setup_test_homeserver(name="test", datastore=None, config=None, reactor=None
         config.media_storage_providers = []
         config.auto_join_rooms = []
         config.limit_usage_by_mau = False
+        config.max_mau_value = 50
         config.mau_limits_reserved_threepids = []
 
         # disable user directory updates, because they get done in the

From 360ba89c50ea5cbf824e54f04d536b89b57f3304 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 8 Aug 2018 11:54:55 +0100
Subject: [PATCH 300/830] Don't fail requests to unbind 3pids for non
 supporting ID servers

Older identity servers may not support the unbind 3pid request, so we
shouldn't fail the requests if we received one of 400/404/501. The
request still fails if we receive e.g. 500 responses, allowing clients
to retry requests on transient identity server errors that otherwise do
support the API.

Fixes #3661
---
 changelog.d/3661.bugfix                 |  1 +
 synapse/handlers/auth.py                | 20 ++++++++++++++---
 synapse/handlers/deactivate_account.py  | 13 +++++++++--
 synapse/handlers/identity.py            | 30 +++++++++++++++++--------
 synapse/rest/client/v1/admin.py         | 11 +++++++--
 synapse/rest/client/v2_alpha/account.py | 22 ++++++++++++++----
 6 files changed, 77 insertions(+), 20 deletions(-)
 create mode 100644 changelog.d/3661.bugfix

diff --git a/changelog.d/3661.bugfix b/changelog.d/3661.bugfix
new file mode 100644
index 000000000000..f2b4703d806b
--- /dev/null
+++ b/changelog.d/3661.bugfix
@@ -0,0 +1 @@
+Fix bug on deleting 3pid when using identity servers that don't support unbind API
diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py
index 184eef09d03b..da17e73fdd05 100644
--- a/synapse/handlers/auth.py
+++ b/synapse/handlers/auth.py
@@ -828,12 +828,26 @@ def add_threepid(self, user_id, medium, address, validated_at):
 
     @defer.inlineCallbacks
     def delete_threepid(self, user_id, medium, address):
+        """Attempts to unbind the 3pid on the identity servers and deletes it
+        from the local database.
+
+        Args:
+            user_id (str)
+            medium (str)
+            address (str)
+
+        Returns:
+            Deferred[bool]: Returns True if successfully unbound the 3pid on
+            the identity server, False if identity server doesn't support the
+            unbind API.
+        """
+
         # 'Canonicalise' email addresses as per above
         if medium == 'email':
             address = address.lower()
 
         identity_handler = self.hs.get_handlers().identity_handler
-        yield identity_handler.unbind_threepid(
+        result = yield identity_handler.try_unbind_threepid(
             user_id,
             {
                 'medium': medium,
@@ -841,10 +855,10 @@ def delete_threepid(self, user_id, medium, address):
             },
         )
 
-        ret = yield self.store.user_delete_threepid(
+        yield self.store.user_delete_threepid(
             user_id, medium, address,
         )
-        defer.returnValue(ret)
+        defer.returnValue(result)
 
     def _save_session(self, session):
         # TODO: Persistent storage
diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py
index b3c5a9ee642c..b078df4a762e 100644
--- a/synapse/handlers/deactivate_account.py
+++ b/synapse/handlers/deactivate_account.py
@@ -51,7 +51,8 @@ def deactivate_account(self, user_id, erase_data):
             erase_data (bool): whether to GDPR-erase the user's data
 
         Returns:
-            Deferred
+            Deferred[bool]: True if identity server supports removing
+            threepids, otherwise False.
         """
         # FIXME: Theoretically there is a race here wherein user resets
         # password using threepid.
@@ -60,16 +61,22 @@ def deactivate_account(self, user_id, erase_data):
         # leave the user still active so they can try again.
         # Ideally we would prevent password resets and then do this in the
         # background thread.
+
+        # This will be set to false if the identity server doesn't support
+        # unbinding
+        identity_server_supports_unbinding = True
+
         threepids = yield self.store.user_get_threepids(user_id)
         for threepid in threepids:
             try:
-                yield self._identity_handler.unbind_threepid(
+                result = yield self._identity_handler.try_unbind_threepid(
                     user_id,
                     {
                         'medium': threepid['medium'],
                         'address': threepid['address'],
                     },
                 )
+                identity_server_supports_unbinding &= result
             except Exception:
                 # Do we want this to be a fatal error or should we carry on?
                 logger.exception("Failed to remove threepid from ID server")
@@ -103,6 +110,8 @@ def deactivate_account(self, user_id, erase_data):
         # parts users from rooms (if it isn't already running)
         self._start_user_parting()
 
+        defer.returnValue(identity_server_supports_unbinding)
+
     def _start_user_parting(self):
         """
         Start the process that goes through the table of users
diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py
index 1d36d967c3b5..a0f5fecc9602 100644
--- a/synapse/handlers/identity.py
+++ b/synapse/handlers/identity.py
@@ -137,15 +137,19 @@ def bind_threepid(self, creds, mxid):
         defer.returnValue(data)
 
     @defer.inlineCallbacks
-    def unbind_threepid(self, mxid, threepid):
-        """
-        Removes a binding from an identity server
+    def try_unbind_threepid(self, mxid, threepid):
+        """Removes a binding from an identity server
+
         Args:
             mxid (str): Matrix user ID of binding to be removed
             threepid (dict): Dict with medium & address of binding to be removed
 
+        Raises:
+            SynapseError: If we failed to contact the identity server
+
         Returns:
-            Deferred[bool]: True on success, otherwise False
+            Deferred[bool]: True on success, otherwise False if the identity
+            server doesn't support unbinding
         """
         logger.debug("unbinding threepid %r from %s", threepid, mxid)
         if not self.trusted_id_servers:
@@ -175,11 +179,19 @@ def unbind_threepid(self, mxid, threepid):
             content=content,
             destination_is=id_server,
         )
-        yield self.http_client.post_json_get_json(
-            url,
-            content,
-            headers,
-        )
+        try:
+            yield self.http_client.post_json_get_json(
+                url,
+                content,
+                headers,
+            )
+        except HttpResponseException as e:
+            if e.code in (400, 404, 501,):
+                # The remote server probably doesn't support unbinding (yet)
+                defer.returnValue(False)
+            else:
+                raise SynapseError(502, "Failed to contact identity server")
+
         defer.returnValue(True)
 
     @defer.inlineCallbacks
diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py
index 80d625eecce6..ad536ab570f5 100644
--- a/synapse/rest/client/v1/admin.py
+++ b/synapse/rest/client/v1/admin.py
@@ -391,10 +391,17 @@ def on_POST(self, request, target_user_id):
         if not is_admin:
             raise AuthError(403, "You are not a server admin")
 
-        yield self._deactivate_account_handler.deactivate_account(
+        result = yield self._deactivate_account_handler.deactivate_account(
             target_user_id, erase,
         )
-        defer.returnValue((200, {}))
+        if result:
+            id_server_unbind_result = "success"
+        else:
+            id_server_unbind_result = "no-support"
+
+        defer.returnValue((200, {
+            "id_server_unbind_result": id_server_unbind_result,
+        }))
 
 
 class ShutdownRoomRestServlet(ClientV1RestServlet):
diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py
index eeae466d8291..372648cafdab 100644
--- a/synapse/rest/client/v2_alpha/account.py
+++ b/synapse/rest/client/v2_alpha/account.py
@@ -209,10 +209,17 @@ def on_POST(self, request):
         yield self.auth_handler.validate_user_via_ui_auth(
             requester, body, self.hs.get_ip_from_request(request),
         )
-        yield self._deactivate_account_handler.deactivate_account(
+        result = yield self._deactivate_account_handler.deactivate_account(
             requester.user.to_string(), erase,
         )
-        defer.returnValue((200, {}))
+        if result:
+            id_server_unbind_result = "success"
+        else:
+            id_server_unbind_result = "no-support"
+
+        defer.returnValue((200, {
+            "id_server_unbind_result": id_server_unbind_result,
+        }))
 
 
 class EmailThreepidRequestTokenRestServlet(RestServlet):
@@ -364,7 +371,7 @@ def on_POST(self, request):
         user_id = requester.user.to_string()
 
         try:
-            yield self.auth_handler.delete_threepid(
+            ret = yield self.auth_handler.delete_threepid(
                 user_id, body['medium'], body['address']
             )
         except Exception:
@@ -374,7 +381,14 @@ def on_POST(self, request):
             logger.exception("Failed to remove threepid")
             raise SynapseError(500, "Failed to remove threepid")
 
-        defer.returnValue((200, {}))
+        if ret:
+            id_server_unbind_result = "success"
+        else:
+            id_server_unbind_result = "no-support"
+
+        defer.returnValue((200, {
+            "id_server_unbind_result": id_server_unbind_result,
+        }))
 
 
 class WhoamiRestServlet(RestServlet):

From d92675a4861eab5f5eec0e8a39a7505d5bda850d Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 8 Aug 2018 12:06:42 +0100
Subject: [PATCH 301/830] Ability to whitelist specific threepids against
 monthly active user limiting

---
 changelog.d/3662.feature | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3662.feature

diff --git a/changelog.d/3662.feature b/changelog.d/3662.feature
new file mode 100644
index 000000000000..daacef086d03
--- /dev/null
+++ b/changelog.d/3662.feature
@@ -0,0 +1 @@
+Ability to whitelist specific threepids against monthly active user limiting

From 312ae747466f7c60c0148b75446f357263330148 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 8 Aug 2018 13:33:16 +0100
Subject: [PATCH 302/830] typos

---
 synapse/storage/monthly_active_users.py                  | 4 ++--
 synapse/storage/schema/delta/51/monthly_active_users.sql | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py
index abe1e6bb99ad..8b3beaf26a1f 100644
--- a/synapse/storage/monthly_active_users.py
+++ b/synapse/storage/monthly_active_users.py
@@ -50,7 +50,7 @@ def _reap_users(txn):
             # If MAU user count still exceeds the MAU threshold, then delete on
             # a least recently active basis.
             # Note it is not possible to write this query using OFFSET due to
-            # incompatibilities in how sqlite an postgres support the feature.
+            # incompatibilities in how sqlite and postgres support the feature.
             # sqlite requires 'LIMIT -1 OFFSET ?', the LIMIT must be present
             # While Postgres does not require 'LIMIT', but also does not support
             # negative LIMIT values. So there is no way to write it that both can
@@ -78,7 +78,7 @@ def _reap_users(txn):
 
     @cached(num_args=0)
     def get_monthly_active_count(self):
-        """Generates current count of monthly active users.abs
+        """Generates current count of monthly active users
 
         Returns:
             Defered[int]: Number of current monthly active users
diff --git a/synapse/storage/schema/delta/51/monthly_active_users.sql b/synapse/storage/schema/delta/51/monthly_active_users.sql
index 10aac90ce137..c9d537d5a306 100644
--- a/synapse/storage/schema/delta/51/monthly_active_users.sql
+++ b/synapse/storage/schema/delta/51/monthly_active_users.sql
@@ -18,7 +18,7 @@ CREATE TABLE monthly_active_users (
     user_id TEXT NOT NULL,
     -- Last time we saw the user. Not guaranteed to be accurate due to rate limiting
     -- on updates, Granularity of updates governed by
-    -- syanpse.storage.monthly_active_users.LAST_SEEN_GRANULARITY
+    -- synapse.storage.monthly_active_users.LAST_SEEN_GRANULARITY
     -- Measured in ms since epoch.
     timestamp BIGINT NOT NULL
 );

From 54685d294dd15bb8b9a9928b8d6f371ae4236e25 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 8 Aug 2018 15:38:54 +0100
Subject: [PATCH 303/830] Ability to disable client/server Synapse via conf
 toggle

---
 changelog.d/3655.feature | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3655.feature

diff --git a/changelog.d/3655.feature b/changelog.d/3655.feature
new file mode 100644
index 000000000000..1134e549e7b3
--- /dev/null
+++ b/changelog.d/3655.feature
@@ -0,0 +1 @@
+Ability to disable client/server Synapse via conf toggle

From 839a317c9627ff0d61f91504a2cfca275f31d7b2 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 8 Aug 2018 17:39:04 +0100
Subject: [PATCH 304/830] fix pep8 too many lines

---
 synapse/api/errors.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index 69e6ffb5a320..dc3bed5fcbbd 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -62,7 +62,6 @@ class Codes(object):
     INCOMPATIBLE_ROOM_VERSION = "M_INCOMPATIBLE_ROOM_VERSION"
 
 
-
 class CodeMessageException(RuntimeError):
     """An exception with integer code and message string attributes.
 

From e92fb00f32c63de6ea50ba1cbbadf74060ea143d Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 8 Aug 2018 17:54:49 +0100
Subject: [PATCH 305/830] sync auth blocking

---
 synapse/handlers/sync.py    | 16 +++++++++-----
 tests/handlers/test_sync.py | 42 +++++++++++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+), 5 deletions(-)
 create mode 100644 tests/handlers/test_sync.py

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index dff1f67dcb45..f748d9afb099 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -191,6 +191,7 @@ def __init__(self, hs):
         self.clock = hs.get_clock()
         self.response_cache = ResponseCache(hs, "sync")
         self.state = hs.get_state_handler()
+        self.auth = hs.get_auth()
 
         # ExpiringCache((User, Device)) -> LruCache(state_key => event_id)
         self.lazy_loaded_members_cache = ExpiringCache(
@@ -198,18 +199,23 @@ def __init__(self, hs):
             max_len=0, expiry_ms=LAZY_LOADED_MEMBERS_CACHE_MAX_AGE,
         )
 
+    @defer.inlineCallbacks
     def wait_for_sync_for_user(self, sync_config, since_token=None, timeout=0,
                                full_state=False):
         """Get the sync for a client if we have new data for it now. Otherwise
         wait for new data to arrive on the server. If the timeout expires, then
         return an empty sync result.
         Returns:
-            A Deferred SyncResult.
+            Deferred[SyncResult]
         """
-        return self.response_cache.wrap(
-            sync_config.request_key,
-            self._wait_for_sync_for_user,
-            sync_config, since_token, timeout, full_state,
+        yield self.auth.check_auth_blocking()
+
+        defer.returnValue(
+            self.response_cache.wrap(
+                sync_config.request_key,
+                self._wait_for_sync_for_user,
+                sync_config, since_token, timeout, full_state,
+            )
         )
 
     @defer.inlineCallbacks
diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py
new file mode 100644
index 000000000000..3b1b4d492367
--- /dev/null
+++ b/tests/handlers/test_sync.py
@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from twisted.internet import defer
+
+import tests.unittest
+import tests.utils
+from tests.utils import setup_test_homeserver
+from synapse.handlers.sync import SyncHandler, SyncConfig
+from synapse.types import UserID
+
+
+class SyncTestCase(tests.unittest.TestCase):
+    """ Tests Sync Handler. """
+
+    @defer.inlineCallbacks
+    def setUp(self):
+        self.hs = yield setup_test_homeserver()
+        self.sync_handler = SyncHandler(self.hs)
+
+    @defer.inlineCallbacks
+    def test_wait_for_sync_for_user_auth_blocking(self):
+        sync_config = SyncConfig(
+            user=UserID("@user","server"),
+            filter_collection=None,
+            is_guest=False,
+            request_key="request_key",
+            device_id="device_id",
+        )
+        res = yield self.sync_handler.wait_for_sync_for_user(sync_config)
+        print res

From d5c0ce4cadb4848da174bf5a5563de0ebba4790f Mon Sep 17 00:00:00 2001
From: Jeroen <vo.jeroen@gmail.com>
Date: Wed, 8 Aug 2018 19:25:01 +0200
Subject: [PATCH 306/830] updated docstring for ServerContextFactory

---
 synapse/crypto/context_factory.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py
index 569377e65363..29a75e187374 100644
--- a/synapse/crypto/context_factory.py
+++ b/synapse/crypto/context_factory.py
@@ -26,7 +26,7 @@
 
 class ServerContextFactory(ContextFactory):
     """Factory for PyOpenSSL SSL contexts that are used to handle incoming
-    connections and to make connections to remote servers."""
+    connections."""
 
     def __init__(self, config):
         self._context = SSL.Context(SSL.SSLv23_METHOD)

From 2511f3f8a082585e680dad200069dec77b066a6a Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Thu, 9 Aug 2018 12:22:01 +1000
Subject: [PATCH 307/830] Test fixes for Python 3 (#3647)

---
 changelog.d/3647.misc                       |  1 +
 tests/handlers/test_typing.py               |  4 +++-
 tests/rest/client/test_transactions.py      |  4 ++--
 tests/rest/client/v1/test_admin.py          | 10 +++++-----
 tests/rest/client/v1/test_profile.py        |  8 ++++----
 tests/rest/client/v1/utils.py               | 10 +++++-----
 tests/rest/client/v2_alpha/test_filter.py   | 22 ++++++++++-----------
 tests/rest/client/v2_alpha/test_register.py | 14 ++++++-------
 tests/rest/client/v2_alpha/test_sync.py     |  4 ++--
 tests/server.py                             | 22 +++++++++++++++++++--
 tests/storage/test_event_federation.py      |  2 +-
 tests/storage/test_state.py                 |  2 +-
 tests/test_server.py                        | 11 ++++-------
 tests/utils.py                              |  9 +++++----
 14 files changed, 71 insertions(+), 52 deletions(-)
 create mode 100644 changelog.d/3647.misc

diff --git a/changelog.d/3647.misc b/changelog.d/3647.misc
new file mode 100644
index 000000000000..dbc66dae6087
--- /dev/null
+++ b/changelog.d/3647.misc
@@ -0,0 +1 @@
+Tests now correctly execute on Python 3.
diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py
index 2c263af1a3a0..f422cf3c5a5f 100644
--- a/tests/handlers/test_typing.py
+++ b/tests/handlers/test_typing.py
@@ -48,7 +48,9 @@ def _expect_edu(destination, edu_type, content, origin="test"):
 
 
 def _make_edu_json(origin, edu_type, content):
-    return json.dumps(_expect_edu("test", edu_type, content, origin=origin))
+    return json.dumps(
+        _expect_edu("test", edu_type, content, origin=origin)
+    ).encode('utf8')
 
 
 class TypingNotificationsTestCase(unittest.TestCase):
diff --git a/tests/rest/client/test_transactions.py b/tests/rest/client/test_transactions.py
index 34e68ae82fa4..d46c27e7e9f7 100644
--- a/tests/rest/client/test_transactions.py
+++ b/tests/rest/client/test_transactions.py
@@ -85,7 +85,7 @@ def cb():
             try:
                 yield self.cache.fetch_or_execute(self.mock_key, cb)
             except Exception as e:
-                self.assertEqual(e.message, "boo")
+                self.assertEqual(e.args[0], "boo")
             self.assertIs(LoggingContext.current_context(), test_context)
 
             res = yield self.cache.fetch_or_execute(self.mock_key, cb)
@@ -111,7 +111,7 @@ def cb():
             try:
                 yield self.cache.fetch_or_execute(self.mock_key, cb)
             except Exception as e:
-                self.assertEqual(e.message, "boo")
+                self.assertEqual(e.args[0], "boo")
             self.assertIs(LoggingContext.current_context(), test_context)
 
             res = yield self.cache.fetch_or_execute(self.mock_key, cb)
diff --git a/tests/rest/client/v1/test_admin.py b/tests/rest/client/v1/test_admin.py
index 8c90145601a6..fb28883d30c4 100644
--- a/tests/rest/client/v1/test_admin.py
+++ b/tests/rest/client/v1/test_admin.py
@@ -140,7 +140,7 @@ def test_register_incorrect_nonce(self):
                 "admin": True,
                 "mac": want_mac,
             }
-        ).encode('utf8')
+        )
         request, channel = make_request("POST", self.url, body.encode('utf8'))
         render(request, self.resource, self.clock)
 
@@ -168,7 +168,7 @@ def test_register_correct_nonce(self):
                 "admin": True,
                 "mac": want_mac,
             }
-        ).encode('utf8')
+        )
         request, channel = make_request("POST", self.url, body.encode('utf8'))
         render(request, self.resource, self.clock)
 
@@ -195,7 +195,7 @@ def test_nonce_reuse(self):
                 "admin": True,
                 "mac": want_mac,
             }
-        ).encode('utf8')
+        )
         request, channel = make_request("POST", self.url, body.encode('utf8'))
         render(request, self.resource, self.clock)
 
@@ -253,7 +253,7 @@ def nonce():
         self.assertEqual('Invalid username', channel.json_body["error"])
 
         # Must not have null bytes
-        body = json.dumps({"nonce": nonce(), "username": b"abcd\x00"})
+        body = json.dumps({"nonce": nonce(), "username": u"abcd\u0000"})
         request, channel = make_request("POST", self.url, body.encode('utf8'))
         render(request, self.resource, self.clock)
 
@@ -289,7 +289,7 @@ def nonce():
         self.assertEqual('Invalid password', channel.json_body["error"])
 
         # Must not have null bytes
-        body = json.dumps({"nonce": nonce(), "username": "a", "password": b"abcd\x00"})
+        body = json.dumps({"nonce": nonce(), "username": "a", "password": u"abcd\u0000"})
         request, channel = make_request("POST", self.url, body.encode('utf8'))
         render(request, self.resource, self.clock)
 
diff --git a/tests/rest/client/v1/test_profile.py b/tests/rest/client/v1/test_profile.py
index d71cc8e0db73..0516ce3cfbd2 100644
--- a/tests/rest/client/v1/test_profile.py
+++ b/tests/rest/client/v1/test_profile.py
@@ -80,7 +80,7 @@ def test_set_my_name(self):
         (code, response) = yield self.mock_resource.trigger(
             "PUT",
             "/profile/%s/displayname" % (myid),
-            '{"displayname": "Frank Jr."}'
+            b'{"displayname": "Frank Jr."}'
         )
 
         self.assertEquals(200, code)
@@ -95,7 +95,7 @@ def test_set_my_name_noauth(self):
 
         (code, response) = yield self.mock_resource.trigger(
             "PUT", "/profile/%s/displayname" % ("@4567:test"),
-            '{"displayname": "Frank Jr."}'
+            b'{"displayname": "Frank Jr."}'
         )
 
         self.assertTrue(
@@ -122,7 +122,7 @@ def test_set_other_name(self):
 
         (code, response) = yield self.mock_resource.trigger(
             "PUT", "/profile/%s/displayname" % ("@opaque:elsewhere"),
-            '{"displayname":"bob"}'
+            b'{"displayname":"bob"}'
         )
 
         self.assertTrue(
@@ -151,7 +151,7 @@ def test_set_my_avatar(self):
         (code, response) = yield self.mock_resource.trigger(
             "PUT",
             "/profile/%s/avatar_url" % (myid),
-            '{"avatar_url": "http://my.server/pic.gif"}'
+            b'{"avatar_url": "http://my.server/pic.gif"}'
         )
 
         self.assertEquals(200, code)
diff --git a/tests/rest/client/v1/utils.py b/tests/rest/client/v1/utils.py
index 41de8e0762ec..e3bc5f378d22 100644
--- a/tests/rest/client/v1/utils.py
+++ b/tests/rest/client/v1/utils.py
@@ -105,7 +105,7 @@ def register(self, user_id):
                 "password": "test",
                 "type": "m.login.password"
             }))
-        self.assertEquals(200, code)
+        self.assertEquals(200, code, msg=response)
         defer.returnValue(response)
 
     @defer.inlineCallbacks
@@ -149,14 +149,14 @@ class RestHelper(object):
     def create_room_as(self, room_creator, is_public=True, tok=None):
         temp_id = self.auth_user_id
         self.auth_user_id = room_creator
-        path = b"/_matrix/client/r0/createRoom"
+        path = "/_matrix/client/r0/createRoom"
         content = {}
         if not is_public:
             content["visibility"] = "private"
         if tok:
-            path = path + b"?access_token=%s" % tok.encode('ascii')
+            path = path + "?access_token=%s" % tok
 
-        request, channel = make_request(b"POST", path, json.dumps(content).encode('utf8'))
+        request, channel = make_request("POST", path, json.dumps(content).encode('utf8'))
         request.render(self.resource)
         wait_until_result(self.hs.get_reactor(), channel)
 
@@ -205,7 +205,7 @@ def change_membership(self, room, src, targ, membership, tok=None, expect_code=2
         data = {"membership": membership}
 
         request, channel = make_request(
-            b"PUT", path.encode('ascii'), json.dumps(data).encode('utf8')
+            "PUT", path, json.dumps(data).encode('utf8')
         )
 
         request.render(self.resource)
diff --git a/tests/rest/client/v2_alpha/test_filter.py b/tests/rest/client/v2_alpha/test_filter.py
index e890f0feac46..de33b10a5f5e 100644
--- a/tests/rest/client/v2_alpha/test_filter.py
+++ b/tests/rest/client/v2_alpha/test_filter.py
@@ -33,7 +33,7 @@
 
 class FilterTestCase(unittest.TestCase):
 
-    USER_ID = b"@apple:test"
+    USER_ID = "@apple:test"
     EXAMPLE_FILTER = {"room": {"timeline": {"types": ["m.room.message"]}}}
     EXAMPLE_FILTER_JSON = b'{"room": {"timeline": {"types": ["m.room.message"]}}}'
     TO_REGISTER = [filter]
@@ -72,8 +72,8 @@ def get_user_by_req(request, allow_guest=False, rights="access"):
 
     def test_add_filter(self):
         request, channel = make_request(
-            b"POST",
-            b"/_matrix/client/r0/user/%s/filter" % (self.USER_ID),
+            "POST",
+            "/_matrix/client/r0/user/%s/filter" % (self.USER_ID),
             self.EXAMPLE_FILTER_JSON,
         )
         request.render(self.resource)
@@ -87,8 +87,8 @@ def test_add_filter(self):
 
     def test_add_filter_for_other_user(self):
         request, channel = make_request(
-            b"POST",
-            b"/_matrix/client/r0/user/%s/filter" % (b"@watermelon:test"),
+            "POST",
+            "/_matrix/client/r0/user/%s/filter" % ("@watermelon:test"),
             self.EXAMPLE_FILTER_JSON,
         )
         request.render(self.resource)
@@ -101,8 +101,8 @@ def test_add_filter_non_local_user(self):
         _is_mine = self.hs.is_mine
         self.hs.is_mine = lambda target_user: False
         request, channel = make_request(
-            b"POST",
-            b"/_matrix/client/r0/user/%s/filter" % (self.USER_ID),
+            "POST",
+            "/_matrix/client/r0/user/%s/filter" % (self.USER_ID),
             self.EXAMPLE_FILTER_JSON,
         )
         request.render(self.resource)
@@ -119,7 +119,7 @@ def test_get_filter(self):
         self.clock.advance(1)
         filter_id = filter_id.result
         request, channel = make_request(
-            b"GET", b"/_matrix/client/r0/user/%s/filter/%s" % (self.USER_ID, filter_id)
+            "GET", "/_matrix/client/r0/user/%s/filter/%s" % (self.USER_ID, filter_id)
         )
         request.render(self.resource)
         wait_until_result(self.clock, channel)
@@ -129,7 +129,7 @@ def test_get_filter(self):
 
     def test_get_filter_non_existant(self):
         request, channel = make_request(
-            b"GET", "/_matrix/client/r0/user/%s/filter/12382148321" % (self.USER_ID)
+            "GET", "/_matrix/client/r0/user/%s/filter/12382148321" % (self.USER_ID)
         )
         request.render(self.resource)
         wait_until_result(self.clock, channel)
@@ -141,7 +141,7 @@ def test_get_filter_non_existant(self):
     # in errors.py
     def test_get_filter_invalid_id(self):
         request, channel = make_request(
-            b"GET", "/_matrix/client/r0/user/%s/filter/foobar" % (self.USER_ID)
+            "GET", "/_matrix/client/r0/user/%s/filter/foobar" % (self.USER_ID)
         )
         request.render(self.resource)
         wait_until_result(self.clock, channel)
@@ -151,7 +151,7 @@ def test_get_filter_invalid_id(self):
     # No ID also returns an invalid_id error
     def test_get_filter_no_id(self):
         request, channel = make_request(
-            b"GET", "/_matrix/client/r0/user/%s/filter/" % (self.USER_ID)
+            "GET", "/_matrix/client/r0/user/%s/filter/" % (self.USER_ID)
         )
         request.render(self.resource)
         wait_until_result(self.clock, channel)
diff --git a/tests/rest/client/v2_alpha/test_register.py b/tests/rest/client/v2_alpha/test_register.py
index e004d8fc73d9..f6293f11a871 100644
--- a/tests/rest/client/v2_alpha/test_register.py
+++ b/tests/rest/client/v2_alpha/test_register.py
@@ -81,7 +81,7 @@ def test_POST_appservice_registration_valid(self):
             "access_token": token,
             "home_server": self.hs.hostname,
         }
-        self.assertDictContainsSubset(det_data, json.loads(channel.result["body"]))
+        self.assertDictContainsSubset(det_data, channel.json_body)
 
     def test_POST_appservice_registration_invalid(self):
         self.appservice = None  # no application service exists
@@ -102,7 +102,7 @@ def test_POST_bad_password(self):
 
         self.assertEquals(channel.result["code"], b"400", channel.result)
         self.assertEquals(
-            json.loads(channel.result["body"])["error"], "Invalid password"
+            channel.json_body["error"], "Invalid password"
         )
 
     def test_POST_bad_username(self):
@@ -113,7 +113,7 @@ def test_POST_bad_username(self):
 
         self.assertEquals(channel.result["code"], b"400", channel.result)
         self.assertEquals(
-            json.loads(channel.result["body"])["error"], "Invalid username"
+            channel.json_body["error"], "Invalid username"
         )
 
     def test_POST_user_valid(self):
@@ -140,7 +140,7 @@ def test_POST_user_valid(self):
             "device_id": device_id,
         }
         self.assertEquals(channel.result["code"], b"200", channel.result)
-        self.assertDictContainsSubset(det_data, json.loads(channel.result["body"]))
+        self.assertDictContainsSubset(det_data, channel.json_body)
         self.auth_handler.get_login_tuple_for_user_id(
             user_id, device_id=device_id, initial_device_display_name=None
         )
@@ -158,7 +158,7 @@ def test_POST_disabled_registration(self):
 
         self.assertEquals(channel.result["code"], b"403", channel.result)
         self.assertEquals(
-            json.loads(channel.result["body"])["error"],
+            channel.json_body["error"],
             "Registration has been disabled",
         )
 
@@ -178,7 +178,7 @@ def test_POST_guest_registration(self):
             "device_id": "guest_device",
         }
         self.assertEquals(channel.result["code"], b"200", channel.result)
-        self.assertDictContainsSubset(det_data, json.loads(channel.result["body"]))
+        self.assertDictContainsSubset(det_data, channel.json_body)
 
     def test_POST_disabled_guest_registration(self):
         self.hs.config.allow_guest_access = False
@@ -189,5 +189,5 @@ def test_POST_disabled_guest_registration(self):
 
         self.assertEquals(channel.result["code"], b"403", channel.result)
         self.assertEquals(
-            json.loads(channel.result["body"])["error"], "Guest access is disabled"
+            channel.json_body["error"], "Guest access is disabled"
         )
diff --git a/tests/rest/client/v2_alpha/test_sync.py b/tests/rest/client/v2_alpha/test_sync.py
index 03ec3993b2df..bafc0d1df07d 100644
--- a/tests/rest/client/v2_alpha/test_sync.py
+++ b/tests/rest/client/v2_alpha/test_sync.py
@@ -32,7 +32,7 @@
 
 class FilterTestCase(unittest.TestCase):
 
-    USER_ID = b"@apple:test"
+    USER_ID = "@apple:test"
     TO_REGISTER = [sync]
 
     def setUp(self):
@@ -68,7 +68,7 @@ def get_user_by_req(request, allow_guest=False, rights="access"):
             r.register_servlets(self.hs, self.resource)
 
     def test_sync_argless(self):
-        request, channel = make_request(b"GET", b"/_matrix/client/r0/sync")
+        request, channel = make_request("GET", "/_matrix/client/r0/sync")
         request.render(self.resource)
         wait_until_result(self.clock, channel)
 
diff --git a/tests/server.py b/tests/server.py
index c611dd6059dd..e249668d2104 100644
--- a/tests/server.py
+++ b/tests/server.py
@@ -11,6 +11,7 @@
 from twisted.test.proto_helpers import MemoryReactorClock
 
 from synapse.http.site import SynapseRequest
+from synapse.util import Clock
 
 from tests.utils import setup_test_homeserver as _sth
 
@@ -28,7 +29,13 @@ class FakeChannel(object):
     def json_body(self):
         if not self.result:
             raise Exception("No result yet.")
-        return json.loads(self.result["body"])
+        return json.loads(self.result["body"].decode('utf8'))
+
+    @property
+    def code(self):
+        if not self.result:
+            raise Exception("No result yet.")
+        return int(self.result["code"])
 
     def writeHeaders(self, version, code, reason, headers):
         self.result["version"] = version
@@ -79,11 +86,16 @@ def make_request(method, path, content=b""):
     Make a web request using the given method and path, feed it the
     content, and return the Request and the Channel underneath.
     """
+    if not isinstance(method, bytes):
+        method = method.encode('ascii')
+
+    if not isinstance(path, bytes):
+        path = path.encode('ascii')
 
     # Decorate it to be the full path
     if not path.startswith(b"/_matrix"):
         path = b"/_matrix/client/r0/" + path
-        path = path.replace("//", "/")
+        path = path.replace(b"//", b"/")
 
     if isinstance(content, text_type):
         content = content.encode('utf8')
@@ -191,3 +203,9 @@ def _(res):
     clock.threadpool = ThreadPool()
     pool.threadpool = ThreadPool()
     return d
+
+
+def get_clock():
+    clock = ThreadedMemoryReactorClock()
+    hs_clock = Clock(clock)
+    return (clock, hs_clock)
diff --git a/tests/storage/test_event_federation.py b/tests/storage/test_event_federation.py
index 30683e7888ed..69412c5aadd8 100644
--- a/tests/storage/test_event_federation.py
+++ b/tests/storage/test_event_federation.py
@@ -49,7 +49,7 @@ def insert_event(txn, i):
                 'INSERT INTO event_reference_hashes '
                 '(event_id, algorithm, hash) '
                 "VALUES (?, 'sha256', ?)"
-            ), (event_id, 'ffff'))
+            ), (event_id, b'ffff'))
 
         for i in range(0, 11):
             yield self.store.runInteraction("insert", insert_event, i)
diff --git a/tests/storage/test_state.py b/tests/storage/test_state.py
index 7a76d67b8c8b..f7871cd42611 100644
--- a/tests/storage/test_state.py
+++ b/tests/storage/test_state.py
@@ -176,7 +176,7 @@ def test_get_state_for_event(self):
 
         room_id = self.room.to_string()
         group_ids = yield self.store.get_state_groups_ids(room_id, [e5.event_id])
-        group = group_ids.keys()[0]
+        group = list(group_ids.keys())[0]
 
         # test _get_some_state_from_cache correctly filters out members with types=[]
         (state_dict, is_all) = yield self.store._get_some_state_from_cache(
diff --git a/tests/test_server.py b/tests/test_server.py
index 7e063c029096..fc396226ea27 100644
--- a/tests/test_server.py
+++ b/tests/test_server.py
@@ -1,4 +1,3 @@
-import json
 import re
 
 from twisted.internet.defer import Deferred
@@ -104,9 +103,8 @@ def _callback(request, **kwargs):
         request.render(res)
 
         self.assertEqual(channel.result["code"], b'403')
-        reply_body = json.loads(channel.result["body"])
-        self.assertEqual(reply_body["error"], "Forbidden!!one!")
-        self.assertEqual(reply_body["errcode"], "M_FORBIDDEN")
+        self.assertEqual(channel.json_body["error"], "Forbidden!!one!")
+        self.assertEqual(channel.json_body["errcode"], "M_FORBIDDEN")
 
     def test_no_handler(self):
         """
@@ -126,6 +124,5 @@ def _callback(request, **kwargs):
         request.render(res)
 
         self.assertEqual(channel.result["code"], b'400')
-        reply_body = json.loads(channel.result["body"])
-        self.assertEqual(reply_body["error"], "Unrecognized request")
-        self.assertEqual(reply_body["errcode"], "M_UNRECOGNIZED")
+        self.assertEqual(channel.json_body["error"], "Unrecognized request")
+        self.assertEqual(channel.json_body["errcode"], "M_UNRECOGNIZED")
diff --git a/tests/utils.py b/tests/utils.py
index 151db4b89078..5d49692c589d 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -153,8 +153,9 @@ def setup_test_homeserver(name="test", datastore=None, config=None, reactor=None
     # Need to let the HS build an auth handler and then mess with it
     # because AuthHandler's constructor requires the HS, so we can't make one
     # beforehand and pass it in to the HS's constructor (chicken / egg)
-    hs.get_auth_handler().hash = lambda p: hashlib.md5(p).hexdigest()
-    hs.get_auth_handler().validate_hash = lambda p, h: hashlib.md5(p).hexdigest() == h
+    hs.get_auth_handler().hash = lambda p: hashlib.md5(p.encode('utf8')).hexdigest()
+    hs.get_auth_handler().validate_hash = lambda p, h: hashlib.md5(
+        p.encode('utf8')).hexdigest() == h
 
     fed = kargs.get("resource_for_federation", None)
     if fed:
@@ -227,8 +228,8 @@ def trigger(self, http_method, path, content, mock_request, federation_auth=Fals
         mock_content.configure_mock(**config)
         mock_request.content = mock_content
 
-        mock_request.method = http_method
-        mock_request.uri = path
+        mock_request.method = http_method.encode('ascii')
+        mock_request.uri = path.encode('ascii')
 
         mock_request.getClientIP.return_value = "-"
 

From 62564797f5f9c6b1295a98a9742ae226b87a135e Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 9 Aug 2018 09:56:10 +0100
Subject: [PATCH 308/830] Fixup wording and remove dead code

---
 changelog.d/3632.misc             | 1 +
 synapse/replication/http/_base.py | 3 +--
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/changelog.d/3632.misc b/changelog.d/3632.misc
index e69de29bb2d1..9d64bbe83b65 100644
--- a/changelog.d/3632.misc
+++ b/changelog.d/3632.misc
@@ -0,0 +1 @@
+Refactor HTTP replication endpoints to reduce code duplication
diff --git a/synapse/replication/http/_base.py b/synapse/replication/http/_base.py
index 53a0fd459a82..5e5376cf58fa 100644
--- a/synapse/replication/http/_base.py
+++ b/synapse/replication/http/_base.py
@@ -183,7 +183,6 @@ def register(self, http_server):
         """
 
         url_args = list(self.PATH_ARGS)
-        method = "GET"
         handler = self._handle_request
         method = self.METHOD
 
@@ -201,7 +200,7 @@ def register(self, http_server):
 
     def _cached_handler(self, request, txn_id, **kwargs):
         """Called on new incoming requests when caching is enabled. Checks
-        if their is a cached response for the request and returns that,
+        if there is a cached response for the request and returns that,
         otherwise calls `_handle_request` and caches its response.
         """
         # We just use the txn_id here, but we probably also want to use the

From bf7598f582cdfbd7db0fed55afce28bcc51a4801 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 9 Aug 2018 10:09:56 +0100
Subject: [PATCH 309/830] Log when we 3pid/unbind request fails

---
 synapse/handlers/identity.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py
index a0f5fecc9602..5feb3f22a691 100644
--- a/synapse/handlers/identity.py
+++ b/synapse/handlers/identity.py
@@ -188,8 +188,10 @@ def try_unbind_threepid(self, mxid, threepid):
         except HttpResponseException as e:
             if e.code in (400, 404, 501,):
                 # The remote server probably doesn't support unbinding (yet)
+                logger.warn("Received %d response while unbinding threepid", e.code)
                 defer.returnValue(False)
             else:
+                logger.error("Failed to unbind threepid on identity server: %s", e)
                 raise SynapseError(502, "Failed to contact identity server")
 
         defer.returnValue(True)

From 3d6f9fba19f0aa32a700a4ccc949ad329135eb71 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Thu, 9 Aug 2018 19:19:30 +1000
Subject: [PATCH 310/830] fix the changelog & template

---
 CHANGES.md     | 98 +++++++++++++++++++++++++-------------------------
 pyproject.toml |  2 +-
 2 files changed, 50 insertions(+), 50 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 299780e9ec84..8317e44f37c3 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -4,65 +4,65 @@ Synapse 0.33.2rc1 (2018-08-07)
 Features
 --------
 
-- add support for the lazy_loaded_members filter as per MSC1227 ([\#2970](https://github.com/matrix-org/synapse/issues/2970>))
-- add support for the include_redundant_members filter param as per MSC1227 ([\#3331](https://github.com/matrix-org/synapse/issues/3331>))
-- Add metrics to track resource usage by background processes ([\#3553](https://github.com/matrix-org/synapse/issues/3553>), [\#3556](https://github.com/matrix-org/synapse/issues/3556>), [\#3604](https://github.com/matrix-org/synapse/issues/3604>), [\#3610](https://github.com/matrix-org/synapse/issues/3610>))
-- Add `code` label to `synapse_http_server_response_time_seconds` prometheus metric ([\#3554](https://github.com/matrix-org/synapse/issues/3554>))
-- Add support for client_reader to handle more APIs ([\#3555](https://github.com/matrix-org/synapse/issues/3555>), [\#3597](https://github.com/matrix-org/synapse/issues/3597>))
-- make the /context API filter & lazy-load aware as per MSC1227 ([\#3567](https://github.com/matrix-org/synapse/issues/3567>))
-- Add ability to limit number of monthly active users on the server ([\#3630](https://github.com/matrix-org/synapse/issues/3630>))
-- When we fail to join a room over federation, pass the error code back to the client. ([\#3639](https://github.com/matrix-org/synapse/issues/3639>))
-- Add a new /admin/register API for non-interactively creating users. ([\#3415](https://github.com/matrix-org/synapse/issues/3415>))
+- add support for the lazy_loaded_members filter as per MSC1227 ([\#2970](https://github.com/matrix-org/synapse/issues/2970))
+- add support for the include_redundant_members filter param as per MSC1227 ([\#3331](https://github.com/matrix-org/synapse/issues/3331))
+- Add metrics to track resource usage by background processes ([\#3553](https://github.com/matrix-org/synapse/issues/3553), [\#3556](https://github.com/matrix-org/synapse/issues/3556), [\#3604](https://github.com/matrix-org/synapse/issues/3604), [\#3610](https://github.com/matrix-org/synapse/issues/3610))
+- Add `code` label to `synapse_http_server_response_time_seconds` prometheus metric ([\#3554](https://github.com/matrix-org/synapse/issues/3554))
+- Add support for client_reader to handle more APIs ([\#3555](https://github.com/matrix-org/synapse/issues/3555), [\#3597](https://github.com/matrix-org/synapse/issues/3597))
+- make the /context API filter & lazy-load aware as per MSC1227 ([\#3567](https://github.com/matrix-org/synapse/issues/3567))
+- Add ability to limit number of monthly active users on the server ([\#3630](https://github.com/matrix-org/synapse/issues/3630))
+- When we fail to join a room over federation, pass the error code back to the client. ([\#3639](https://github.com/matrix-org/synapse/issues/3639))
+- Add a new /admin/register API for non-interactively creating users. ([\#3415](https://github.com/matrix-org/synapse/issues/3415))
 
 
 Bugfixes
 --------
 
-- Make /directory/list API return 404 for room not found instead of 400 ([\#2952](https://github.com/matrix-org/synapse/issues/2952>))
-- Default inviter_display_name to mxid for email invites ([\#3391](https://github.com/matrix-org/synapse/issues/3391>))
-- Don't generate TURN credentials if no TURN config options are set ([\#3514](https://github.com/matrix-org/synapse/issues/3514>))
-- Correctly announce deleted devices over federation ([\#3520](https://github.com/matrix-org/synapse/issues/3520>))
-- Catch failures saving metrics captured by Measure, and instead log the faulty metrics information for further analysis. ([\#3548](https://github.com/matrix-org/synapse/issues/3548>))
-- Unicode passwords are now normalised before hashing, preventing the instance where two different devices or browsers might send a different UTF-8 sequence for the password. ([\#3569](https://github.com/matrix-org/synapse/issues/3569>))
-- Fix potential stack overflow and deadlock under heavy load ([\#3570](https://github.com/matrix-org/synapse/issues/3570>))
-- Respond with M_NOT_FOUND when profiles are not found locally or over federation. Fixes #3585 ([\#3585](https://github.com/matrix-org/synapse/issues/3585>))
-- Fix failure to persist events over federation under load ([\#3601](https://github.com/matrix-org/synapse/issues/3601>))
-- Fix updating of cached remote profiles ([\#3605](https://github.com/matrix-org/synapse/issues/3605>))
-- Fix 'tuple index out of range' error ([\#3607](https://github.com/matrix-org/synapse/issues/3607>))
-- Only import secrets when available (fix for py < 3.6) ([\#3626](https://github.com/matrix-org/synapse/issues/3626>))
+- Make /directory/list API return 404 for room not found instead of 400 ([\#2952](https://github.com/matrix-org/synapse/issues/2952))
+- Default inviter_display_name to mxid for email invites ([\#3391](https://github.com/matrix-org/synapse/issues/3391))
+- Don't generate TURN credentials if no TURN config options are set ([\#3514](https://github.com/matrix-org/synapse/issues/3514))
+- Correctly announce deleted devices over federation ([\#3520](https://github.com/matrix-org/synapse/issues/3520))
+- Catch failures saving metrics captured by Measure, and instead log the faulty metrics information for further analysis. ([\#3548](https://github.com/matrix-org/synapse/issues/3548))
+- Unicode passwords are now normalised before hashing, preventing the instance where two different devices or browsers might send a different UTF-8 sequence for the password. ([\#3569](https://github.com/matrix-org/synapse/issues/3569))
+- Fix potential stack overflow and deadlock under heavy load ([\#3570](https://github.com/matrix-org/synapse/issues/3570))
+- Respond with M_NOT_FOUND when profiles are not found locally or over federation. Fixes #3585 ([\#3585](https://github.com/matrix-org/synapse/issues/3585))
+- Fix failure to persist events over federation under load ([\#3601](https://github.com/matrix-org/synapse/issues/3601))
+- Fix updating of cached remote profiles ([\#3605](https://github.com/matrix-org/synapse/issues/3605))
+- Fix 'tuple index out of range' error ([\#3607](https://github.com/matrix-org/synapse/issues/3607))
+- Only import secrets when available (fix for py < 3.6) ([\#3626](https://github.com/matrix-org/synapse/issues/3626))
 
 
 Internal Changes
 ----------------
 
-- Remove redundant checks on who_forgot_in_room ([\#3350](https://github.com/matrix-org/synapse/issues/3350>))
-- Remove unnecessary event re-signing hacks ([\#3367](https://github.com/matrix-org/synapse/issues/3367>))
-- Rewrite cache list decorator ([\#3384](https://github.com/matrix-org/synapse/issues/3384>))
-- Move v1-only REST APIs into their own module. ([\#3460](https://github.com/matrix-org/synapse/issues/3460>))
-- Replace more instances of Python 2-only iteritems and itervalues uses. ([\#3562](https://github.com/matrix-org/synapse/issues/3562>))
-- Refactor EventContext to accept state during init ([\#3577](https://github.com/matrix-org/synapse/issues/3577>))
-- Improve Dockerfile and docker-compose instructions ([\#3543](https://github.com/matrix-org/synapse/issues/3543>))
-- Release notes are now in the Markdown format. ([\#3552](https://github.com/matrix-org/synapse/issues/3552>))
-- add config for pep8 ([\#3559](https://github.com/matrix-org/synapse/issues/3559>))
-- Merge Linearizer and Limiter ([\#3571](https://github.com/matrix-org/synapse/issues/3571>), [\#3572](https://github.com/matrix-org/synapse/issues/3572>))
-- Lazily load state on master process when using workers to reduce DB consumption ([\#3579](https://github.com/matrix-org/synapse/issues/3579>), [\#3581](https://github.com/matrix-org/synapse/issues/3581>), [\#3582](https://github.com/matrix-org/synapse/issues/3582>), [\#3584](https://github.com/matrix-org/synapse/issues/3584>))
-- Fixes and optimisations for resolve_state_groups ([\#3586](https://github.com/matrix-org/synapse/issues/3586>))
-- Improve logging for exceptions when handling PDUs ([\#3587](https://github.com/matrix-org/synapse/issues/3587>))
-- Add some measure blocks to persist_events ([\#3590](https://github.com/matrix-org/synapse/issues/3590>))
-- Fix some random logcontext leaks. ([\#3591](https://github.com/matrix-org/synapse/issues/3591>), [\#3606](https://github.com/matrix-org/synapse/issues/3606>))
-- Speed up calculating state deltas in persist_event loop ([\#3592](https://github.com/matrix-org/synapse/issues/3592>))
-- Attempt to reduce amount of state pulled out of DB during persist_events ([\#3595](https://github.com/matrix-org/synapse/issues/3595>))
-- Fix a documentation typo in on_make_leave_request ([\#3609](https://github.com/matrix-org/synapse/issues/3609>))
-- Make EventStore inherit from EventFederationStore ([\#3612](https://github.com/matrix-org/synapse/issues/3612>))
-- Remove some redundant joins on event_edges.room_id ([\#3613](https://github.com/matrix-org/synapse/issues/3613>))
-- Stop populating events.content ([\#3614](https://github.com/matrix-org/synapse/issues/3614>))
-- Update the /send_leave path registration to use event_id rather than a transaction ID. ([\#3616](https://github.com/matrix-org/synapse/issues/3616>))
-- Refactor FederationHandler to move DB writes into separate functions ([\#3621](https://github.com/matrix-org/synapse/issues/3621>))
-- Remove unused field "pdu_failures" from transactions. ([\#3628](https://github.com/matrix-org/synapse/issues/3628>))
-- rename replication_layer to federation_client ([\#3634](https://github.com/matrix-org/synapse/issues/3634>))
-- Factor out exception handling in federation_client ([\#3638](https://github.com/matrix-org/synapse/issues/3638>))
-- Refactor location of docker build script. ([\#3644](https://github.com/matrix-org/synapse/issues/3644>))
-- Update CONTRIBUTING to mention newsfragments. ([\#3645](https://github.com/matrix-org/synapse/issues/3645>))
+- Remove redundant checks on who_forgot_in_room ([\#3350](https://github.com/matrix-org/synapse/issues/3350))
+- Remove unnecessary event re-signing hacks ([\#3367](https://github.com/matrix-org/synapse/issues/3367))
+- Rewrite cache list decorator ([\#3384](https://github.com/matrix-org/synapse/issues/3384))
+- Move v1-only REST APIs into their own module. ([\#3460](https://github.com/matrix-org/synapse/issues/3460))
+- Replace more instances of Python 2-only iteritems and itervalues uses. ([\#3562](https://github.com/matrix-org/synapse/issues/3562))
+- Refactor EventContext to accept state during init ([\#3577](https://github.com/matrix-org/synapse/issues/3577))
+- Improve Dockerfile and docker-compose instructions ([\#3543](https://github.com/matrix-org/synapse/issues/3543))
+- Release notes are now in the Markdown format. ([\#3552](https://github.com/matrix-org/synapse/issues/3552))
+- add config for pep8 ([\#3559](https://github.com/matrix-org/synapse/issues/3559))
+- Merge Linearizer and Limiter ([\#3571](https://github.com/matrix-org/synapse/issues/3571), [\#3572](https://github.com/matrix-org/synapse/issues/3572))
+- Lazily load state on master process when using workers to reduce DB consumption ([\#3579](https://github.com/matrix-org/synapse/issues/3579), [\#3581](https://github.com/matrix-org/synapse/issues/3581), [\#3582](https://github.com/matrix-org/synapse/issues/3582), [\#3584](https://github.com/matrix-org/synapse/issues/3584))
+- Fixes and optimisations for resolve_state_groups ([\#3586](https://github.com/matrix-org/synapse/issues/3586))
+- Improve logging for exceptions when handling PDUs ([\#3587](https://github.com/matrix-org/synapse/issues/3587))
+- Add some measure blocks to persist_events ([\#3590](https://github.com/matrix-org/synapse/issues/3590))
+- Fix some random logcontext leaks. ([\#3591](https://github.com/matrix-org/synapse/issues/3591), [\#3606](https://github.com/matrix-org/synapse/issues/3606))
+- Speed up calculating state deltas in persist_event loop ([\#3592](https://github.com/matrix-org/synapse/issues/3592))
+- Attempt to reduce amount of state pulled out of DB during persist_events ([\#3595](https://github.com/matrix-org/synapse/issues/3595))
+- Fix a documentation typo in on_make_leave_request ([\#3609](https://github.com/matrix-org/synapse/issues/3609))
+- Make EventStore inherit from EventFederationStore ([\#3612](https://github.com/matrix-org/synapse/issues/3612))
+- Remove some redundant joins on event_edges.room_id ([\#3613](https://github.com/matrix-org/synapse/issues/3613))
+- Stop populating events.content ([\#3614](https://github.com/matrix-org/synapse/issues/3614))
+- Update the /send_leave path registration to use event_id rather than a transaction ID. ([\#3616](https://github.com/matrix-org/synapse/issues/3616))
+- Refactor FederationHandler to move DB writes into separate functions ([\#3621](https://github.com/matrix-org/synapse/issues/3621))
+- Remove unused field "pdu_failures" from transactions. ([\#3628](https://github.com/matrix-org/synapse/issues/3628))
+- rename replication_layer to federation_client ([\#3634](https://github.com/matrix-org/synapse/issues/3634))
+- Factor out exception handling in federation_client ([\#3638](https://github.com/matrix-org/synapse/issues/3638))
+- Refactor location of docker build script. ([\#3644](https://github.com/matrix-org/synapse/issues/3644))
+- Update CONTRIBUTING to mention newsfragments. ([\#3645](https://github.com/matrix-org/synapse/issues/3645))
 
 
 Synapse 0.33.1 (2018-08-02)
diff --git a/pyproject.toml b/pyproject.toml
index f4d6f0c6bbfa..dd099dc9c81e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -2,7 +2,7 @@
     package = "synapse"
     filename = "CHANGES.md"
     directory = "changelog.d"
-    issue_format = "[\\#{issue}](https://github.com/matrix-org/synapse/issues/{issue}>)"
+    issue_format = "[\\#{issue}](https://github.com/matrix-org/synapse/issues/{issue})"
 
     [[tool.towncrier.type]]
         directory = "feature"

From dafe90a6be17f045f1e27ed8703320e6b0f1dbe8 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Thu, 9 Aug 2018 19:20:41 +1000
Subject: [PATCH 311/830] 0.33.2

---
 synapse/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/__init__.py b/synapse/__init__.py
index 807d78d79f91..a14d578e36a7 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -17,4 +17,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.33.2rc1"
+__version__ = "0.33.2"

From 67dbe4c89944cf5957f806ad63f60880e2242b23 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Thu, 9 Aug 2018 19:21:07 +1000
Subject: [PATCH 312/830] 0.33.2 changelog

---
 CHANGES.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/CHANGES.md b/CHANGES.md
index 8317e44f37c3..a299110a6b1b 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,9 @@
+Synapse 0.33.2 (2018-08-09)
+===========================
+
+No significant changes.
+
+
 Synapse 0.33.2rc1 (2018-08-07)
 ==============================
 

From 72d1902bbeaa029e7003aecaeaf9bc5a41d17646 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 9 Aug 2018 10:23:49 +0100
Subject: [PATCH 313/830] Fixup doc comments

---
 synapse/federation/federation_server.py | 11 +++++++++++
 synapse/replication/http/federation.py  | 17 +++++++++++++++++
 2 files changed, 28 insertions(+)

diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index d23c1cf13b99..3bc26c4ab43f 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -811,6 +811,13 @@ def on_query(self, query_type, args):
 
 
 class ReplicationFederationHandlerRegistry(FederationHandlerRegistry):
+    """A FederationHandlerRegistry for worker processes.
+
+    When receiving EDU or queries it will check if an appropriate handler has
+    been registered on the worker, if there isn't one then it calls off to the
+    master process.
+    """
+
     def __init__(self, hs):
         self.config = hs.config
         self.http_client = hs.get_simple_http_client()
@@ -822,6 +829,8 @@ def __init__(self, hs):
         super(ReplicationFederationHandlerRegistry, self).__init__()
 
     def on_edu(self, edu_type, origin, content):
+        """Overrides FederationHandlerRegistry
+        """
         handler = self.edu_handlers.get(edu_type)
         if handler:
             return super(ReplicationFederationHandlerRegistry, self).on_edu(
@@ -835,6 +844,8 @@ def on_edu(self, edu_type, origin, content):
         )
 
     def on_query(self, query_type, args):
+        """Overrides FederationHandlerRegistry
+        """
         handler = self.query_handlers.get(query_type)
         if handler:
             return handler(args)
diff --git a/synapse/replication/http/federation.py b/synapse/replication/http/federation.py
index 3fa7bd64c7b5..3e6cbbf5a19a 100644
--- a/synapse/replication/http/federation.py
+++ b/synapse/replication/http/federation.py
@@ -157,6 +157,15 @@ def _notify_persisted_event(self, event, max_stream_id):
 class ReplicationFederationSendEduRestServlet(ReplicationEndpoint):
     """Handles EDUs newly received from federation, including persisting and
     notifying.
+
+    Request format:
+
+        POST /_synapse/replication/fed_send_edu/:edu_type/:txn_id
+
+        {
+            "origin": ...,
+            "content: { ... }
+        }
     """
 
     NAME = "fed_send_edu"
@@ -196,6 +205,14 @@ def _handle_request(self, request, edu_type):
 
 class ReplicationGetQueryRestServlet(ReplicationEndpoint):
     """Handle responding to queries from federation.
+
+    Request format:
+
+        POST /_synapse/replication/fed_query/:query_type
+
+        {
+            "args": { ... }
+        }
     """
 
     NAME = "fed_query"

From b179537f2a51f4de52e2625939cc32eeba75cd6b Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 9 Aug 2018 10:29:48 +0100
Subject: [PATCH 314/830] Move clean_room_for_join to master

---
 synapse/handlers/federation.py         | 16 ++++++++++--
 synapse/replication/http/federation.py | 35 ++++++++++++++++++++++++++
 2 files changed, 49 insertions(+), 2 deletions(-)

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 37d2307d0abc..acabca1d258c 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -50,6 +50,7 @@
 )
 from synapse.events.validator import EventValidator
 from synapse.replication.http.federation import (
+    ReplicationCleanRoomRestServlet,
     ReplicationFederationSendEventsRestServlet,
 )
 from synapse.replication.http.membership import ReplicationUserJoinedLeftRoomRestServlet
@@ -104,6 +105,9 @@ def __init__(self, hs):
         self._notify_user_membership_change = (
             ReplicationUserJoinedLeftRoomRestServlet.make_client(hs)
         )
+        self._clean_room_for_join_client = (
+            ReplicationCleanRoomRestServlet.make_client(hs)
+        )
 
         # When joining a room we need to queue any events for that room up
         self.room_queues = {}
@@ -2388,8 +2392,16 @@ def _notify_persisted_event(self, event, max_stream_id):
         )
 
     def _clean_room_for_join(self, room_id):
-        # TODO move this out to master
-        return self.store.clean_room_for_join(room_id)
+        """Called to clean up any data in DB for a given room, ready for the
+        server to join the room.
+
+        Args:
+            room_id (str)
+        """
+        if self.config.worker_app:
+            return self._clean_room_for_join_client(room_id)
+        else:
+            return self.store.clean_room_for_join(room_id)
 
     def user_joined_room(self, user, room_id):
         """Called when a new user has joined the room
diff --git a/synapse/replication/http/federation.py b/synapse/replication/http/federation.py
index 3e6cbbf5a19a..7b0b1cd32ed1 100644
--- a/synapse/replication/http/federation.py
+++ b/synapse/replication/http/federation.py
@@ -256,7 +256,42 @@ def _handle_request(self, request, query_type):
         defer.returnValue((200, result))
 
 
+class ReplicationCleanRoomRestServlet(ReplicationEndpoint):
+    """Called to clean up any data in DB for a given room, ready for the
+    server to join the room.
+
+    Request format:
+
+        POST /_synapse/replication/fed_query/:fed_cleanup_room/:txn_id
+
+        {}
+    """
+
+    NAME = "fed_cleanup_room"
+    PATH_ARGS = ("room_id",)
+
+    def __init__(self, hs):
+        super(ReplicationCleanRoomRestServlet, self).__init__(hs)
+
+        self.store = hs.get_datastore()
+
+    @staticmethod
+    def _serialize_payload(room_id, args):
+        """
+        Args:
+            room_id (str)
+        """
+        return {}
+
+    @defer.inlineCallbacks
+    def _handle_request(self, request, room_id):
+        yield self.store.clean_room_for_join(room_id)
+
+        defer.returnValue((200, {}))
+
+
 def register_servlets(hs, http_server):
     ReplicationFederationSendEventsRestServlet(hs).register(http_server)
     ReplicationFederationSendEduRestServlet(hs).register(http_server)
     ReplicationGetQueryRestServlet(hs).register(http_server)
+    ReplicationCleanRoomRestServlet(hs).register(http_server)

From 8876ce7f77ecdf543cc23fb8333e9020791e9376 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 9 Aug 2018 10:30:50 +0100
Subject: [PATCH 315/830] Newsfile

---
 changelog.d/3653.feature | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3653.feature

diff --git a/changelog.d/3653.feature b/changelog.d/3653.feature
new file mode 100644
index 000000000000..6c5422994f34
--- /dev/null
+++ b/changelog.d/3653.feature
@@ -0,0 +1 @@
+Support more federation endpoints on workers

From 5c6226707d2588c9f2669512adabc00687295445 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 9 Aug 2018 10:33:55 +0100
Subject: [PATCH 316/830] Update docs/workers.rst

---
 docs/workers.rst | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/docs/workers.rst b/docs/workers.rst
index c5b37c3ded73..ac9efb621f06 100644
--- a/docs/workers.rst
+++ b/docs/workers.rst
@@ -173,10 +173,23 @@ endpoints matching the following regular expressions::
     ^/_matrix/federation/v1/backfill/
     ^/_matrix/federation/v1/get_missing_events/
     ^/_matrix/federation/v1/publicRooms
+    ^/_matrix/federation/v1/query/
+    ^/_matrix/federation/v1/make_join/
+    ^/_matrix/federation/v1/make_leave/
+    ^/_matrix/federation/v1/send_join/
+    ^/_matrix/federation/v1/send_leave/
+    ^/_matrix/federation/v1/invite/
+    ^/_matrix/federation/v1/query_auth/
+    ^/_matrix/federation/v1/event_auth/
+    ^/_matrix/federation/v1/exchange_third_party_invite/
+    ^/_matrix/federation/v1/send/
 
 The above endpoints should all be routed to the federation_reader worker by the
 reverse-proxy configuration.
 
+The `^/_matrix/federation/v1/send/` endpoint must only be handled by a single
+instance.
+
 ``synapse.app.federation_sender``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

From 54a9bea88c6df7107869d9a038a6f9a55d16e67b Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 9 Aug 2018 10:39:29 +0100
Subject: [PATCH 317/830] Newsfile

---
 changelog.d/3659.feature | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3659.feature

diff --git a/changelog.d/3659.feature b/changelog.d/3659.feature
new file mode 100644
index 000000000000..a5b4821c0953
--- /dev/null
+++ b/changelog.d/3659.feature
@@ -0,0 +1 @@
+Support profile API endpoints on workers

From a6c813761aa38d9ea0ff6db7303b25dfc7b77712 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 9 Aug 2018 10:41:08 +0100
Subject: [PATCH 318/830] Docstrings

---
 synapse/replication/http/profile.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/synapse/replication/http/profile.py b/synapse/replication/http/profile.py
index c4d54c936fe4..aa700c243435 100644
--- a/synapse/replication/http/profile.py
+++ b/synapse/replication/http/profile.py
@@ -25,9 +25,19 @@
 
 
 class ReplicationHandleProfileChangeRestServlet(ReplicationEndpoint):
+    """Notifies that a users profile has changed
+
+    Request format:
+
+        POST /_synapse/replication/profile_changed/:user_id
+
+        {
+            "requester": ...
+        }
+    """
+
     NAME = "profile_changed"
     PATH_ARGS = ("user_id",)
-    POST = True
 
     def __init__(self, hs):
         super(ReplicationHandleProfileChangeRestServlet, self).__init__(hs)

From 69ce057ea613f425d5ef6ace03d0019a8e4fdf49 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Thu, 9 Aug 2018 12:26:27 +0100
Subject: [PATCH 319/830] block sync if auth checks fail

---
 synapse/handlers/sync.py    | 12 +++++-------
 tests/handlers/test_sync.py | 19 +++++++++++++------
 2 files changed, 18 insertions(+), 13 deletions(-)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index f748d9afb099..776ddca638c8 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -209,14 +209,12 @@ def wait_for_sync_for_user(self, sync_config, since_token=None, timeout=0,
             Deferred[SyncResult]
         """
         yield self.auth.check_auth_blocking()
-
-        defer.returnValue(
-            self.response_cache.wrap(
-                sync_config.request_key,
-                self._wait_for_sync_for_user,
-                sync_config, since_token, timeout, full_state,
-            )
+        res = yield self.response_cache.wrap(
+            sync_config.request_key,
+            self._wait_for_sync_for_user,
+            sync_config, since_token, timeout, full_state,
         )
+        defer.returnValue(res)
 
     @defer.inlineCallbacks
     def _wait_for_sync_for_user(self, sync_config, since_token, timeout,
diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py
index 3b1b4d492367..497e4bd933a0 100644
--- a/tests/handlers/test_sync.py
+++ b/tests/handlers/test_sync.py
@@ -14,11 +14,14 @@
 # limitations under the License.
 from twisted.internet import defer
 
+from synapse.api.errors import AuthError
+from synapse.api.filtering import DEFAULT_FILTER_COLLECTION
+from synapse.handlers.sync import SyncConfig, SyncHandler
+from synapse.types import UserID
+
 import tests.unittest
 import tests.utils
 from tests.utils import setup_test_homeserver
-from synapse.handlers.sync import SyncHandler, SyncConfig
-from synapse.types import UserID
 
 
 class SyncTestCase(tests.unittest.TestCase):
@@ -32,11 +35,15 @@ def setUp(self):
     @defer.inlineCallbacks
     def test_wait_for_sync_for_user_auth_blocking(self):
         sync_config = SyncConfig(
-            user=UserID("@user","server"),
-            filter_collection=None,
+            user=UserID("@user", "server"),
+            filter_collection=DEFAULT_FILTER_COLLECTION,
             is_guest=False,
             request_key="request_key",
             device_id="device_id",
         )
-        res = yield self.sync_handler.wait_for_sync_for_user(sync_config)
-        print res
+        # Ensure that an exception is not thrown
+        yield self.sync_handler.wait_for_sync_for_user(sync_config)
+        self.hs.config.hs_disabled = True
+
+        with self.assertRaises(AuthError):
+            yield self.sync_handler.wait_for_sync_for_user(sync_config)

From d967653705854dac98cde06cb3de54113e704e60 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Christian=20Gr=C3=BCnhage?=
 <jan.christian@gruenhage.xyz>
Date: Thu, 9 Aug 2018 13:21:30 +0200
Subject: [PATCH 320/830] update docker base-image to alpine 3.8

---
 changelog.d/3669.misc | 1 +
 docker/Dockerfile     | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/3669.misc

diff --git a/changelog.d/3669.misc b/changelog.d/3669.misc
new file mode 100644
index 000000000000..fc579ddc607d
--- /dev/null
+++ b/changelog.d/3669.misc
@@ -0,0 +1 @@
+Update docker base image from alpine 3.7 to 3.8.
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 26fb3a6bfffd..777976217d4f 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -1,4 +1,4 @@
-FROM docker.io/python:2-alpine3.7
+FROM docker.io/python:2-alpine3.8
 
 RUN apk add --no-cache --virtual .nacl_deps \
         build-base \

From c6b28fb4791d88e064cd88686dd32e7ed7834525 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Thu, 9 Aug 2018 12:45:58 +0100
Subject: [PATCH 321/830] Where server is disabled, block ability for locked
 out users to read new messages

---
 changelog.d/3670.feature | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3670.feature

diff --git a/changelog.d/3670.feature b/changelog.d/3670.feature
new file mode 100644
index 000000000000..ba00f2d2ecb8
--- /dev/null
+++ b/changelog.d/3670.feature
@@ -0,0 +1 @@
+Where server is disabled, block ability for locked out users to read new messages

From 119451dcd192ff5397a3f9630f14556737e82677 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 7 Aug 2018 15:22:40 +0100
Subject: [PATCH 322/830] Refactor state module

We split out the actual state resolution algorithm to prepare for having
multiple versions.
---
 synapse/{state.py => state/__init__.py} | 300 +---------------------
 synapse/state/v1.py                     | 321 ++++++++++++++++++++++++
 2 files changed, 325 insertions(+), 296 deletions(-)
 rename synapse/{state.py => state/__init__.py} (67%)
 create mode 100644 synapse/state/v1.py

diff --git a/synapse/state.py b/synapse/state/__init__.py
similarity index 67%
rename from synapse/state.py
rename to synapse/state/__init__.py
index e1092b97a9b4..8c091d07c91b 100644
--- a/synapse/state.py
+++ b/synapse/state/__init__.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,20 +14,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
-import hashlib
 import logging
 from collections import namedtuple
 
-from six import iteritems, iterkeys, itervalues
+from six import iteritems, itervalues
 
 from frozendict import frozendict
 
 from twisted.internet import defer
 
-from synapse import event_auth
 from synapse.api.constants import EventTypes
-from synapse.api.errors import AuthError
 from synapse.events.snapshot import EventContext
 from synapse.util.async import Linearizer
 from synapse.util.caches import CACHE_SIZE_FACTOR
@@ -34,6 +31,8 @@
 from synapse.util.logutils import log_function
 from synapse.util.metrics import Measure
 
+from .v1 import resolve_events_with_factory, resolve_events_with_state_map
+
 logger = logging.getLogger(__name__)
 
 
@@ -573,294 +572,3 @@ def _make_state_cache_entry(
         prev_group=prev_group,
         delta_ids=delta_ids,
     )
-
-
-def _ordered_events(events):
-    def key_func(e):
-        return -int(e.depth), hashlib.sha1(e.event_id.encode('ascii')).hexdigest()
-
-    return sorted(events, key=key_func)
-
-
-def resolve_events_with_state_map(state_sets, state_map):
-    """
-    Args:
-        state_sets(list): List of dicts of (type, state_key) -> event_id,
-            which are the different state groups to resolve.
-        state_map(dict): a dict from event_id to event, for all events in
-            state_sets.
-
-    Returns
-        dict[(str, str), str]:
-            a map from (type, state_key) to event_id.
-    """
-    if len(state_sets) == 1:
-        return state_sets[0]
-
-    unconflicted_state, conflicted_state = _seperate(
-        state_sets,
-    )
-
-    auth_events = _create_auth_events_from_maps(
-        unconflicted_state, conflicted_state, state_map
-    )
-
-    return _resolve_with_state(
-        unconflicted_state, conflicted_state, auth_events, state_map
-    )
-
-
-def _seperate(state_sets):
-    """Takes the state_sets and figures out which keys are conflicted and
-    which aren't. i.e., which have multiple different event_ids associated
-    with them in different state sets.
-
-    Args:
-        state_sets(iterable[dict[(str, str), str]]):
-            List of dicts of (type, state_key) -> event_id, which are the
-            different state groups to resolve.
-
-    Returns:
-        (dict[(str, str), str], dict[(str, str), set[str]]):
-            A tuple of (unconflicted_state, conflicted_state), where:
-
-            unconflicted_state is a dict mapping (type, state_key)->event_id
-            for unconflicted state keys.
-
-            conflicted_state is a dict mapping (type, state_key) to a set of
-            event ids for conflicted state keys.
-    """
-    state_set_iterator = iter(state_sets)
-    unconflicted_state = dict(next(state_set_iterator))
-    conflicted_state = {}
-
-    for state_set in state_set_iterator:
-        for key, value in iteritems(state_set):
-            # Check if there is an unconflicted entry for the state key.
-            unconflicted_value = unconflicted_state.get(key)
-            if unconflicted_value is None:
-                # There isn't an unconflicted entry so check if there is a
-                # conflicted entry.
-                ls = conflicted_state.get(key)
-                if ls is None:
-                    # There wasn't a conflicted entry so haven't seen this key before.
-                    # Therefore it isn't conflicted yet.
-                    unconflicted_state[key] = value
-                else:
-                    # This key is already conflicted, add our value to the conflict set.
-                    ls.add(value)
-            elif unconflicted_value != value:
-                # If the unconflicted value is not the same as our value then we
-                # have a new conflict. So move the key from the unconflicted_state
-                # to the conflicted state.
-                conflicted_state[key] = {value, unconflicted_value}
-                unconflicted_state.pop(key, None)
-
-    return unconflicted_state, conflicted_state
-
-
-@defer.inlineCallbacks
-def resolve_events_with_factory(state_sets, event_map, state_map_factory):
-    """
-    Args:
-        state_sets(list): List of dicts of (type, state_key) -> event_id,
-            which are the different state groups to resolve.
-
-        event_map(dict[str,FrozenEvent]|None):
-            a dict from event_id to event, for any events that we happen to
-            have in flight (eg, those currently being persisted). This will be
-            used as a starting point fof finding the state we need; any missing
-            events will be requested via state_map_factory.
-
-            If None, all events will be fetched via state_map_factory.
-
-        state_map_factory(func): will be called
-            with a list of event_ids that are needed, and should return with
-            a Deferred of dict of event_id to event.
-
-    Returns
-        Deferred[dict[(str, str), str]]:
-            a map from (type, state_key) to event_id.
-    """
-    if len(state_sets) == 1:
-        defer.returnValue(state_sets[0])
-
-    unconflicted_state, conflicted_state = _seperate(
-        state_sets,
-    )
-
-    needed_events = set(
-        event_id
-        for event_ids in itervalues(conflicted_state)
-        for event_id in event_ids
-    )
-    if event_map is not None:
-        needed_events -= set(iterkeys(event_map))
-
-    logger.info("Asking for %d conflicted events", len(needed_events))
-
-    # dict[str, FrozenEvent]: a map from state event id to event. Only includes
-    # the state events which are in conflict (and those in event_map)
-    state_map = yield state_map_factory(needed_events)
-    if event_map is not None:
-        state_map.update(event_map)
-
-    # get the ids of the auth events which allow us to authenticate the
-    # conflicted state, picking only from the unconflicting state.
-    #
-    # dict[(str, str), str]: a map from state key to event id
-    auth_events = _create_auth_events_from_maps(
-        unconflicted_state, conflicted_state, state_map
-    )
-
-    new_needed_events = set(itervalues(auth_events))
-    new_needed_events -= needed_events
-    if event_map is not None:
-        new_needed_events -= set(iterkeys(event_map))
-
-    logger.info("Asking for %d auth events", len(new_needed_events))
-
-    state_map_new = yield state_map_factory(new_needed_events)
-    state_map.update(state_map_new)
-
-    defer.returnValue(_resolve_with_state(
-        unconflicted_state, conflicted_state, auth_events, state_map
-    ))
-
-
-def _create_auth_events_from_maps(unconflicted_state, conflicted_state, state_map):
-    auth_events = {}
-    for event_ids in itervalues(conflicted_state):
-        for event_id in event_ids:
-            if event_id in state_map:
-                keys = event_auth.auth_types_for_event(state_map[event_id])
-                for key in keys:
-                    if key not in auth_events:
-                        event_id = unconflicted_state.get(key, None)
-                        if event_id:
-                            auth_events[key] = event_id
-    return auth_events
-
-
-def _resolve_with_state(unconflicted_state_ids, conflicted_state_ids, auth_event_ids,
-                        state_map):
-    conflicted_state = {}
-    for key, event_ids in iteritems(conflicted_state_ids):
-        events = [state_map[ev_id] for ev_id in event_ids if ev_id in state_map]
-        if len(events) > 1:
-            conflicted_state[key] = events
-        elif len(events) == 1:
-            unconflicted_state_ids[key] = events[0].event_id
-
-    auth_events = {
-        key: state_map[ev_id]
-        for key, ev_id in iteritems(auth_event_ids)
-        if ev_id in state_map
-    }
-
-    try:
-        resolved_state = _resolve_state_events(
-            conflicted_state, auth_events
-        )
-    except Exception:
-        logger.exception("Failed to resolve state")
-        raise
-
-    new_state = unconflicted_state_ids
-    for key, event in iteritems(resolved_state):
-        new_state[key] = event.event_id
-
-    return new_state
-
-
-def _resolve_state_events(conflicted_state, auth_events):
-    """ This is where we actually decide which of the conflicted state to
-    use.
-
-    We resolve conflicts in the following order:
-        1. power levels
-        2. join rules
-        3. memberships
-        4. other events.
-    """
-    resolved_state = {}
-    if POWER_KEY in conflicted_state:
-        events = conflicted_state[POWER_KEY]
-        logger.debug("Resolving conflicted power levels %r", events)
-        resolved_state[POWER_KEY] = _resolve_auth_events(
-            events, auth_events)
-
-    auth_events.update(resolved_state)
-
-    for key, events in iteritems(conflicted_state):
-        if key[0] == EventTypes.JoinRules:
-            logger.debug("Resolving conflicted join rules %r", events)
-            resolved_state[key] = _resolve_auth_events(
-                events,
-                auth_events
-            )
-
-    auth_events.update(resolved_state)
-
-    for key, events in iteritems(conflicted_state):
-        if key[0] == EventTypes.Member:
-            logger.debug("Resolving conflicted member lists %r", events)
-            resolved_state[key] = _resolve_auth_events(
-                events,
-                auth_events
-            )
-
-    auth_events.update(resolved_state)
-
-    for key, events in iteritems(conflicted_state):
-        if key not in resolved_state:
-            logger.debug("Resolving conflicted state %r:%r", key, events)
-            resolved_state[key] = _resolve_normal_events(
-                events, auth_events
-            )
-
-    return resolved_state
-
-
-def _resolve_auth_events(events, auth_events):
-    reverse = [i for i in reversed(_ordered_events(events))]
-
-    auth_keys = set(
-        key
-        for event in events
-        for key in event_auth.auth_types_for_event(event)
-    )
-
-    new_auth_events = {}
-    for key in auth_keys:
-        auth_event = auth_events.get(key, None)
-        if auth_event:
-            new_auth_events[key] = auth_event
-
-    auth_events = new_auth_events
-
-    prev_event = reverse[0]
-    for event in reverse[1:]:
-        auth_events[(prev_event.type, prev_event.state_key)] = prev_event
-        try:
-            # The signatures have already been checked at this point
-            event_auth.check(event, auth_events, do_sig_check=False, do_size_check=False)
-            prev_event = event
-        except AuthError:
-            return prev_event
-
-    return event
-
-
-def _resolve_normal_events(events, auth_events):
-    for event in _ordered_events(events):
-        try:
-            # The signatures have already been checked at this point
-            event_auth.check(event, auth_events, do_sig_check=False, do_size_check=False)
-            return event
-        except AuthError:
-            pass
-
-    # Use the last event (the one with the least depth) if they all fail
-    # the auth check.
-    return event
diff --git a/synapse/state/v1.py b/synapse/state/v1.py
new file mode 100644
index 000000000000..3a1f7054a172
--- /dev/null
+++ b/synapse/state/v1.py
@@ -0,0 +1,321 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import hashlib
+import logging
+
+from six import iteritems, iterkeys, itervalues
+
+from twisted.internet import defer
+
+from synapse import event_auth
+from synapse.api.constants import EventTypes
+from synapse.api.errors import AuthError
+
+logger = logging.getLogger(__name__)
+
+
+POWER_KEY = (EventTypes.PowerLevels, "")
+
+
+def resolve_events_with_state_map(state_sets, state_map):
+    """
+    Args:
+        state_sets(list): List of dicts of (type, state_key) -> event_id,
+            which are the different state groups to resolve.
+        state_map(dict): a dict from event_id to event, for all events in
+            state_sets.
+
+    Returns
+        dict[(str, str), str]:
+            a map from (type, state_key) to event_id.
+    """
+    if len(state_sets) == 1:
+        return state_sets[0]
+
+    unconflicted_state, conflicted_state = _seperate(
+        state_sets,
+    )
+
+    auth_events = _create_auth_events_from_maps(
+        unconflicted_state, conflicted_state, state_map
+    )
+
+    return _resolve_with_state(
+        unconflicted_state, conflicted_state, auth_events, state_map
+    )
+
+
+@defer.inlineCallbacks
+def resolve_events_with_factory(state_sets, event_map, state_map_factory):
+    """
+    Args:
+        state_sets(list): List of dicts of (type, state_key) -> event_id,
+            which are the different state groups to resolve.
+
+        event_map(dict[str,FrozenEvent]|None):
+            a dict from event_id to event, for any events that we happen to
+            have in flight (eg, those currently being persisted). This will be
+            used as a starting point fof finding the state we need; any missing
+            events will be requested via state_map_factory.
+
+            If None, all events will be fetched via state_map_factory.
+
+        state_map_factory(func): will be called
+            with a list of event_ids that are needed, and should return with
+            a Deferred of dict of event_id to event.
+
+    Returns
+        Deferred[dict[(str, str), str]]:
+            a map from (type, state_key) to event_id.
+    """
+    if len(state_sets) == 1:
+        defer.returnValue(state_sets[0])
+
+    unconflicted_state, conflicted_state = _seperate(
+        state_sets,
+    )
+
+    needed_events = set(
+        event_id
+        for event_ids in itervalues(conflicted_state)
+        for event_id in event_ids
+    )
+    if event_map is not None:
+        needed_events -= set(iterkeys(event_map))
+
+    logger.info("Asking for %d conflicted events", len(needed_events))
+
+    # dict[str, FrozenEvent]: a map from state event id to event. Only includes
+    # the state events which are in conflict (and those in event_map)
+    state_map = yield state_map_factory(needed_events)
+    if event_map is not None:
+        state_map.update(event_map)
+
+    # get the ids of the auth events which allow us to authenticate the
+    # conflicted state, picking only from the unconflicting state.
+    #
+    # dict[(str, str), str]: a map from state key to event id
+    auth_events = _create_auth_events_from_maps(
+        unconflicted_state, conflicted_state, state_map
+    )
+
+    new_needed_events = set(itervalues(auth_events))
+    new_needed_events -= needed_events
+    if event_map is not None:
+        new_needed_events -= set(iterkeys(event_map))
+
+    logger.info("Asking for %d auth events", len(new_needed_events))
+
+    state_map_new = yield state_map_factory(new_needed_events)
+    state_map.update(state_map_new)
+
+    defer.returnValue(_resolve_with_state(
+        unconflicted_state, conflicted_state, auth_events, state_map
+    ))
+
+
+def _seperate(state_sets):
+    """Takes the state_sets and figures out which keys are conflicted and
+    which aren't. i.e., which have multiple different event_ids associated
+    with them in different state sets.
+
+    Args:
+        state_sets(iterable[dict[(str, str), str]]):
+            List of dicts of (type, state_key) -> event_id, which are the
+            different state groups to resolve.
+
+    Returns:
+        (dict[(str, str), str], dict[(str, str), set[str]]):
+            A tuple of (unconflicted_state, conflicted_state), where:
+
+            unconflicted_state is a dict mapping (type, state_key)->event_id
+            for unconflicted state keys.
+
+            conflicted_state is a dict mapping (type, state_key) to a set of
+            event ids for conflicted state keys.
+    """
+    state_set_iterator = iter(state_sets)
+    unconflicted_state = dict(next(state_set_iterator))
+    conflicted_state = {}
+
+    for state_set in state_set_iterator:
+        for key, value in iteritems(state_set):
+            # Check if there is an unconflicted entry for the state key.
+            unconflicted_value = unconflicted_state.get(key)
+            if unconflicted_value is None:
+                # There isn't an unconflicted entry so check if there is a
+                # conflicted entry.
+                ls = conflicted_state.get(key)
+                if ls is None:
+                    # There wasn't a conflicted entry so haven't seen this key before.
+                    # Therefore it isn't conflicted yet.
+                    unconflicted_state[key] = value
+                else:
+                    # This key is already conflicted, add our value to the conflict set.
+                    ls.add(value)
+            elif unconflicted_value != value:
+                # If the unconflicted value is not the same as our value then we
+                # have a new conflict. So move the key from the unconflicted_state
+                # to the conflicted state.
+                conflicted_state[key] = {value, unconflicted_value}
+                unconflicted_state.pop(key, None)
+
+    return unconflicted_state, conflicted_state
+
+
+def _create_auth_events_from_maps(unconflicted_state, conflicted_state, state_map):
+    auth_events = {}
+    for event_ids in itervalues(conflicted_state):
+        for event_id in event_ids:
+            if event_id in state_map:
+                keys = event_auth.auth_types_for_event(state_map[event_id])
+                for key in keys:
+                    if key not in auth_events:
+                        event_id = unconflicted_state.get(key, None)
+                        if event_id:
+                            auth_events[key] = event_id
+    return auth_events
+
+
+def _resolve_with_state(unconflicted_state_ids, conflicted_state_ids, auth_event_ids,
+                        state_map):
+    conflicted_state = {}
+    for key, event_ids in iteritems(conflicted_state_ids):
+        events = [state_map[ev_id] for ev_id in event_ids if ev_id in state_map]
+        if len(events) > 1:
+            conflicted_state[key] = events
+        elif len(events) == 1:
+            unconflicted_state_ids[key] = events[0].event_id
+
+    auth_events = {
+        key: state_map[ev_id]
+        for key, ev_id in iteritems(auth_event_ids)
+        if ev_id in state_map
+    }
+
+    try:
+        resolved_state = _resolve_state_events(
+            conflicted_state, auth_events
+        )
+    except Exception:
+        logger.exception("Failed to resolve state")
+        raise
+
+    new_state = unconflicted_state_ids
+    for key, event in iteritems(resolved_state):
+        new_state[key] = event.event_id
+
+    return new_state
+
+
+def _resolve_state_events(conflicted_state, auth_events):
+    """ This is where we actually decide which of the conflicted state to
+    use.
+
+    We resolve conflicts in the following order:
+        1. power levels
+        2. join rules
+        3. memberships
+        4. other events.
+    """
+    resolved_state = {}
+    if POWER_KEY in conflicted_state:
+        events = conflicted_state[POWER_KEY]
+        logger.debug("Resolving conflicted power levels %r", events)
+        resolved_state[POWER_KEY] = _resolve_auth_events(
+            events, auth_events)
+
+    auth_events.update(resolved_state)
+
+    for key, events in iteritems(conflicted_state):
+        if key[0] == EventTypes.JoinRules:
+            logger.debug("Resolving conflicted join rules %r", events)
+            resolved_state[key] = _resolve_auth_events(
+                events,
+                auth_events
+            )
+
+    auth_events.update(resolved_state)
+
+    for key, events in iteritems(conflicted_state):
+        if key[0] == EventTypes.Member:
+            logger.debug("Resolving conflicted member lists %r", events)
+            resolved_state[key] = _resolve_auth_events(
+                events,
+                auth_events
+            )
+
+    auth_events.update(resolved_state)
+
+    for key, events in iteritems(conflicted_state):
+        if key not in resolved_state:
+            logger.debug("Resolving conflicted state %r:%r", key, events)
+            resolved_state[key] = _resolve_normal_events(
+                events, auth_events
+            )
+
+    return resolved_state
+
+
+def _resolve_auth_events(events, auth_events):
+    reverse = [i for i in reversed(_ordered_events(events))]
+
+    auth_keys = set(
+        key
+        for event in events
+        for key in event_auth.auth_types_for_event(event)
+    )
+
+    new_auth_events = {}
+    for key in auth_keys:
+        auth_event = auth_events.get(key, None)
+        if auth_event:
+            new_auth_events[key] = auth_event
+
+    auth_events = new_auth_events
+
+    prev_event = reverse[0]
+    for event in reverse[1:]:
+        auth_events[(prev_event.type, prev_event.state_key)] = prev_event
+        try:
+            # The signatures have already been checked at this point
+            event_auth.check(event, auth_events, do_sig_check=False, do_size_check=False)
+            prev_event = event
+        except AuthError:
+            return prev_event
+
+    return event
+
+
+def _resolve_normal_events(events, auth_events):
+    for event in _ordered_events(events):
+        try:
+            # The signatures have already been checked at this point
+            event_auth.check(event, auth_events, do_sig_check=False, do_size_check=False)
+            return event
+        except AuthError:
+            pass
+
+    # Use the last event (the one with the least depth) if they all fail
+    # the auth check.
+    return event
+
+
+def _ordered_events(events):
+    def key_func(e):
+        return -int(e.depth), hashlib.sha1(e.event_id.encode('ascii')).hexdigest()
+
+    return sorted(events, key=key_func)

From 152c0aa58edd1453f6c8c16a82cdb92165d0adba Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 7 Aug 2018 15:27:19 +0100
Subject: [PATCH 323/830] Add constants for room versions

---
 synapse/api/constants.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/synapse/api/constants.py b/synapse/api/constants.py
index b0da506f6d64..912bf024bfef 100644
--- a/synapse/api/constants.py
+++ b/synapse/api/constants.py
@@ -97,9 +97,14 @@ class ThirdPartyEntityKind(object):
     LOCATION = "location"
 
 
+class RoomVersions(object):
+    V1 = "1"
+    VDH_TEST = "vdh-test-version"
+
+
 # the version we will give rooms which are created on this server
-DEFAULT_ROOM_VERSION = "1"
+DEFAULT_ROOM_VERSION = RoomVersions.V1
 
 # vdh-test-version is a placeholder to get room versioning support working and tested
 # until we have a working v2.
-KNOWN_ROOM_VERSIONS = {"1", "vdh-test-version"}
+KNOWN_ROOM_VERSIONS = {RoomVersions.V1, RoomVersions.VDH_TEST}

From ce6db0e5473e31292567b11599eb334c3275c564 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 8 Aug 2018 17:01:57 +0100
Subject: [PATCH 324/830] Choose state algorithm based on room version

---
 synapse/handlers/federation.py  |   8 ++-
 synapse/handlers/room_member.py |   5 +-
 synapse/state/__init__.py       | 104 ++++++++++++++++++++++++++++----
 synapse/storage/events.py       |   4 +-
 4 files changed, 105 insertions(+), 16 deletions(-)

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 0dffd44e22e0..75a819dd116f 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -274,8 +274,9 @@ def fetch(ev_ids):
                             ev_ids, get_prev_content=False, check_redacted=False
                         )
 
+                    room_version = yield self.store.get_room_version(pdu.room_id)
                     state_map = yield resolve_events_with_factory(
-                        state_groups, {pdu.event_id: pdu}, fetch
+                        room_version, state_groups, {pdu.event_id: pdu}, fetch
                     )
 
                     state = (yield self.store.get_events(state_map.values())).values()
@@ -1811,7 +1812,10 @@ def do_auth(self, origin, event, context, auth_events):
                     (d.type, d.state_key): d for d in different_events if d
                 })
 
-                new_state = self.state_handler.resolve_events(
+                room_version = yield self.store.get_room_version(event.room_id)
+
+                new_state = yield self.state_handler.resolve_events(
+                    room_version,
                     [list(local_view.values()), list(remote_view.values())],
                     event
                 )
diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 0d4a3f4677e5..b6010bb41eca 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -341,9 +341,10 @@ def _update_membership(
         prev_events_and_hashes = yield self.store.get_prev_events_for_room(
             room_id,
         )
-        latest_event_ids = (
+        latest_event_ids = [
             event_id for (event_id, _, _) in prev_events_and_hashes
-        )
+        ]
+
         current_state_ids = yield self.state_handler.get_current_state_ids(
             room_id, latest_event_ids=latest_event_ids,
         )
diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py
index 8c091d07c91b..222daa0b28ae 100644
--- a/synapse/state/__init__.py
+++ b/synapse/state/__init__.py
@@ -23,16 +23,15 @@
 
 from twisted.internet import defer
 
-from synapse.api.constants import EventTypes
+from synapse.api.constants import EventTypes, RoomVersions
 from synapse.events.snapshot import EventContext
+from synapse.state import v1
 from synapse.util.async import Linearizer
 from synapse.util.caches import CACHE_SIZE_FACTOR
 from synapse.util.caches.expiringcache import ExpiringCache
 from synapse.util.logutils import log_function
 from synapse.util.metrics import Measure
 
-from .v1 import resolve_events_with_factory, resolve_events_with_state_map
-
 logger = logging.getLogger(__name__)
 
 
@@ -263,8 +262,14 @@ def compute_event_context(self, event, old_state=None):
             defer.returnValue(context)
 
         logger.debug("calling resolve_state_groups from compute_event_context")
+        if event.type == EventTypes.Create:
+            room_version = event.content.get("room_version", RoomVersions.V1)
+        else:
+            room_version = None
+
         entry = yield self.resolve_state_groups_for_events(
             event.room_id, [e for e, _ in event.prev_events],
+            explicit_room_version=room_version,
         )
 
         prev_state_ids = entry.state
@@ -332,13 +337,17 @@ def compute_event_context(self, event, old_state=None):
         defer.returnValue(context)
 
     @defer.inlineCallbacks
-    def resolve_state_groups_for_events(self, room_id, event_ids):
+    def resolve_state_groups_for_events(self, room_id, event_ids,
+                                        explicit_room_version=None):
         """ Given a list of event_ids this method fetches the state at each
         event, resolves conflicts between them and returns them.
 
         Args:
-            room_id (str):
-            event_ids (list[str]):
+            room_id (str)
+            event_ids (list[str])
+            explicit_room_version (str|None): If set uses the the given room
+                version to choose the resolution algorithm. If None, then
+                checks the database for room version.
 
         Returns:
             Deferred[_StateCacheEntry]: resolved state
@@ -364,8 +373,13 @@ def resolve_state_groups_for_events(self, room_id, event_ids):
                 delta_ids=delta_ids,
             ))
 
+        room_version = explicit_room_version
+        if not room_version:
+            room_version = yield self.store.get_room_version(room_id)
+
         result = yield self._state_resolution_handler.resolve_state_groups(
-            room_id, state_groups_ids, None, self._state_map_factory,
+            room_id, room_version, state_groups_ids, None,
+            self._state_map_factory,
         )
         defer.returnValue(result)
 
@@ -374,7 +388,8 @@ def _state_map_factory(self, ev_ids):
             ev_ids, get_prev_content=False, check_redacted=False,
         )
 
-    def resolve_events(self, state_sets, event):
+    @defer.inlineCallbacks
+    def resolve_events(self, room_version, state_sets, event):
         logger.info(
             "Resolving state for %s with %d groups", event.room_id, len(state_sets)
         )
@@ -389,14 +404,18 @@ def resolve_events(self, state_sets, event):
             for ev in st
         }
 
+        room_version = yield self.store.get_room_version(event.room_id)
+
         with Measure(self.clock, "state._resolve_events"):
-            new_state = resolve_events_with_state_map(state_set_ids, state_map)
+            new_state = resolve_events_with_state_map(
+                room_version, state_set_ids, state_map,
+            )
 
         new_state = {
             key: state_map[ev_id] for key, ev_id in iteritems(new_state)
         }
 
-        return new_state
+        defer.returnValue(new_state)
 
 
 class StateResolutionHandler(object):
@@ -429,7 +448,7 @@ def start_caching(self):
     @defer.inlineCallbacks
     @log_function
     def resolve_state_groups(
-        self, room_id, state_groups_ids, event_map, state_map_factory,
+        self, room_id, room_version, state_groups_ids, event_map, state_map_factory,
     ):
         """Resolves conflicts between a set of state groups
 
@@ -438,6 +457,7 @@ def resolve_state_groups(
 
         Args:
             room_id (str): room we are resolving for (used for logging)
+            room_version (str): version of the room
             state_groups_ids (dict[int, dict[(str, str), str]]):
                  map from state group id to the state in that state group
                 (where 'state' is a map from state key to event id)
@@ -491,6 +511,7 @@ def resolve_state_groups(
                 logger.info("Resolving conflicted state for %r", room_id)
                 with Measure(self.clock, "state._resolve_events"):
                     new_state = yield resolve_events_with_factory(
+                        room_version,
                         list(itervalues(state_groups_ids)),
                         event_map=event_map,
                         state_map_factory=state_map_factory,
@@ -572,3 +593,64 @@ def _make_state_cache_entry(
         prev_group=prev_group,
         delta_ids=delta_ids,
     )
+
+
+def resolve_events_with_state_map(room_version, state_sets, state_map):
+    """
+    Args:
+        room_version(str): Version of the room
+        state_sets(list): List of dicts of (type, state_key) -> event_id,
+            which are the different state groups to resolve.
+        state_map(dict): a dict from event_id to event, for all events in
+            state_sets.
+
+    Returns
+        dict[(str, str), str]:
+            a map from (type, state_key) to event_id.
+    """
+    if room_version in (RoomVersions.V1, RoomVersions.VDH_TEST,):
+        return v1.resolve_events_with_state_map(
+            state_sets, state_map,
+        )
+    else:
+        # This should only happen if we added a version but forgot to add it to
+        # the list above.
+        raise Exception(
+            "No state resolution algorithm defined for version %r" % (room_version,)
+        )
+
+
+def resolve_events_with_factory(room_version, state_sets, event_map, state_map_factory):
+    """
+    Args:
+        room_version(str): Version of the room
+
+        state_sets(list): List of dicts of (type, state_key) -> event_id,
+            which are the different state groups to resolve.
+
+        event_map(dict[str,FrozenEvent]|None):
+            a dict from event_id to event, for any events that we happen to
+            have in flight (eg, those currently being persisted). This will be
+            used as a starting point fof finding the state we need; any missing
+            events will be requested via state_map_factory.
+
+            If None, all events will be fetched via state_map_factory.
+
+        state_map_factory(func): will be called
+            with a list of event_ids that are needed, and should return with
+            a Deferred of dict of event_id to event.
+
+    Returns
+        Deferred[dict[(str, str), str]]:
+            a map from (type, state_key) to event_id.
+    """
+    if room_version in (RoomVersions.V1, RoomVersions.VDH_TEST,):
+        return v1.resolve_events_with_factory(
+            state_sets, event_map, state_map_factory,
+        )
+    else:
+        # This should only happen if we added a version but forgot to add it to
+        # the list above.
+        raise Exception(
+            "No state resolution algorithm defined for version %r" % (room_version,)
+        )
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index ce32e8fefd23..dc68d365c236 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -705,9 +705,11 @@ def get_events(ev_ids):
         }
 
         events_map = {ev.event_id: ev for ev, _ in events_context}
+        room_version = yield self.get_room_version(room_id)
+
         logger.debug("calling resolve_state_groups from preserve_events")
         res = yield self._state_resolution_handler.resolve_state_groups(
-            room_id, state_groups, events_map, get_events
+            room_id, room_version, state_groups, events_map, get_events
         )
 
         defer.returnValue((res.state, None))

From bb99b1f5507e009bc32a82ba6ef29ba2d91e6b65 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 9 Aug 2018 14:44:10 +0100
Subject: [PATCH 325/830] Add fast path in state res for zero prev events

---
 synapse/state/__init__.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py
index 222daa0b28ae..f1c26d10719e 100644
--- a/synapse/state/__init__.py
+++ b/synapse/state/__init__.py
@@ -361,7 +361,12 @@ def resolve_state_groups_for_events(self, room_id, event_ids,
             room_id, event_ids
         )
 
-        if len(state_groups_ids) == 1:
+        if len(state_groups_ids) == 0:
+            defer.returnValue(_StateCacheEntry(
+                state={},
+                state_group=None,
+            ))
+        elif len(state_groups_ids) == 1:
             name, state_list = list(state_groups_ids.items()).pop()
 
             prev_group, delta_ids = yield self.store.get_state_group_delta(name)

From 3e19beb941f3f797262b051d47227018898bb36f Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 9 Aug 2018 14:33:49 +0100
Subject: [PATCH 326/830] Fix tests

---
 .../replication/slave/storage/test_events.py  |  3 +-
 tests/storage/test_redaction.py               |  4 ++-
 tests/storage/test_roommember.py              |  4 ++-
 tests/test_state.py                           |  6 +++-
 tests/test_visibility.py                      |  4 ++-
 tests/utils.py                                | 30 +++++++++++++++++++
 6 files changed, 46 insertions(+), 5 deletions(-)

diff --git a/tests/replication/slave/storage/test_events.py b/tests/replication/slave/storage/test_events.py
index f5b47f5ec0c2..8c08985fa952 100644
--- a/tests/replication/slave/storage/test_events.py
+++ b/tests/replication/slave/storage/test_events.py
@@ -122,6 +122,7 @@ def test_backfilled_redactions(self):
 
     @defer.inlineCallbacks
     def test_invites(self):
+        yield self.persist(type="m.room.create", key="", creator=USER_ID)
         yield self.check("get_invited_rooms_for_user", [USER_ID_2], [])
         event = yield self.persist(
             type="m.room.member", key=USER_ID_2, membership="invite"
@@ -134,7 +135,7 @@ def test_invites(self):
 
     @defer.inlineCallbacks
     def test_push_actions_for_user(self):
-        yield self.persist(type="m.room.create", creator=USER_ID)
+        yield self.persist(type="m.room.create", key="", creator=USER_ID)
         yield self.persist(type="m.room.join", key=USER_ID, membership="join")
         yield self.persist(
             type="m.room.join", sender=USER_ID, key=USER_ID_2, membership="join"
diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py
index 475ec900c4a2..05f40eaa9476 100644
--- a/tests/storage/test_redaction.py
+++ b/tests/storage/test_redaction.py
@@ -22,7 +22,7 @@
 from synapse.types import RoomID, UserID
 
 from tests import unittest
-from tests.utils import setup_test_homeserver
+from tests.utils import create_room, setup_test_homeserver
 
 
 class RedactionTestCase(unittest.TestCase):
@@ -43,6 +43,8 @@ def setUp(self):
 
         self.room1 = RoomID.from_string("!abc123:test")
 
+        yield create_room(hs, self.room1.to_string(), self.u_alice.to_string())
+
         self.depth = 1
 
     @defer.inlineCallbacks
diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py
index c5fd54f67e88..034ea3fbbf77 100644
--- a/tests/storage/test_roommember.py
+++ b/tests/storage/test_roommember.py
@@ -22,7 +22,7 @@
 from synapse.types import RoomID, UserID
 
 from tests import unittest
-from tests.utils import setup_test_homeserver
+from tests.utils import create_room, setup_test_homeserver
 
 
 class RoomMemberStoreTestCase(unittest.TestCase):
@@ -47,6 +47,8 @@ def setUp(self):
 
         self.room = RoomID.from_string("!abc123:test")
 
+        yield create_room(hs, self.room.to_string(), self.u_alice.to_string())
+
     @defer.inlineCallbacks
     def inject_room_member(self, room, user, membership, replaces_state=None):
         builder = self.event_builder_factory.new({
diff --git a/tests/test_state.py b/tests/test_state.py
index 429a18cbf75e..770e94437a5c 100644
--- a/tests/test_state.py
+++ b/tests/test_state.py
@@ -18,7 +18,7 @@
 from twisted.internet import defer
 
 from synapse.api.auth import Auth
-from synapse.api.constants import EventTypes, Membership
+from synapse.api.constants import EventTypes, Membership, RoomVersions
 from synapse.events import FrozenEvent
 from synapse.state import StateHandler, StateResolutionHandler
 
@@ -108,6 +108,9 @@ def register_event_context(self, event, context):
     def register_event_id_state_group(self, event_id, state_group):
         self._event_to_state_group[event_id] = state_group
 
+    def get_room_version(self, room_id):
+        return RoomVersions.V1
+
 
 class DictObj(dict):
     def __init__(self, **kwargs):
@@ -167,6 +170,7 @@ def test_branch_no_conflict(self):
                 "START": DictObj(
                     type=EventTypes.Create,
                     state_key="",
+                    content={},
                     depth=1,
                 ),
                 "A": DictObj(
diff --git a/tests/test_visibility.py b/tests/test_visibility.py
index 0dc1a924d331..15ebb0aa081d 100644
--- a/tests/test_visibility.py
+++ b/tests/test_visibility.py
@@ -21,7 +21,7 @@
 from synapse.visibility import filter_events_for_server
 
 import tests.unittest
-from tests.utils import setup_test_homeserver
+from tests.utils import create_room, setup_test_homeserver
 
 logger = logging.getLogger(__name__)
 
@@ -36,6 +36,8 @@ def setUp(self):
         self.event_builder_factory = self.hs.get_event_builder_factory()
         self.store = self.hs.get_datastore()
 
+        yield create_room(self.hs, TEST_ROOM_ID, "@someone:ROOM")
+
     @defer.inlineCallbacks
     def test_filtering(self):
         #
diff --git a/tests/utils.py b/tests/utils.py
index 3f173049346f..8930a3a23056 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -21,6 +21,7 @@
 
 from twisted.internet import defer, reactor
 
+from synapse.api.constants import EventTypes
 from synapse.api.errors import CodeMessageException, cs_error
 from synapse.federation.transport import server
 from synapse.http.server import HttpServer
@@ -445,3 +446,32 @@ def assert_had_no_calls(self):
                     "call(%s)" % _format_call(c[0], c[1]) for c in calls
                 ])
             )
+
+
+@defer.inlineCallbacks
+def create_room(hs, room_id, creator_id):
+    """Creates and persist a creation event for the given room
+
+    Args:
+        hs
+        room_id (str)
+        creator_id (str)
+    """
+
+    store = hs.get_datastore()
+    event_builder_factory = hs.get_event_builder_factory()
+    event_creation_handler = hs.get_event_creation_handler()
+
+    builder = event_builder_factory.new({
+        "type": EventTypes.Create,
+        "state_key": "",
+        "sender": creator_id,
+        "room_id": room_id,
+        "content": {},
+    })
+
+    event, context = yield event_creation_handler.create_new_client_event(
+        builder
+    )
+
+    yield store.persist_event(event, context)

From 5075e444f413b0b564dbc525a641cb5784acf2ad Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 9 Aug 2018 14:56:25 +0100
Subject: [PATCH 327/830] Newsfile

---
 changelog.d/3673.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3673.misc

diff --git a/changelog.d/3673.misc b/changelog.d/3673.misc
new file mode 100644
index 000000000000..d672111fb9ae
--- /dev/null
+++ b/changelog.d/3673.misc
@@ -0,0 +1 @@
+Refactor state module to support multiple room versions

From 09cf13089858902f3cdcb49b9f9bc3d214ba6337 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Thu, 9 Aug 2018 17:39:12 +0100
Subject: [PATCH 328/830] only block on sync where user is not part of the mau
 cohort

---
 synapse/api/auth.py         | 13 ++++++++++--
 synapse/handlers/sync.py    |  7 ++++++-
 tests/handlers/test_sync.py | 40 ++++++++++++++++++++++++++++---------
 3 files changed, 48 insertions(+), 12 deletions(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 9c62ec43742d..170039fc8262 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -775,17 +775,26 @@ def check_in_room_or_world_readable(self, room_id, user_id):
             )
 
     @defer.inlineCallbacks
-    def check_auth_blocking(self):
+    def check_auth_blocking(self, user_id=None):
         """Checks if the user should be rejected for some external reason,
         such as monthly active user limiting or global disable flag
+
+        Args:
+            user_id(str): If present, checks for presence against existing MAU cohort
         """
         if self.hs.config.hs_disabled:
             raise AuthError(
                 403, self.hs.config.hs_disabled_message, errcode=Codes.HS_DISABLED
             )
         if self.hs.config.limit_usage_by_mau is True:
+            # If the user is already part of the MAU cohort
+            if user_id:
+                timestamp = yield self.store._user_last_seen_monthly_active(user_id)
+                if timestamp:
+                    return
+            # Else if there is no room in the MAU bucket, bail
             current_mau = yield self.store.get_monthly_active_count()
             if current_mau >= self.hs.config.max_mau_value:
                 raise AuthError(
                     403, "MAU Limit Exceeded", errcode=Codes.MAU_LIMIT_EXCEEDED
-                )
+            )
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 776ddca638c8..d3b26a4106b2 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -208,7 +208,12 @@ def wait_for_sync_for_user(self, sync_config, since_token=None, timeout=0,
         Returns:
             Deferred[SyncResult]
         """
-        yield self.auth.check_auth_blocking()
+        # If the user is not part of the mau group, then check that limits have
+        # not been exceeded (if not part of the group by this point, almost certain
+        # auth_blocking will occur)
+        user_id = sync_config.user.to_string()
+        yield self.auth.check_auth_blocking(user_id)
+
         res = yield self.response_cache.wrap(
             sync_config.request_key,
             self._wait_for_sync_for_user,
diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py
index 497e4bd933a0..b95a8743a71c 100644
--- a/tests/handlers/test_sync.py
+++ b/tests/handlers/test_sync.py
@@ -13,8 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 from twisted.internet import defer
+from synapse.api.errors import AuthError, Codes
 
-from synapse.api.errors import AuthError
 from synapse.api.filtering import DEFAULT_FILTER_COLLECTION
 from synapse.handlers.sync import SyncConfig, SyncHandler
 from synapse.types import UserID
@@ -31,19 +31,41 @@ class SyncTestCase(tests.unittest.TestCase):
     def setUp(self):
         self.hs = yield setup_test_homeserver()
         self.sync_handler = SyncHandler(self.hs)
+        self.store = self.hs.get_datastore()
 
     @defer.inlineCallbacks
     def test_wait_for_sync_for_user_auth_blocking(self):
-        sync_config = SyncConfig(
-            user=UserID("@user", "server"),
+
+        user_id1 = "@user1:server"
+        user_id2 = "@user2:server"
+        sync_config = self._generate_sync_config(user_id1)
+
+        self.hs.config.limit_usage_by_mau = True
+        self.hs.config.max_mau_value = 1
+
+        # Check that the happy case does not throw errors
+        yield self.store.upsert_monthly_active_user(user_id1)
+        yield self.sync_handler.wait_for_sync_for_user(sync_config)
+
+        # Test that global lock works
+        self.hs.config.hs_disabled = True
+        with self.assertRaises(AuthError) as e:
+            yield self.sync_handler.wait_for_sync_for_user(sync_config)
+        self.assertEquals(e.exception.errcode, Codes.HS_DISABLED)
+
+        self.hs.config.hs_disabled = False
+
+        sync_config = self._generate_sync_config(user_id2)
+
+        with self.assertRaises(AuthError) as e:
+            yield self.sync_handler.wait_for_sync_for_user(sync_config)
+        self.assertEquals(e.exception.errcode, Codes.MAU_LIMIT_EXCEEDED)
+
+    def _generate_sync_config(self, user_id):
+        return SyncConfig(
+            user=UserID(user_id.split(":")[0][1:], user_id.split(":")[1]),
             filter_collection=DEFAULT_FILTER_COLLECTION,
             is_guest=False,
             request_key="request_key",
             device_id="device_id",
         )
-        # Ensure that an exception is not thrown
-        yield self.sync_handler.wait_for_sync_for_user(sync_config)
-        self.hs.config.hs_disabled = True
-
-        with self.assertRaises(AuthError):
-            yield self.sync_handler.wait_for_sync_for_user(sync_config)

From 04df7142598b531e8e400611e3f92b21afeabab6 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Thu, 9 Aug 2018 17:41:52 +0100
Subject: [PATCH 329/830] fix imports

---
 tests/handlers/test_sync.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py
index b95a8743a71c..cfd37f3138d3 100644
--- a/tests/handlers/test_sync.py
+++ b/tests/handlers/test_sync.py
@@ -13,8 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 from twisted.internet import defer
-from synapse.api.errors import AuthError, Codes
 
+from synapse.api.errors import AuthError, Codes
 from synapse.api.filtering import DEFAULT_FILTER_COLLECTION
 from synapse.handlers.sync import SyncConfig, SyncHandler
 from synapse.types import UserID

From c1f9dec92ac8abaa693cc591438087d8282c6844 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Thu, 9 Aug 2018 17:43:26 +0100
Subject: [PATCH 330/830] fix errant parenthesis

---
 synapse/api/auth.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 170039fc8262..df6022ff6957 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -797,4 +797,4 @@ def check_auth_blocking(self, user_id=None):
             if current_mau >= self.hs.config.max_mau_value:
                 raise AuthError(
                     403, "MAU Limit Exceeded", errcode=Codes.MAU_LIMIT_EXCEEDED
-            )
+                )

From 885ea9c602ca4002385a6d73c94c35e0958de809 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Thu, 9 Aug 2018 18:02:12 +0100
Subject: [PATCH 331/830] rename _user_last_seen_monthly_active

---
 synapse/api/auth.py                        |  2 +-
 synapse/storage/monthly_active_users.py    | 10 +++++-----
 tests/storage/test_client_ips.py           | 12 ++++++------
 tests/storage/test_monthly_active_users.py | 13 +++++++------
 4 files changed, 19 insertions(+), 18 deletions(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index df6022ff6957..c31c6a6a0867 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -789,7 +789,7 @@ def check_auth_blocking(self, user_id=None):
         if self.hs.config.limit_usage_by_mau is True:
             # If the user is already part of the MAU cohort
             if user_id:
-                timestamp = yield self.store._user_last_seen_monthly_active(user_id)
+                timestamp = yield self.store.user_last_seen_monthly_active(user_id)
                 if timestamp:
                     return
             # Else if there is no room in the MAU bucket, bail
diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py
index d47dcef3a0a5..07211432af79 100644
--- a/synapse/storage/monthly_active_users.py
+++ b/synapse/storage/monthly_active_users.py
@@ -113,7 +113,7 @@ def _reap_users(txn):
         # is racy.
         # Have resolved to invalidate the whole cache for now and do
         # something about it if and when the perf becomes significant
-        self._user_last_seen_monthly_active.invalidate_all()
+        self.user_last_seen_monthly_active.invalidate_all()
         self.get_monthly_active_count.invalidate_all()
 
     @cached(num_args=0)
@@ -152,11 +152,11 @@ def upsert_monthly_active_user(self, user_id):
             lock=False,
         )
         if is_insert:
-            self._user_last_seen_monthly_active.invalidate((user_id,))
+            self.user_last_seen_monthly_active.invalidate((user_id,))
             self.get_monthly_active_count.invalidate(())
 
     @cached(num_args=1)
-    def _user_last_seen_monthly_active(self, user_id):
+    def user_last_seen_monthly_active(self, user_id):
         """
             Checks if a given user is part of the monthly active user group
             Arguments:
@@ -173,7 +173,7 @@ def _user_last_seen_monthly_active(self, user_id):
             },
             retcol="timestamp",
             allow_none=True,
-            desc="_user_last_seen_monthly_active",
+            desc="user_last_seen_monthly_active",
         ))
 
     @defer.inlineCallbacks
@@ -185,7 +185,7 @@ def populate_monthly_active_users(self, user_id):
             user_id(str): the user_id to query
         """
         if self.hs.config.limit_usage_by_mau:
-            last_seen_timestamp = yield self._user_last_seen_monthly_active(user_id)
+            last_seen_timestamp = yield self.user_last_seen_monthly_active(user_id)
             now = self.hs.get_clock().time_msec()
 
             # We want to reduce to the total number of db writes, and are happy
diff --git a/tests/storage/test_client_ips.py b/tests/storage/test_client_ips.py
index 7a58c6eb24e5..6d2bb3058ea2 100644
--- a/tests/storage/test_client_ips.py
+++ b/tests/storage/test_client_ips.py
@@ -64,7 +64,7 @@ def test_disabled_monthly_active_user(self):
         yield self.store.insert_client_ip(
             user_id, "access_token", "ip", "user_agent", "device_id",
         )
-        active = yield self.store._user_last_seen_monthly_active(user_id)
+        active = yield self.store.user_last_seen_monthly_active(user_id)
         self.assertFalse(active)
 
     @defer.inlineCallbacks
@@ -80,7 +80,7 @@ def test_adding_monthly_active_user_when_full(self):
         yield self.store.insert_client_ip(
             user_id, "access_token", "ip", "user_agent", "device_id",
         )
-        active = yield self.store._user_last_seen_monthly_active(user_id)
+        active = yield self.store.user_last_seen_monthly_active(user_id)
         self.assertFalse(active)
 
     @defer.inlineCallbacks
@@ -88,13 +88,13 @@ def test_adding_monthly_active_user_when_space(self):
         self.hs.config.limit_usage_by_mau = True
         self.hs.config.max_mau_value = 50
         user_id = "@user:server"
-        active = yield self.store._user_last_seen_monthly_active(user_id)
+        active = yield self.store.user_last_seen_monthly_active(user_id)
         self.assertFalse(active)
 
         yield self.store.insert_client_ip(
             user_id, "access_token", "ip", "user_agent", "device_id",
         )
-        active = yield self.store._user_last_seen_monthly_active(user_id)
+        active = yield self.store.user_last_seen_monthly_active(user_id)
         self.assertTrue(active)
 
     @defer.inlineCallbacks
@@ -103,7 +103,7 @@ def test_updating_monthly_active_user_when_space(self):
         self.hs.config.max_mau_value = 50
         user_id = "@user:server"
 
-        active = yield self.store._user_last_seen_monthly_active(user_id)
+        active = yield self.store.user_last_seen_monthly_active(user_id)
         self.assertFalse(active)
 
         yield self.store.insert_client_ip(
@@ -112,5 +112,5 @@ def test_updating_monthly_active_user_when_space(self):
         yield self.store.insert_client_ip(
             user_id, "access_token", "ip", "user_agent", "device_id",
         )
-        active = yield self.store._user_last_seen_monthly_active(user_id)
+        active = yield self.store.user_last_seen_monthly_active(user_id)
         self.assertTrue(active)
diff --git a/tests/storage/test_monthly_active_users.py b/tests/storage/test_monthly_active_users.py
index cbd480cd4215..be74c3021812 100644
--- a/tests/storage/test_monthly_active_users.py
+++ b/tests/storage/test_monthly_active_users.py
@@ -66,9 +66,9 @@ def test_initialise_reserved_users(self):
 
         # Test user is marked as active
 
-        timestamp = yield self.store._user_last_seen_monthly_active(user1)
+        timestamp = yield self.store.user_last_seen_monthly_active(user1)
         self.assertTrue(timestamp)
-        timestamp = yield self.store._user_last_seen_monthly_active(user2)
+        timestamp = yield self.store.user_last_seen_monthly_active(user2)
         self.assertTrue(timestamp)
 
         # Test that users are never removed from the db.
@@ -92,17 +92,18 @@ def test_can_insert_and_count_mau(self):
         self.assertEqual(1, count)
 
     @defer.inlineCallbacks
-    def test__user_last_seen_monthly_active(self):
+    def test_user_last_seen_monthly_active(self):
         user_id1 = "@user1:server"
         user_id2 = "@user2:server"
         user_id3 = "@user3:server"
-        result = yield self.store._user_last_seen_monthly_active(user_id1)
+
+        result = yield self.store.user_last_seen_monthly_active(user_id1)
         self.assertFalse(result == 0)
         yield self.store.upsert_monthly_active_user(user_id1)
         yield self.store.upsert_monthly_active_user(user_id2)
-        result = yield self.store._user_last_seen_monthly_active(user_id1)
+        result = yield self.store.user_last_seen_monthly_active(user_id1)
         self.assertTrue(result > 0)
-        result = yield self.store._user_last_seen_monthly_active(user_id3)
+        result = yield self.store.user_last_seen_monthly_active(user_id3)
         self.assertFalse(result == 0)
 
     @defer.inlineCallbacks

From 64899341dc8988b40b86dd49034aae27ddcb8c44 Mon Sep 17 00:00:00 2001
From: Jeroen <vo.jeroen@gmail.com>
Date: Thu, 9 Aug 2018 21:04:22 +0200
Subject: [PATCH 332/830] include private functions from twisted

---
 synapse/crypto/context_factory.py | 37 +++++++++++++++++++++++++++++--
 1 file changed, 35 insertions(+), 2 deletions(-)

diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py
index 29a75e187374..08c41a92b672 100644
--- a/synapse/crypto/context_factory.py
+++ b/synapse/crypto/context_factory.py
@@ -16,10 +16,10 @@
 from zope.interface import implementer
 
 from OpenSSL import SSL, crypto
-from twisted.internet._idna import _idnaBytes
-from twisted.internet._sslverify import _defaultCurveName, _tolerateErrors
+from twisted.internet._sslverify import _defaultCurveName
 from twisted.internet.interfaces import IOpenSSLClientConnectionCreator
 from twisted.internet.ssl import CertificateOptions, ContextFactory
+from twisted.python.failure import Failure
 
 logger = logging.getLogger(__name__)
 
@@ -53,6 +53,39 @@ def getContext(self):
         return self._context
 
 
+def _idnaBytes(text):
+    """
+    Convert some text typed by a human into some ASCII bytes. This is a
+    copy of twisted.internet._idna._idnaBytes. For documentation, see the
+    twisted documentation.
+    """
+    try:
+        import idna
+    except ImportError:
+        return text.encode("idna")
+    else:
+        return idna.encode(text)
+
+
+def _tolerateErrors(wrapped):
+    """
+    Wrap up an info_callback for pyOpenSSL so that if something goes wrong
+    the error is immediately logged and the connection is dropped if possible.
+    This is a copy of twisted.internet._sslverify._tolerateErrors. For
+    documentation, see the twisted documentation.
+    """
+
+    def infoCallback(connection, where, ret):
+        try:
+            return wrapped(connection, where, ret)
+        except:  # noqa: E722, taken from the twisted implementation
+            f = Failure()
+            logger.exception("Error during info_callback")
+            connection.get_app_data().failVerification(f)
+
+    return infoCallback
+
+
 @implementer(IOpenSSLClientConnectionCreator)
 class ClientTLSOptions(object):
     """

From 2e9c73e8cab5af901957f5809f3be415d6fc0908 Mon Sep 17 00:00:00 2001
From: Jeroen <vo.jeroen@gmail.com>
Date: Thu, 9 Aug 2018 21:31:26 +0200
Subject: [PATCH 333/830] more generic conversion of str/bytes to unicode

---
 synapse/crypto/context_factory.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py
index 08c41a92b672..1a391adec198 100644
--- a/synapse/crypto/context_factory.py
+++ b/synapse/crypto/context_factory.py
@@ -123,6 +123,6 @@ def __init__(self, config):
 
     def get_options(self, host):
         return ClientTLSOptions(
-            unicode(host),
+            host.decode('utf-8'),
             CertificateOptions(verify=False).getContext()
         )

From 01021c812f86466a5f35db8afd443b9cb17bd092 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Thu, 9 Aug 2018 22:16:00 +0100
Subject: [PATCH 334/830] wip at implementing MSC 7075

---
 .../resource_limits_server_notices.py         | 84 +++++++++++++++++++
 1 file changed, 84 insertions(+)
 create mode 100644 synapse/server_notices/resource_limits_server_notices.py

diff --git a/synapse/server_notices/resource_limits_server_notices.py b/synapse/server_notices/resource_limits_server_notices.py
new file mode 100644
index 000000000000..33b1d80cd9df
--- /dev/null
+++ b/synapse/server_notices/resource_limits_server_notices.py
@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+
+from six import iteritems, string_types
+
+from twisted.internet import defer
+
+from synapse.api.errors import SynapseError
+from synapse.api.urls import ConsentURIBuilder
+from synapse.config import ConfigError
+from synapse.types import get_localpart_from_id
+
+logger = logging.getLogger(__name__)
+
+
+class ResourceLimitsServerNotices(object):
+    """
+    """
+    def __init__(self, hs):
+        """
+
+        Args:
+            hs (synapse.server.HomeServer):
+        """
+        self._server_notices_manager = hs.get_server_notices_manager()
+        self._store = hs.get_datastore()
+        self._api = hs.get_api()
+        self._server_notice_content = hs.config.user_consent_server_notice_content
+        self._limit_usage_by_mau = config.limit_usage_by_mau = False
+        self._hs_disabled.config.hs_disabled = False
+
+        self._notified = set()
+        self._resouce_limited = False
+        # Config checks?
+
+    @defer.inlineCallbacks
+    def maybe_send_server_notice_to_user(self, user_id):
+        """Check if we need to send a notice to this user, and does so if so
+
+        Args:
+            user_id (str): user to check
+
+        Returns:
+            Deferred
+        """
+        if self._limit_usage_by_mau is False and self._hs_disabled is False:
+            # not enabled
+            return
+
+        timestamp = yield self.store.user_last_seen_monthly_active(user_id)
+        if timestamp is None:
+            # This user will be blocked from receiving the notice anyway
+            return
+        try:
+            yield self.api.check_auth_blocking()
+            if self._resouce_limited:
+                # Need to start removing notices
+                pass
+        except AuthError as e:
+            # Need to start notifying of blocking
+            if not self._resouce_limited:
+                pass
+
+            # need to send a message.
+            try:
+                yield self._server_notices_manager.send_notice(
+                    user_id, content,
+                )
+
+            except SynapseError as e:
+                logger.error("Error sending server notice about resource limits: %s", e)

From 638d35ef082cb7f24d43fcb585c8bae52645bd6b Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 10 Aug 2018 10:59:09 +0100
Subject: [PATCH 335/830] Fix linearizer cancellation on twisted < 18.7

Turns out that cancellation of inlineDeferreds didn't really work properly
until Twisted 18.7. This commit refactors Linearizer.queue to avoid
inlineCallbacks.
---
 synapse/util/async.py | 111 ++++++++++++++++++++++++++----------------
 1 file changed, 68 insertions(+), 43 deletions(-)

diff --git a/synapse/util/async.py b/synapse/util/async.py
index a7094e2fb463..9b3f2f4b96df 100644
--- a/synapse/util/async.py
+++ b/synapse/util/async.py
@@ -188,62 +188,30 @@ def __init__(self, name=None, max_count=1, clock=None):
         # things blocked from executing.
         self.key_to_defer = {}
 
-    @defer.inlineCallbacks
     def queue(self, key):
+        # we avoid doing defer.inlineCallbacks here, so that cancellation works correctly.
+        # (https://twistedmatrix.com/trac/ticket/4632 meant that cancellations were not
+        # propagated inside inlineCallbacks until Twisted 18.7)
         entry = self.key_to_defer.setdefault(key, [0, collections.OrderedDict()])
 
         # If the number of things executing is greater than the maximum
         # then add a deferred to the list of blocked items
-        # When on of the things currently executing finishes it will callback
+        # When one of the things currently executing finishes it will callback
         # this item so that it can continue executing.
         if entry[0] >= self.max_count:
-            new_defer = defer.Deferred()
-            entry[1][new_defer] = 1
-
-            logger.info(
-                "Waiting to acquire linearizer lock %r for key %r", self.name, key,
-            )
-            try:
-                yield make_deferred_yieldable(new_defer)
-            except Exception as e:
-                if isinstance(e, CancelledError):
-                    logger.info(
-                        "Cancelling wait for linearizer lock %r for key %r",
-                        self.name, key,
-                    )
-                else:
-                    logger.warn(
-                        "Unexpected exception waiting for linearizer lock %r for key %r",
-                        self.name, key,
-                    )
-
-                # we just have to take ourselves back out of the queue.
-                del entry[1][new_defer]
-                raise
-
-            logger.info("Acquired linearizer lock %r for key %r", self.name, key)
-            entry[0] += 1
-
-            # if the code holding the lock completes synchronously, then it
-            # will recursively run the next claimant on the list. That can
-            # relatively rapidly lead to stack exhaustion. This is essentially
-            # the same problem as http://twistedmatrix.com/trac/ticket/9304.
-            #
-            # In order to break the cycle, we add a cheeky sleep(0) here to
-            # ensure that we fall back to the reactor between each iteration.
-            #
-            # (This needs to happen while we hold the lock, and the context manager's exit
-            # code must be synchronous, so this is the only sensible place.)
-            yield self._clock.sleep(0)
-
+            res = self._await_lock(key)
         else:
             logger.info(
                 "Acquired uncontended linearizer lock %r for key %r", self.name, key,
             )
             entry[0] += 1
+            res = defer.succeed(None)
+
+        # once we successfully get the lock, we need to return a context manager which
+        # will release the lock.
 
         @contextmanager
-        def _ctx_manager():
+        def _ctx_manager(_):
             try:
                 yield
             finally:
@@ -264,7 +232,64 @@ def _ctx_manager():
                     # map.
                     del self.key_to_defer[key]
 
-        defer.returnValue(_ctx_manager())
+        res.addCallback(_ctx_manager)
+        return res
+
+    def _await_lock(self, key):
+        """Helper for queue: adds a deferred to the queue
+
+        Assumes that we've already checked that we've reached the limit of the number
+        of lock-holders we allow. Creates a new deferred which is added to the list, and
+        adds some management around cancellations.
+
+        Returns the deferred, which will callback once we have secured the lock.
+
+        """
+        entry = self.key_to_defer[key]
+
+        logger.info(
+            "Waiting to acquire linearizer lock %r for key %r", self.name, key,
+        )
+
+        new_defer = make_deferred_yieldable(defer.Deferred())
+        entry[1][new_defer] = 1
+
+        def cb(_r):
+            logger.info("Acquired linearizer lock %r for key %r", self.name, key)
+            entry[0] += 1
+
+            # if the code holding the lock completes synchronously, then it
+            # will recursively run the next claimant on the list. That can
+            # relatively rapidly lead to stack exhaustion. This is essentially
+            # the same problem as http://twistedmatrix.com/trac/ticket/9304.
+            #
+            # In order to break the cycle, we add a cheeky sleep(0) here to
+            # ensure that we fall back to the reactor between each iteration.
+            #
+            # (This needs to happen while we hold the lock, and the context manager's exit
+            # code must be synchronous, so this is the only sensible place.)
+            return self._clock.sleep(0)
+
+        def eb(e):
+            logger.info("defer %r got err %r", new_defer, e)
+            if isinstance(e, CancelledError):
+                logger.info(
+                    "Cancelling wait for linearizer lock %r for key %r",
+                    self.name, key,
+                )
+
+            else:
+                logger.warn(
+                    "Unexpected exception waiting for linearizer lock %r for key %r",
+                    self.name, key,
+                )
+
+            # we just have to take ourselves back out of the queue.
+            del entry[1][new_defer]
+            return e
+
+        new_defer.addCallbacks(cb, eb)
+        return new_defer
 
 
 class ReadWriteLock(object):

From 178ab76ac0e804ad519800f270e121b241139246 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 10 Aug 2018 11:05:23 +0100
Subject: [PATCH 336/830] changelog

---
 changelog.d/3676.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3676.bugfix

diff --git a/changelog.d/3676.bugfix b/changelog.d/3676.bugfix
new file mode 100644
index 000000000000..7b23a2773a55
--- /dev/null
+++ b/changelog.d/3676.bugfix
@@ -0,0 +1 @@
+Make the tests pass on Twisted < 18.7.0

From c75b71a397d91c013a65a7f57d2c13199cf4c204 Mon Sep 17 00:00:00 2001
From: Andrej Shadura <andrew.shadura@collabora.co.uk>
Date: Fri, 10 Aug 2018 13:22:39 +0200
Subject: [PATCH 337/830] Use recaptcha_ajax.js directly from Google

The script recaptcha_ajax.js contains minified non-free code which
we probably cannot redistribute. Since it talks to Google servers
anyway, it is better to just download it from Google directly instead
of shipping it.

This fixes #1932.
---
 changelog.d/1932.bugfix                       |   1 +
 synapse/static/client/register/index.html     |   2 +-
 .../client/register/js/recaptcha_ajax.js      | 195 ------------------
 3 files changed, 2 insertions(+), 196 deletions(-)
 create mode 100644 changelog.d/1932.bugfix
 delete mode 100644 synapse/static/client/register/js/recaptcha_ajax.js

diff --git a/changelog.d/1932.bugfix b/changelog.d/1932.bugfix
new file mode 100644
index 000000000000..caa551627beb
--- /dev/null
+++ b/changelog.d/1932.bugfix
@@ -0,0 +1 @@
+Don’t ship recaptcha_ajax.js, use it directly from Google
diff --git a/synapse/static/client/register/index.html b/synapse/static/client/register/index.html
index 600b3ee41eab..886f2edd1fad 100644
--- a/synapse/static/client/register/index.html
+++ b/synapse/static/client/register/index.html
@@ -4,7 +4,7 @@
 <meta name='viewport' content='width=device-width, initial-scale=1, user-scalable=no, minimum-scale=1.0, maximum-scale=1.0'> 
 <link rel="stylesheet" href="style.css">
 <script src="js/jquery-2.1.3.min.js"></script>
-<script src="js/recaptcha_ajax.js"></script>
+<script src="https://www.google.com/recaptcha/api/js/recaptcha_ajax.js"></script>
 <script src="register_config.js"></script>
 <script src="js/register.js"></script>
 </head>
diff --git a/synapse/static/client/register/js/recaptcha_ajax.js b/synapse/static/client/register/js/recaptcha_ajax.js
deleted file mode 100644
index d0e71e5b8824..000000000000
--- a/synapse/static/client/register/js/recaptcha_ajax.js
+++ /dev/null
@@ -1,195 +0,0 @@
-(function(){var h,k=this,l=function(a){return void 0!==a},ba=function(){},n=function(a){var b=typeof a;if("object"==b)if(a){if(a instanceof Array)return"array";if(a instanceof Object)return b;var c=Object.prototype.toString.call(a);if("[object Window]"==c)return"object";if("[object Array]"==c||"number"==typeof a.length&&"undefined"!=typeof a.splice&&"undefined"!=typeof a.propertyIsEnumerable&&!a.propertyIsEnumerable("splice"))return"array";if("[object Function]"==c||"undefined"!=typeof a.call&&"undefined"!=typeof a.propertyIsEnumerable&&
-!a.propertyIsEnumerable("call"))return"function"}else return"null";else if("function"==b&&"undefined"==typeof a.call)return"object";return b},p=function(a){return"array"==n(a)},ca=function(a){var b=n(a);return"array"==b||"object"==b&&"number"==typeof a.length},q=function(a){return"string"==typeof a},r=function(a){return"function"==n(a)},da=function(a){var b=typeof a;return"object"==b&&null!=a||"function"==b},ea=function(a,b,c){return a.call.apply(a.bind,arguments)},fa=function(a,b,c){if(!a)throw Error();
-if(2<arguments.length){var d=Array.prototype.slice.call(arguments,2);return function(){var c=Array.prototype.slice.call(arguments);Array.prototype.unshift.apply(c,d);return a.apply(b,c)}}return function(){return a.apply(b,arguments)}},s=function(a,b,c){s=Function.prototype.bind&&-1!=Function.prototype.bind.toString().indexOf("native code")?ea:fa;return s.apply(null,arguments)},ga=function(a,b){var c=Array.prototype.slice.call(arguments,1);return function(){var b=c.slice();b.push.apply(b,arguments);
-return a.apply(this,b)}},ha=Date.now||function(){return+new Date},ia=null,t=function(a,b){var c=a.split("."),d=k;c[0]in d||!d.execScript||d.execScript("var "+c[0]);for(var e;c.length&&(e=c.shift());)!c.length&&l(b)?d[e]=b:d=d[e]?d[e]:d[e]={}},u=function(a,b){function c(){}c.prototype=b.prototype;a.superClass_=b.prototype;a.prototype=new c;a.base=function(a,c,g){return b.prototype[c].apply(a,Array.prototype.slice.call(arguments,2))}};
-Function.prototype.bind=Function.prototype.bind||function(a,b){if(1<arguments.length){var c=Array.prototype.slice.call(arguments,1);c.unshift(this,a);return s.apply(null,c)}return s(this,a)};var v={};t("RecaptchaTemplates",v);v.VertHtml='<table id="recaptcha_table" class="recaptchatable" > <tr> <td colspan="6" class=\'recaptcha_r1_c1\'></td> </tr> <tr> <td class=\'recaptcha_r2_c1\'></td> <td colspan="4" class=\'recaptcha_image_cell\'><center><div id="recaptcha_image"></div></center></td> <td class=\'recaptcha_r2_c2\'></td> </tr> <tr> <td rowspan="6" class=\'recaptcha_r3_c1\'></td> <td colspan="4" class=\'recaptcha_r3_c2\'></td> <td rowspan="6" class=\'recaptcha_r3_c3\'></td> </tr> <tr> <td rowspan="3" class=\'recaptcha_r4_c1\' height="49"> <div class="recaptcha_input_area"> <input name="recaptcha_response_field" id="recaptcha_response_field" type="text" autocorrect="off" autocapitalize="off" placeholder="" /> <span id="recaptcha_privacy" class="recaptcha_only_if_privacy"></span> </div> </td> <td rowspan="4" class=\'recaptcha_r4_c2\'></td> <td><a id=\'recaptcha_reload_btn\'><img id=\'recaptcha_reload\' width="25" height="17" /></a></td> <td rowspan="4" class=\'recaptcha_r4_c4\'></td> </tr> <tr> <td><a id=\'recaptcha_switch_audio_btn\' class="recaptcha_only_if_image"><img id=\'recaptcha_switch_audio\' width="25" height="16" alt="" /></a><a id=\'recaptcha_switch_img_btn\' class="recaptcha_only_if_audio"><img id=\'recaptcha_switch_img\' width="25" height="16" alt=""/></a></td> </tr> <tr> <td><a id=\'recaptcha_whatsthis_btn\'><img id=\'recaptcha_whatsthis\' width="25" height="16" /></a></td> </tr> <tr> <td class=\'recaptcha_r7_c1\'></td> <td class=\'recaptcha_r8_c1\'></td> </tr> </table> ';v.CleanCss=".recaptchatable td img{display:block}.recaptchatable .recaptcha_image_cell center img{height:57px}.recaptchatable .recaptcha_image_cell center{height:57px}.recaptchatable .recaptcha_image_cell{background-color:white;height:57px;padding:7px!important}.recaptchatable,#recaptcha_area tr,#recaptcha_area td,#recaptcha_area th{margin:0!important;border:0!important;border-collapse:collapse!important;vertical-align:middle!important}.recaptchatable *{margin:0;padding:0;border:0;color:black;position:static;top:auto;left:auto;right:auto;bottom:auto}.recaptchatable #recaptcha_image{position:relative;margin:auto;border:1px solid #dfdfdf!important}.recaptchatable #recaptcha_image #recaptcha_challenge_image{display:block}.recaptchatable #recaptcha_image #recaptcha_ad_image{display:block;position:absolute;top:0}.recaptchatable a img{border:0}.recaptchatable a,.recaptchatable a:hover{cursor:pointer;outline:none;border:0!important;padding:0!important;text-decoration:none;color:blue;background:none!important;font-weight:normal}.recaptcha_input_area{position:relative!important;background:none!important}.recaptchatable label.recaptcha_input_area_text{border:1px solid #dfdfdf!important;margin:0!important;padding:0!important;position:static!important;top:auto!important;left:auto!important;right:auto!important;bottom:auto!important}.recaptcha_theme_red label.recaptcha_input_area_text,.recaptcha_theme_white label.recaptcha_input_area_text{color:black!important}.recaptcha_theme_blackglass label.recaptcha_input_area_text{color:white!important}.recaptchatable #recaptcha_response_field{font-size:11pt}.recaptcha_theme_blackglass #recaptcha_response_field,.recaptcha_theme_white #recaptcha_response_field{border:1px solid gray}.recaptcha_theme_red #recaptcha_response_field{border:1px solid #cca940}.recaptcha_audio_cant_hear_link{font-size:7pt;color:black}.recaptchatable{line-height:1em;border:1px solid #dfdfdf!important}.recaptcha_error_text{color:red}.recaptcha_only_if_privacy{float:right;text-align:right;margin-right:7px}#recaptcha-ad-choices{position:absolute;height:15px;top:0;right:0}#recaptcha-ad-choices img{height:15px}.recaptcha-ad-choices-collapsed{width:30px;height:15px;display:block}.recaptcha-ad-choices-expanded{width:75px;height:15px;display:none}#recaptcha-ad-choices:hover .recaptcha-ad-choices-collapsed{display:none}#recaptcha-ad-choices:hover .recaptcha-ad-choices-expanded{display:block}";v.CleanHtml='<table id="recaptcha_table" class="recaptchatable"> <tr height="73"> <td class=\'recaptcha_image_cell\' width="302"><center><div id="recaptcha_image"></div></center></td> <td style="padding: 10px 7px 7px 7px;"> <a id=\'recaptcha_reload_btn\'><img id=\'recaptcha_reload\' width="25" height="18" alt="" /></a> <a id=\'recaptcha_switch_audio_btn\' class="recaptcha_only_if_image"><img id=\'recaptcha_switch_audio\' width="25" height="15" alt="" /></a><a id=\'recaptcha_switch_img_btn\' class="recaptcha_only_if_audio"><img id=\'recaptcha_switch_img\' width="25" height="15" alt=""/></a> <a id=\'recaptcha_whatsthis_btn\'><img id=\'recaptcha_whatsthis\' width="25" height="16" /></a> </td> <td style="padding: 18px 7px 18px 7px;"> <img id=\'recaptcha_logo\' alt="" width="71" height="36" /> </td> </tr> <tr> <td style="padding-left: 7px;"> <div class="recaptcha_input_area" style="padding-top: 2px; padding-bottom: 7px;"> <input style="border: 1px solid #3c3c3c; width: 302px;" name="recaptcha_response_field" id="recaptcha_response_field" type="text" /> </div> </td> <td colspan=2><span id="recaptcha_privacy" class="recaptcha_only_if_privacy"></span></td> </tr> </table> ';v.VertCss=".recaptchatable td img{display:block}.recaptchatable .recaptcha_r1_c1{background:url('IMGROOT/sprite.png') 0 -63px no-repeat;width:318px;height:9px}.recaptchatable .recaptcha_r2_c1{background:url('IMGROOT/sprite.png') -18px 0 no-repeat;width:9px;height:57px}.recaptchatable .recaptcha_r2_c2{background:url('IMGROOT/sprite.png') -27px 0 no-repeat;width:9px;height:57px}.recaptchatable .recaptcha_r3_c1{background:url('IMGROOT/sprite.png') 0 0 no-repeat;width:9px;height:63px}.recaptchatable .recaptcha_r3_c2{background:url('IMGROOT/sprite.png') -18px -57px no-repeat;width:300px;height:6px}.recaptchatable .recaptcha_r3_c3{background:url('IMGROOT/sprite.png') -9px 0 no-repeat;width:9px;height:63px}.recaptchatable .recaptcha_r4_c1{background:url('IMGROOT/sprite.png') -43px 0 no-repeat;width:171px;height:49px}.recaptchatable .recaptcha_r4_c2{background:url('IMGROOT/sprite.png') -36px 0 no-repeat;width:7px;height:57px}.recaptchatable .recaptcha_r4_c4{background:url('IMGROOT/sprite.png') -214px 0 no-repeat;width:97px;height:57px}.recaptchatable .recaptcha_r7_c1{background:url('IMGROOT/sprite.png') -43px -49px no-repeat;width:171px;height:8px}.recaptchatable .recaptcha_r8_c1{background:url('IMGROOT/sprite.png') -43px -49px no-repeat;width:25px;height:8px}.recaptchatable .recaptcha_image_cell center img{height:57px}.recaptchatable .recaptcha_image_cell center{height:57px}.recaptchatable .recaptcha_image_cell{background-color:white;height:57px}#recaptcha_area,#recaptcha_table{width:318px!important}.recaptchatable,#recaptcha_area tr,#recaptcha_area td,#recaptcha_area th{margin:0!important;border:0!important;padding:0!important;border-collapse:collapse!important;vertical-align:middle!important}.recaptchatable *{margin:0;padding:0;border:0;font-family:helvetica,sans-serif;font-size:8pt;color:black;position:static;top:auto;left:auto;right:auto;bottom:auto}.recaptchatable #recaptcha_image{position:relative;margin:auto}.recaptchatable #recaptcha_image #recaptcha_challenge_image{display:block}.recaptchatable #recaptcha_image #recaptcha_ad_image{display:block;position:absolute;top:0}.recaptchatable img{border:0!important;margin:0!important;padding:0!important}.recaptchatable a,.recaptchatable a:hover{cursor:pointer;outline:none;border:0!important;padding:0!important;text-decoration:none;color:blue;background:none!important;font-weight:normal}.recaptcha_input_area{position:relative!important;width:153px!important;height:45px!important;margin-left:7px!important;margin-right:7px!important;background:none!important}.recaptchatable label.recaptcha_input_area_text{margin:0!important;padding:0!important;position:static!important;top:auto!important;left:auto!important;right:auto!important;bottom:auto!important;background:none!important;height:auto!important;width:auto!important}.recaptcha_theme_red label.recaptcha_input_area_text,.recaptcha_theme_white label.recaptcha_input_area_text{color:black!important}.recaptcha_theme_blackglass label.recaptcha_input_area_text{color:white!important}.recaptchatable #recaptcha_response_field{width:153px!important;position:relative!important;bottom:7px!important;padding:0!important;margin:15px 0 0 0!important;font-size:10pt}.recaptcha_theme_blackglass #recaptcha_response_field,.recaptcha_theme_white #recaptcha_response_field{border:1px solid gray}.recaptcha_theme_red #recaptcha_response_field{border:1px solid #cca940}.recaptcha_audio_cant_hear_link{font-size:7pt;color:black}.recaptchatable{line-height:1!important}#recaptcha_instructions_error{color:red!important}.recaptcha_only_if_privacy{float:right;text-align:right}#recaptcha-ad-choices{position:absolute;height:15px;top:0;right:0}#recaptcha-ad-choices img{height:15px}.recaptcha-ad-choices-collapsed{width:30px;height:15px;display:block}.recaptcha-ad-choices-expanded{width:75px;height:15px;display:none}#recaptcha-ad-choices:hover .recaptcha-ad-choices-collapsed{display:none}#recaptcha-ad-choices:hover .recaptcha-ad-choices-expanded{display:block}";var w={visual_challenge:"Get a visual challenge",audio_challenge:"Get an audio challenge",refresh_btn:"Get a new challenge",instructions_visual:"Type the text:",instructions_audio:"Type what you hear:",help_btn:"Help",play_again:"Play sound again",cant_hear_this:"Download sound as MP3",incorrect_try_again:"Incorrect. Try again.",image_alt_text:"reCAPTCHA challenge image",privacy_and_terms:"Privacy & Terms"},ja={visual_challenge:"\u0627\u0644\u062d\u0635\u0648\u0644 \u0639\u0644\u0649 \u062a\u062d\u062f\u064d \u0645\u0631\u0626\u064a",
-audio_challenge:"\u0627\u0644\u062d\u0635\u0648\u0644 \u0639\u0644\u0649 \u062a\u062d\u062f\u064d \u0635\u0648\u062a\u064a",refresh_btn:"\u0627\u0644\u062d\u0635\u0648\u0644 \u0639\u0644\u0649 \u062a\u062d\u062f\u064d \u062c\u062f\u064a\u062f",instructions_visual:"\u064a\u0631\u062c\u0649 \u0643\u062a\u0627\u0628\u0629 \u0627\u0644\u0646\u0635:",instructions_audio:"\u0627\u0643\u062a\u0628 \u0645\u0627 \u062a\u0633\u0645\u0639\u0647:",help_btn:"\u0645\u0633\u0627\u0639\u062f\u0629",play_again:"\u062a\u0634\u063a\u064a\u0644 \u0627\u0644\u0635\u0648\u062a \u0645\u0631\u0629 \u0623\u062e\u0631\u0649",
-cant_hear_this:"\u062a\u0646\u0632\u064a\u0644 \u0627\u0644\u0635\u0648\u062a \u0628\u062a\u0646\u0633\u064a\u0642 MP3",incorrect_try_again:"\u063a\u064a\u0631 \u0635\u062d\u064a\u062d. \u0623\u0639\u062f \u0627\u0644\u0645\u062d\u0627\u0648\u0644\u0629.",image_alt_text:"\u0635\u0648\u0631\u0629 \u0627\u0644\u062a\u062d\u062f\u064a \u0645\u0646 reCAPTCHA",privacy_and_terms:"\u0627\u0644\u062e\u0635\u0648\u0635\u064a\u0629 \u0648\u0627\u0644\u0628\u0646\u0648\u062f"},ka={visual_challenge:"Obtener una pista visual",
-audio_challenge:"Obtener una pista sonora",refresh_btn:"Obtener una pista nueva",instructions_visual:"Introduzca el texto:",instructions_audio:"Escribe lo que oigas:",help_btn:"Ayuda",play_again:"Volver a reproducir el sonido",cant_hear_this:"Descargar el sonido en MP3",incorrect_try_again:"Incorrecto. Vu\u00e9lvelo a intentar.",image_alt_text:"Pista de imagen reCAPTCHA",privacy_and_terms:"Privacidad y condiciones"},la={visual_challenge:"Kumuha ng pagsubok na visual",audio_challenge:"Kumuha ng pagsubok na audio",
-refresh_btn:"Kumuha ng bagong pagsubok",instructions_visual:"I-type ang teksto:",instructions_audio:"I-type ang iyong narinig",help_btn:"Tulong",play_again:"I-play muli ang tunog",cant_hear_this:"I-download ang tunog bilang MP3",incorrect_try_again:"Hindi wasto. Muling subukan.",image_alt_text:"larawang panghamon ng reCAPTCHA",privacy_and_terms:"Privacy at Mga Tuntunin"},ma={visual_challenge:"Test visuel",audio_challenge:"Test audio",refresh_btn:"Nouveau test",instructions_visual:"Saisissez le texte\u00a0:",
-instructions_audio:"Qu'entendez-vous ?",help_btn:"Aide",play_again:"R\u00e9\u00e9couter",cant_hear_this:"T\u00e9l\u00e9charger l'audio au format MP3",incorrect_try_again:"Incorrect. Veuillez r\u00e9essayer.",image_alt_text:"Image reCAPTCHA",privacy_and_terms:"Confidentialit\u00e9 et conditions d'utilisation"},na={visual_challenge:"Dapatkan kata pengujian berbentuk visual",audio_challenge:"Dapatkan kata pengujian berbentuk audio",refresh_btn:"Dapatkan kata pengujian baru",instructions_visual:"Ketik teks:",
-instructions_audio:"Ketik yang Anda dengar:",help_btn:"Bantuan",play_again:"Putar suara sekali lagi",cant_hear_this:"Unduh suara sebagai MP3",incorrect_try_again:"Salah. Coba lagi.",image_alt_text:"Gambar tantangan reCAPTCHA",privacy_and_terms:"Privasi & Persyaratan"},oa={visual_challenge:"\u05e7\u05d1\u05dc \u05d0\u05ea\u05d2\u05e8 \u05d7\u05d6\u05d5\u05ea\u05d9",audio_challenge:"\u05e7\u05d1\u05dc \u05d0\u05ea\u05d2\u05e8 \u05e9\u05de\u05e2",refresh_btn:"\u05e7\u05d1\u05dc \u05d0\u05ea\u05d2\u05e8 \u05d7\u05d3\u05e9",
-instructions_visual:"\u05d4\u05e7\u05dc\u05d3 \u05d0\u05ea \u05d4\u05d8\u05e7\u05e1\u05d8:",instructions_audio:"\u05d4\u05e7\u05dc\u05d3 \u05d0\u05ea \u05de\u05d4 \u05e9\u05d0\u05ea\u05d4 \u05e9\u05d5\u05de\u05e2:",help_btn:"\u05e2\u05d6\u05e8\u05d4",play_again:"\u05d4\u05e4\u05e2\u05dc \u05e9\u05d5\u05d1 \u05d0\u05ea \u05d4\u05e9\u05de\u05e2",cant_hear_this:"\u05d4\u05d5\u05e8\u05d3 \u05e9\u05de\u05e2 \u05db-3MP",incorrect_try_again:"\u05e9\u05d2\u05d5\u05d9. \u05e0\u05e1\u05d4 \u05e9\u05d5\u05d1.",
-image_alt_text:"\u05ea\u05de\u05d5\u05e0\u05ea \u05d0\u05ea\u05d2\u05e8 \u05e9\u05dc reCAPTCHA",privacy_and_terms:"\u05e4\u05e8\u05d8\u05d9\u05d5\u05ea \u05d5\u05ea\u05e0\u05d0\u05d9\u05dd"},pa={visual_challenge:"Obter um desafio visual",audio_challenge:"Obter um desafio de \u00e1udio",refresh_btn:"Obter um novo desafio",instructions_visual:"Digite o texto:",instructions_audio:"Digite o que voc\u00ea ouve:",help_btn:"Ajuda",play_again:"Reproduzir som novamente",cant_hear_this:"Fazer download do som no formato MP3",
-incorrect_try_again:"Incorreto. Tente novamente.",image_alt_text:"Imagem de desafio reCAPTCHA",privacy_and_terms:"Privacidade e Termos"},qa={visual_challenge:"Ob\u0163ine\u0163i un cod captcha vizual",audio_challenge:"Ob\u0163ine\u0163i un cod captcha audio",refresh_btn:"Ob\u0163ine\u0163i un nou cod captcha",instructions_visual:"Introduce\u021bi textul:",instructions_audio:"Introduce\u0163i ceea ce auzi\u0163i:",help_btn:"Ajutor",play_again:"Reda\u0163i sunetul din nou",cant_hear_this:"Desc\u0103rca\u0163i fi\u015fierul audio ca MP3",
-incorrect_try_again:"Incorect. \u00cencerca\u0163i din nou.",image_alt_text:"Imagine de verificare reCAPTCHA",privacy_and_terms:"Confiden\u0163ialitate \u015fi termeni"},ra={visual_challenge:"\u6536\u5230\u4e00\u4e2a\u89c6\u9891\u9080\u8bf7",audio_challenge:"\u6362\u4e00\u7ec4\u97f3\u9891\u9a8c\u8bc1\u7801",refresh_btn:"\u6362\u4e00\u7ec4\u9a8c\u8bc1\u7801",instructions_visual:"\u8f93\u5165\u6587\u5b57\uff1a",instructions_audio:"\u8bf7\u952e\u5165\u60a8\u542c\u5230\u7684\u5185\u5bb9\uff1a",help_btn:"\u5e2e\u52a9",
-play_again:"\u91cd\u65b0\u64ad\u653e",cant_hear_this:"\u4ee5 MP3 \u683c\u5f0f\u4e0b\u8f7d\u58f0\u97f3",incorrect_try_again:"\u4e0d\u6b63\u786e\uff0c\u8bf7\u91cd\u8bd5\u3002",image_alt_text:"reCAPTCHA \u9a8c\u8bc1\u56fe\u7247",privacy_and_terms:"\u9690\u79c1\u6743\u548c\u4f7f\u7528\u6761\u6b3e"},sa={en:w,af:{visual_challenge:"Kry 'n visuele verifi\u00ebring",audio_challenge:"Kry 'n klankverifi\u00ebring",refresh_btn:"Kry 'n nuwe verifi\u00ebring",instructions_visual:"",instructions_audio:"Tik wat jy hoor:",
-help_btn:"Hulp",play_again:"Speel geluid weer",cant_hear_this:"Laai die klank af as MP3",incorrect_try_again:"Verkeerd. Probeer weer.",image_alt_text:"reCAPTCHA-uitdagingprent",privacy_and_terms:"Privaatheid en bepalings"},am:{visual_challenge:"\u12e8\u12a5\u12ed\u1273 \u1270\u130b\u1323\u121a \u12a0\u130d\u129d",audio_challenge:"\u120c\u120b \u12a0\u12f2\u1235 \u12e8\u12f5\u121d\u133d \u1325\u12eb\u1244 \u12ed\u1245\u1228\u1265",refresh_btn:"\u120c\u120b \u12a0\u12f2\u1235 \u1325\u12eb\u1244 \u12ed\u1245\u1228\u1265",
-instructions_visual:"",instructions_audio:"\u12e8\u121d\u1275\u1230\u121b\u12cd\u1295 \u1270\u12ed\u1265\u1361-",help_btn:"\u12a5\u1308\u12db",play_again:"\u12f5\u121d\u1339\u1295 \u12a5\u1295\u12f0\u1308\u1293 \u12a0\u132b\u12cd\u1275",cant_hear_this:"\u12f5\u121d\u1339\u1295 \u1260MP3 \u1245\u122d\u133d \u12a0\u12cd\u122d\u12f5",incorrect_try_again:"\u1275\u12ad\u12ad\u120d \u12a0\u12ed\u12f0\u1208\u121d\u1362 \u12a5\u1295\u12f0\u1308\u1293 \u121e\u12ad\u122d\u1362",image_alt_text:"reCAPTCHA \u121d\u1235\u120d \u130d\u1320\u121d",
-privacy_and_terms:"\u130d\u120b\u12ca\u1290\u1275 \u12a5\u1293 \u12cd\u120d"},ar:ja,"ar-EG":ja,bg:{visual_challenge:"\u041f\u043e\u043b\u0443\u0447\u0430\u0432\u0430\u043d\u0435 \u043d\u0430 \u0432\u0438\u0437\u0443\u0430\u043b\u043d\u0430 \u043f\u0440\u043e\u0432\u0435\u0440\u043a\u0430",audio_challenge:"\u0417\u0430\u0440\u0435\u0436\u0434\u0430\u043d\u0435 \u043d\u0430 \u0430\u0443\u0434\u0438\u043e\u0442\u0435\u0441\u0442",refresh_btn:"\u0417\u0430\u0440\u0435\u0436\u0434\u0430\u043d\u0435 \u043d\u0430 \u043d\u043e\u0432 \u0442\u0435\u0441\u0442",
-instructions_visual:"\u0412\u044a\u0432\u0435\u0434\u0435\u0442\u0435 \u0442\u0435\u043a\u0441\u0442\u0430:",instructions_audio:"\u0412\u044a\u0432\u0435\u0434\u0435\u0442\u0435 \u0447\u0443\u0442\u043e\u0442\u043e:",help_btn:"\u041f\u043e\u043c\u043e\u0449",play_again:"\u041f\u043e\u0432\u0442\u043e\u0440\u043d\u043e \u043f\u0443\u0441\u043a\u0430\u043d\u0435 \u043d\u0430 \u0437\u0432\u0443\u043a\u0430",cant_hear_this:"\u0418\u0437\u0442\u0435\u0433\u043b\u044f\u043d\u0435 \u043d\u0430 \u0437\u0432\u0443\u043a\u0430 \u0432\u044a\u0432 \u0444\u043e\u0440\u043c\u0430\u0442 MP3",
-incorrect_try_again:"\u041d\u0435\u043f\u0440\u0430\u0432\u0438\u043b\u043d\u043e. \u041e\u043f\u0438\u0442\u0430\u0439\u0442\u0435 \u043e\u0442\u043d\u043e\u0432\u043e.",image_alt_text:"\u0418\u0437\u043e\u0431\u0440\u0430\u0436\u0435\u043d\u0438\u0435 \u043d\u0430 \u043f\u0440\u043e\u0432\u0435\u0440\u043a\u0430\u0442\u0430 \u0441 reCAPTCHA",privacy_and_terms:"\u041f\u043e\u0432\u0435\u0440\u0438\u0442\u0435\u043b\u043d\u043e\u0441\u0442 \u0438 \u041e\u0431\u0449\u0438 \u0443\u0441\u043b\u043e\u0432\u0438\u044f"},
-bn:{visual_challenge:"\u098f\u0995\u099f\u09bf \u09a6\u09c3\u09b6\u09cd\u09af\u09ae\u09be\u09a8 \u09aa\u09cd\u09b0\u09a4\u09bf\u09a6\u09cd\u09ac\u09a8\u09cd\u09a6\u09cd\u09ac\u09bf\u09a4\u09be \u09aa\u09be\u09a8",audio_challenge:"\u098f\u0995\u099f\u09bf \u0985\u09a1\u09bf\u0993 \u09aa\u09cd\u09b0\u09a4\u09bf\u09a6\u09cd\u09ac\u09a8\u09cd\u09a6\u09cd\u09ac\u09bf\u09a4\u09be  \u09aa\u09be\u09a8",refresh_btn:"\u098f\u0995\u099f\u09bf \u09a8\u09a4\u09c1\u09a8 \u09aa\u09cd\u09b0\u09a4\u09bf\u09a6\u09cd\u09ac\u09a8\u09cd\u09a6\u09cd\u09ac\u09bf\u09a4\u09be  \u09aa\u09be\u09a8",
-instructions_visual:"",instructions_audio:"\u0986\u09aa\u09a8\u09bf \u09af\u09be \u09b6\u09c1\u09a8\u099b\u09c7\u09a8 \u09a4\u09be \u09b2\u09bf\u0996\u09c1\u09a8:",help_btn:"\u09b8\u09b9\u09be\u09df\u09a4\u09be",play_again:"\u0986\u09ac\u09be\u09b0 \u09b8\u09be\u0989\u09a8\u09cd\u09a1 \u09aa\u09cd\u09b2\u09c7 \u0995\u09b0\u09c1\u09a8",cant_hear_this:"MP3 \u09b0\u09c2\u09aa\u09c7 \u09b6\u09ac\u09cd\u09a6 \u09a1\u09be\u0989\u09a8\u09b2\u09cb\u09a1 \u0995\u09b0\u09c1\u09a8",incorrect_try_again:"\u09ac\u09c7\u09a0\u09bf\u0995\u09f7 \u0986\u09ac\u09be\u09b0 \u099a\u09c7\u09b7\u09cd\u099f\u09be \u0995\u09b0\u09c1\u09a8\u09f7",
-image_alt_text:"reCAPTCHA \u099a\u09cd\u09af\u09be\u09b2\u09c7\u099e\u09cd\u099c \u099a\u09bf\u09a4\u09cd\u09b0",privacy_and_terms:"\u0997\u09cb\u09aa\u09a8\u09c0\u09af\u09bc\u09a4\u09be \u0993 \u09b6\u09b0\u09cd\u09a4\u09be\u09ac\u09b2\u09c0"},ca:{visual_challenge:"Obt\u00e9n un repte visual",audio_challenge:"Obteniu una pista sonora",refresh_btn:"Obteniu una pista nova",instructions_visual:"Escriviu el text:",instructions_audio:"Escriviu el que escolteu:",help_btn:"Ajuda",play_again:"Torna a reproduir el so",
-cant_hear_this:"Baixa el so com a MP3",incorrect_try_again:"No \u00e9s correcte. Torna-ho a provar.",image_alt_text:"Imatge del repte de reCAPTCHA",privacy_and_terms:"Privadesa i condicions"},cs:{visual_challenge:"Zobrazit vizu\u00e1ln\u00ed podobu v\u00fdrazu",audio_challenge:"P\u0159ehr\u00e1t zvukovou podobu v\u00fdrazu",refresh_btn:"Zobrazit nov\u00fd v\u00fdraz",instructions_visual:"Zadejte text:",instructions_audio:"Napi\u0161te, co jste sly\u0161eli:",help_btn:"N\u00e1pov\u011bda",play_again:"Znovu p\u0159ehr\u00e1t zvuk",
-cant_hear_this:"St\u00e1hnout zvuk ve form\u00e1tu MP3",incorrect_try_again:"\u0160patn\u011b. Zkuste to znovu.",image_alt_text:"Obr\u00e1zek reCAPTCHA",privacy_and_terms:"Ochrana soukrom\u00ed a smluvn\u00ed podm\u00ednky"},da:{visual_challenge:"Hent en visuel udfordring",audio_challenge:"Hent en lydudfordring",refresh_btn:"Hent en ny udfordring",instructions_visual:"Indtast teksten:",instructions_audio:"Indtast det, du h\u00f8rer:",help_btn:"Hj\u00e6lp",play_again:"Afspil lyden igen",cant_hear_this:"Download lyd som MP3",
-incorrect_try_again:"Forkert. Pr\u00f8v igen.",image_alt_text:"reCAPTCHA-udfordringsbillede",privacy_and_terms:"Privatliv og vilk\u00e5r"},de:{visual_challenge:"Captcha abrufen",audio_challenge:"Audio-Captcha abrufen",refresh_btn:"Neues Captcha abrufen",instructions_visual:"Geben Sie den angezeigten Text ein:",instructions_audio:"Geben Sie das Geh\u00f6rte ein:",help_btn:"Hilfe",play_again:"Wort erneut abspielen",cant_hear_this:"Wort als MP3 herunterladen",incorrect_try_again:"Falsch. Bitte versuchen Sie es erneut.",
-image_alt_text:"reCAPTCHA-Bild",privacy_and_terms:"Datenschutzerkl\u00e4rung & Nutzungsbedingungen"},el:{visual_challenge:"\u039f\u03c0\u03c4\u03b9\u03ba\u03ae \u03c0\u03c1\u03cc\u03ba\u03bb\u03b7\u03c3\u03b7",audio_challenge:"\u0397\u03c7\u03b7\u03c4\u03b9\u03ba\u03ae \u03c0\u03c1\u03cc\u03ba\u03bb\u03b7\u03c3\u03b7",refresh_btn:"\u039d\u03ad\u03b1 \u03c0\u03c1\u03cc\u03ba\u03bb\u03b7\u03c3\u03b7",instructions_visual:"\u03a0\u03bb\u03b7\u03ba\u03c4\u03c1\u03bf\u03bb\u03bf\u03b3\u03ae\u03c3\u03c4\u03b5 \u03c4\u03bf \u03ba\u03b5\u03af\u03bc\u03b5\u03bd\u03bf:",
-instructions_audio:"\u03a0\u03bb\u03b7\u03ba\u03c4\u03c1\u03bf\u03bb\u03bf\u03b3\u03ae\u03c3\u03c4\u03b5 \u03cc\u03c4\u03b9 \u03b1\u03ba\u03bf\u03cd\u03c4\u03b5:",help_btn:"\u0392\u03bf\u03ae\u03b8\u03b5\u03b9\u03b1",play_again:"\u0391\u03bd\u03b1\u03c0\u03b1\u03c1\u03b1\u03b3\u03c9\u03b3\u03ae \u03ae\u03c7\u03bf\u03c5 \u03be\u03b1\u03bd\u03ac",cant_hear_this:"\u039b\u03ae\u03c8\u03b7 \u03ae\u03c7\u03bf\u03c5 \u03c9\u03c2 \u039c\u03a13",incorrect_try_again:"\u039b\u03ac\u03b8\u03bf\u03c2. \u0394\u03bf\u03ba\u03b9\u03bc\u03ac\u03c3\u03c4\u03b5 \u03be\u03b1\u03bd\u03ac.",
-image_alt_text:"\u0395\u03b9\u03ba\u03cc\u03bd\u03b1 \u03c0\u03c1\u03cc\u03ba\u03bb\u03b7\u03c3\u03b7\u03c2 reCAPTCHA",privacy_and_terms:"\u0391\u03c0\u03cc\u03c1\u03c1\u03b7\u03c4\u03bf \u03ba\u03b1\u03b9 \u03cc\u03c1\u03bf\u03b9"},"en-GB":w,"en-US":w,es:ka,"es-419":{visual_challenge:"Enfrentar un desaf\u00edo visual",audio_challenge:"Enfrentar un desaf\u00edo de audio",refresh_btn:"Enfrentar un nuevo desaf\u00edo",instructions_visual:"Escriba el texto:",instructions_audio:"Escribe lo que escuchas:",
-help_btn:"Ayuda",play_again:"Reproducir sonido de nuevo",cant_hear_this:"Descargar sonido en formato MP3",incorrect_try_again:"Incorrecto. Vuelve a intentarlo.",image_alt_text:"Imagen del desaf\u00edo de la reCAPTCHA",privacy_and_terms:"Privacidad y condiciones"},"es-ES":ka,et:{visual_challenge:"Kuva kuvap\u00f5hine robotil\u00f5ks",audio_challenge:"Kuva helip\u00f5hine robotil\u00f5ks",refresh_btn:"Kuva uus robotil\u00f5ks",instructions_visual:"Tippige tekst:",instructions_audio:"Tippige, mida kuulete.",
-help_btn:"Abi",play_again:"Esita heli uuesti",cant_hear_this:"Laadi heli alla MP3-vormingus",incorrect_try_again:"Vale. Proovige uuesti.",image_alt_text:"reCAPTCHA robotil\u00f5ksu kujutis",privacy_and_terms:"Privaatsus ja tingimused"},eu:{visual_challenge:"Eskuratu ikusizko erronka",audio_challenge:"Eskuratu audio-erronka",refresh_btn:"Eskuratu erronka berria",instructions_visual:"",instructions_audio:"Idatzi entzuten duzuna:",help_btn:"Laguntza",play_again:"Erreproduzitu soinua berriro",cant_hear_this:"Deskargatu soinua MP3 gisa",
-incorrect_try_again:"Ez da zuzena. Saiatu berriro.",image_alt_text:"reCAPTCHA erronkaren irudia",privacy_and_terms:"Pribatutasuna eta baldintzak"},fa:{visual_challenge:"\u062f\u0631\u06cc\u0627\u0641\u062a \u06cc\u06a9 \u0645\u0639\u0645\u0627\u06cc \u062f\u06cc\u062f\u0627\u0631\u06cc",audio_challenge:"\u062f\u0631\u06cc\u0627\u0641\u062a \u06cc\u06a9 \u0645\u0639\u0645\u0627\u06cc \u0635\u0648\u062a\u06cc",refresh_btn:"\u062f\u0631\u06cc\u0627\u0641\u062a \u06cc\u06a9 \u0645\u0639\u0645\u0627\u06cc \u062c\u062f\u06cc\u062f",
-instructions_visual:"",instructions_audio:"\u0622\u0646\u0686\u0647 \u0631\u0627 \u06a9\u0647 \u0645\u06cc\u200c\u0634\u0646\u0648\u06cc\u062f \u062a\u0627\u06cc\u067e \u06a9\u0646\u06cc\u062f:",help_btn:"\u0631\u0627\u0647\u0646\u0645\u0627\u06cc\u06cc",play_again:"\u067e\u062e\u0634 \u0645\u062c\u062f\u062f \u0635\u062f\u0627",cant_hear_this:"\u062f\u0627\u0646\u0644\u0648\u062f \u0635\u062f\u0627 \u0628\u0647 \u0635\u0648\u0631\u062a MP3",incorrect_try_again:"\u0646\u0627\u062f\u0631\u0633\u062a. \u062f\u0648\u0628\u0627\u0631\u0647 \u0627\u0645\u062a\u062d\u0627\u0646 \u06a9\u0646\u06cc\u062f.",
-image_alt_text:"\u062a\u0635\u0648\u06cc\u0631 \u0686\u0627\u0644\u0634\u06cc reCAPTCHA",privacy_and_terms:"\u062d\u0631\u06cc\u0645 \u062e\u0635\u0648\u0635\u06cc \u0648 \u0634\u0631\u0627\u06cc\u0637"},fi:{visual_challenge:"Kuvavahvistus",audio_challenge:"\u00c4\u00e4nivahvistus",refresh_btn:"Uusi kuva",instructions_visual:"Kirjoita teksti:",instructions_audio:"Kirjoita kuulemasi:",help_btn:"Ohje",play_again:"Toista \u00e4\u00e4ni uudelleen",cant_hear_this:"Lataa \u00e4\u00e4ni MP3-tiedostona",
-incorrect_try_again:"V\u00e4\u00e4rin. Yrit\u00e4 uudelleen.",image_alt_text:"reCAPTCHA-kuva",privacy_and_terms:"Tietosuoja ja k\u00e4ytt\u00f6ehdot"},fil:la,fr:ma,"fr-CA":{visual_challenge:"Obtenir un test visuel",audio_challenge:"Obtenir un test audio",refresh_btn:"Obtenir un nouveau test",instructions_visual:"Saisissez le texte\u00a0:",instructions_audio:"Tapez ce que vous entendez\u00a0:",help_btn:"Aide",play_again:"Jouer le son de nouveau",cant_hear_this:"T\u00e9l\u00e9charger le son en format MP3",
-incorrect_try_again:"Erreur, essayez \u00e0 nouveau",image_alt_text:"Image reCAPTCHA",privacy_and_terms:"Confidentialit\u00e9 et conditions d'utilisation"},"fr-FR":ma,gl:{visual_challenge:"Obter unha proba visual",audio_challenge:"Obter unha proba de audio",refresh_btn:"Obter unha proba nova",instructions_visual:"",instructions_audio:"Escribe o que escoitas:",help_btn:"Axuda",play_again:"Reproducir o son de novo",cant_hear_this:"Descargar son como MP3",incorrect_try_again:"Incorrecto. T\u00e9ntao de novo.",
-image_alt_text:"Imaxe de proba de reCAPTCHA",privacy_and_terms:"Privacidade e condici\u00f3ns"},gu:{visual_challenge:"\u0a8f\u0a95 \u0aa6\u0ac3\u0ab6\u0acd\u0aaf\u0abe\u0aa4\u0acd\u0aae\u0a95 \u0aaa\u0aa1\u0a95\u0abe\u0ab0 \u0aae\u0ac7\u0ab3\u0ab5\u0acb",audio_challenge:"\u0a8f\u0a95 \u0a91\u0aa1\u0abf\u0a93 \u0aaa\u0aa1\u0a95\u0abe\u0ab0 \u0aae\u0ac7\u0ab3\u0ab5\u0acb",refresh_btn:"\u0a8f\u0a95 \u0aa8\u0ab5\u0acb \u0aaa\u0aa1\u0a95\u0abe\u0ab0 \u0aae\u0ac7\u0ab3\u0ab5\u0acb",instructions_visual:"",
-instructions_audio:"\u0aa4\u0aae\u0ac7 \u0a9c\u0ac7 \u0ab8\u0abe\u0a82\u0aad\u0ab3\u0acb \u0a9b\u0acb \u0aa4\u0ac7 \u0ab2\u0a96\u0acb:",help_btn:"\u0ab8\u0ab9\u0abe\u0aaf",play_again:"\u0aa7\u0acd\u0ab5\u0aa8\u0abf \u0aab\u0ab0\u0ac0\u0aa5\u0ac0 \u0a9a\u0ab2\u0abe\u0ab5\u0acb",cant_hear_this:"MP3 \u0aa4\u0ab0\u0ac0\u0a95\u0ac7 \u0aa7\u0acd\u0ab5\u0aa8\u0abf\u0aa8\u0ac7 \u0aa1\u0abe\u0a89\u0aa8\u0ab2\u0acb\u0aa1 \u0a95\u0ab0\u0acb",incorrect_try_again:"\u0a96\u0acb\u0a9f\u0ac1\u0a82. \u0aab\u0ab0\u0ac0 \u0aaa\u0acd\u0ab0\u0aaf\u0abe\u0ab8 \u0a95\u0ab0\u0acb.",
-image_alt_text:"reCAPTCHA \u0aaa\u0aa1\u0a95\u0abe\u0ab0 \u0a9b\u0aac\u0ac0",privacy_and_terms:"\u0a97\u0acb\u0aaa\u0aa8\u0ac0\u0aaf\u0aa4\u0abe \u0a85\u0aa8\u0ac7 \u0ab6\u0ab0\u0aa4\u0acb"},hi:{visual_challenge:"\u0915\u094b\u0908 \u0935\u093f\u091c\u0941\u0905\u0932 \u091a\u0941\u0928\u094c\u0924\u0940 \u0932\u0947\u0902",audio_challenge:"\u0915\u094b\u0908 \u0911\u0921\u093f\u092f\u094b \u091a\u0941\u0928\u094c\u0924\u0940 \u0932\u0947\u0902",refresh_btn:"\u0915\u094b\u0908 \u0928\u0908 \u091a\u0941\u0928\u094c\u0924\u0940 \u0932\u0947\u0902",
-instructions_visual:"\u091f\u0947\u0915\u094d\u0938\u094d\u091f \u091f\u093e\u0907\u092a \u0915\u0930\u0947\u0902:",instructions_audio:"\u091c\u094b \u0906\u092a \u0938\u0941\u0928 \u0930\u0939\u0947 \u0939\u0948\u0902 \u0909\u0938\u0947 \u0932\u093f\u0916\u0947\u0902:",help_btn:"\u0938\u0939\u093e\u092f\u0924\u093e",play_again:"\u0927\u094d\u200d\u0935\u0928\u093f \u092a\u0941\u0928: \u091a\u0932\u093e\u090f\u0902",cant_hear_this:"\u0927\u094d\u200d\u0935\u0928\u093f \u0915\u094b MP3 \u0915\u0947 \u0930\u0942\u092a \u092e\u0947\u0902 \u0921\u093e\u0909\u0928\u0932\u094b\u0921 \u0915\u0930\u0947\u0902",
-incorrect_try_again:"\u0917\u0932\u0924. \u092a\u0941\u0928: \u092a\u094d\u0930\u092f\u093e\u0938 \u0915\u0930\u0947\u0902.",image_alt_text:"reCAPTCHA \u091a\u0941\u0928\u094c\u0924\u0940 \u091a\u093f\u0924\u094d\u0930",privacy_and_terms:"\u0917\u094b\u092a\u0928\u0940\u092f\u0924\u093e \u0914\u0930 \u0936\u0930\u094d\u0924\u0947\u0902"},hr:{visual_challenge:"Dohvati vizualni upit",audio_challenge:"Dohvati zvu\u010dni upit",refresh_btn:"Dohvati novi upit",instructions_visual:"Unesite tekst:",instructions_audio:"Upi\u0161ite \u0161to \u010dujete:",
-help_btn:"Pomo\u0107",play_again:"Ponovi zvuk",cant_hear_this:"Preuzmi zvuk u MP3 formatu",incorrect_try_again:"Nije to\u010dno. Poku\u0161ajte ponovno.",image_alt_text:"Slikovni izazov reCAPTCHA",privacy_and_terms:"Privatnost i odredbe"},hu:{visual_challenge:"Vizu\u00e1lis kih\u00edv\u00e1s k\u00e9r\u00e9se",audio_challenge:"Hangkih\u00edv\u00e1s k\u00e9r\u00e9se",refresh_btn:"\u00daj kih\u00edv\u00e1s k\u00e9r\u00e9se",instructions_visual:"\u00cdrja be a sz\u00f6veget:",instructions_audio:"\u00cdrja le, amit hall:",
-help_btn:"S\u00fag\u00f3",play_again:"Hang ism\u00e9telt lej\u00e1tsz\u00e1sa",cant_hear_this:"Hang let\u00f6lt\u00e9se MP3 form\u00e1tumban",incorrect_try_again:"Hib\u00e1s. Pr\u00f3b\u00e1lkozzon \u00fajra.",image_alt_text:"reCAPTCHA ellen\u0151rz\u0151 k\u00e9p",privacy_and_terms:"Adatv\u00e9delem \u00e9s Szerz\u0151d\u00e9si Felt\u00e9telek"},hy:{visual_challenge:"\u054d\u057f\u0561\u0576\u0561\u056c \u057f\u0565\u057d\u0578\u0572\u0561\u056f\u0561\u0576 \u056d\u0576\u0564\u056b\u0580",audio_challenge:"\u054d\u057f\u0561\u0576\u0561\u056c \u0571\u0561\u0575\u0576\u0561\u0575\u056b\u0576 \u056d\u0576\u0564\u056b\u0580",
-refresh_btn:"\u054d\u057f\u0561\u0576\u0561\u056c \u0576\u0578\u0580 \u056d\u0576\u0564\u056b\u0580",instructions_visual:"\u0544\u0578\u0582\u057f\u0584\u0561\u0563\u0580\u0565\u0584 \u057f\u0565\u0584\u057d\u057f\u0568\u055d",instructions_audio:"\u0544\u0578\u0582\u057f\u0584\u0561\u0563\u0580\u0565\u0584 \u0561\u0575\u0576, \u056b\u0576\u0579 \u056c\u057d\u0578\u0582\u0574 \u0565\u0584\u055d",help_btn:"\u0555\u0563\u0576\u0578\u0582\u0569\u0575\u0578\u0582\u0576",play_again:"\u0546\u057e\u0561\u0563\u0561\u0580\u056f\u0565\u056c \u0571\u0561\u0575\u0576\u0568 \u056f\u0580\u056f\u056b\u0576",
-cant_hear_this:"\u0532\u0565\u057c\u0576\u0565\u056c \u0571\u0561\u0575\u0576\u0568 \u0578\u0580\u057a\u0565\u057d MP3",incorrect_try_again:"\u054d\u056d\u0561\u056c \u0567: \u0553\u0578\u0580\u0571\u0565\u0584 \u056f\u0580\u056f\u056b\u0576:",image_alt_text:"reCAPTCHA \u057a\u0561\u057f\u056f\u0565\u0580\u0578\u057e \u056d\u0576\u0564\u056b\u0580",privacy_and_terms:"\u0533\u0561\u0572\u057f\u0576\u056b\u0578\u0582\u0569\u0575\u0561\u0576 & \u057a\u0561\u0575\u0574\u0561\u0576\u0576\u0565\u0580"},
-id:na,is:{visual_challenge:"F\u00e1 a\u00f0gangspr\u00f3f sem mynd",audio_challenge:"F\u00e1 a\u00f0gangspr\u00f3f sem hlj\u00f3\u00f0skr\u00e1",refresh_btn:"F\u00e1 n\u00fdtt a\u00f0gangspr\u00f3f",instructions_visual:"",instructions_audio:"Sl\u00e1\u00f0u inn \u00fea\u00f0 sem \u00fe\u00fa heyrir:",help_btn:"Hj\u00e1lp",play_again:"Spila hlj\u00f3\u00f0 aftur",cant_hear_this:"S\u00e6kja hlj\u00f3\u00f0 sem MP3",incorrect_try_again:"Rangt. Reyndu aftur.",image_alt_text:"mynd reCAPTCHA a\u00f0gangspr\u00f3fs",
-privacy_and_terms:"Pers\u00f3nuvernd og skilm\u00e1lar"},it:{visual_challenge:"Verifica visiva",audio_challenge:"Verifica audio",refresh_btn:"Nuova verifica",instructions_visual:"Digita il testo:",instructions_audio:"Digita ci\u00f2 che senti:",help_btn:"Guida",play_again:"Riproduci di nuovo audio",cant_hear_this:"Scarica audio in MP3",incorrect_try_again:"Sbagliato. Riprova.",image_alt_text:"Immagine di verifica reCAPTCHA",privacy_and_terms:"Privacy e Termini"},iw:oa,ja:{visual_challenge:"\u753b\u50cf\u3067\u78ba\u8a8d\u3057\u307e\u3059",
-audio_challenge:"\u97f3\u58f0\u3067\u78ba\u8a8d\u3057\u307e\u3059",refresh_btn:"\u5225\u306e\u5358\u8a9e\u3067\u3084\u308a\u76f4\u3057\u307e\u3059",instructions_visual:"\u30c6\u30ad\u30b9\u30c8\u3092\u5165\u529b:",instructions_audio:"\u805e\u3053\u3048\u305f\u5358\u8a9e\u3092\u5165\u529b\u3057\u307e\u3059:",help_btn:"\u30d8\u30eb\u30d7",play_again:"\u3082\u3046\u4e00\u5ea6\u805e\u304f",cant_hear_this:"MP3 \u3067\u97f3\u58f0\u3092\u30c0\u30a6\u30f3\u30ed\u30fc\u30c9",incorrect_try_again:"\u6b63\u3057\u304f\u3042\u308a\u307e\u305b\u3093\u3002\u3082\u3046\u4e00\u5ea6\u3084\u308a\u76f4\u3057\u3066\u304f\u3060\u3055\u3044\u3002",
-image_alt_text:"reCAPTCHA \u78ba\u8a8d\u7528\u753b\u50cf",privacy_and_terms:"\u30d7\u30e9\u30a4\u30d0\u30b7\u30fc\u3068\u5229\u7528\u898f\u7d04"},kn:{visual_challenge:"\u0ca6\u0cc3\u0cb6\u0ccd\u0caf \u0cb8\u0cb5\u0cbe\u0cb2\u0cca\u0c82\u0ca6\u0ca8\u0ccd\u0ca8\u0cc1 \u0cb8\u0ccd\u0cb5\u0cc0\u0c95\u0cb0\u0cbf\u0cb8\u0cbf",audio_challenge:"\u0c86\u0ca1\u0cbf\u0caf\u0ccb \u0cb8\u0cb5\u0cbe\u0cb2\u0cca\u0c82\u0ca6\u0ca8\u0ccd\u0ca8\u0cc1 \u0cb8\u0ccd\u0cb5\u0cc0\u0c95\u0cb0\u0cbf\u0cb8\u0cbf",refresh_btn:"\u0cb9\u0cca\u0cb8 \u0cb8\u0cb5\u0cbe\u0cb2\u0cca\u0c82\u0ca6\u0ca8\u0ccd\u0ca8\u0cc1 \u0caa\u0ca1\u0cc6\u0caf\u0cbf\u0cb0\u0cbf",
-instructions_visual:"",instructions_audio:"\u0ca8\u0cbf\u0cae\u0c97\u0cc6 \u0c95\u0cc7\u0cb3\u0cbf\u0cb8\u0cc1\u0cb5\u0cc1\u0ca6\u0ca8\u0ccd\u0ca8\u0cc1 \u0c9f\u0cc8\u0caa\u0ccd\u200c \u0cae\u0cbe\u0ca1\u0cbf:",help_btn:"\u0cb8\u0cb9\u0cbe\u0caf",play_again:"\u0ca7\u0ccd\u0cb5\u0ca8\u0cbf\u0caf\u0ca8\u0ccd\u0ca8\u0cc1 \u0cae\u0ca4\u0ccd\u0ca4\u0cc6 \u0caa\u0ccd\u0cb2\u0cc7 \u0cae\u0cbe\u0ca1\u0cbf",cant_hear_this:"\u0ca7\u0ccd\u0cb5\u0ca8\u0cbf\u0caf\u0ca8\u0ccd\u0ca8\u0cc1 MP3 \u0cb0\u0cc2\u0caa\u0ca6\u0cb2\u0ccd\u0cb2\u0cbf \u0ca1\u0ccc\u0ca8\u0ccd\u200c\u0cb2\u0ccb\u0ca1\u0ccd \u0cae\u0cbe\u0ca1\u0cbf",
-incorrect_try_again:"\u0ca4\u0caa\u0ccd\u0caa\u0cbe\u0c97\u0cbf\u0ca6\u0cc6. \u0cae\u0ca4\u0ccd\u0ca4\u0cca\u0cae\u0ccd\u0cae\u0cc6 \u0caa\u0ccd\u0cb0\u0caf\u0ca4\u0ccd\u0ca8\u0cbf\u0cb8\u0cbf.",image_alt_text:"reCAPTCHA \u0cb8\u0cb5\u0cbe\u0cb2\u0cc1 \u0c9a\u0cbf\u0ca4\u0ccd\u0cb0",privacy_and_terms:"\u0c97\u0ccc\u0caa\u0ccd\u0caf\u0ca4\u0cc6 \u0cae\u0ca4\u0ccd\u0ca4\u0cc1 \u0ca8\u0cbf\u0caf\u0cae\u0c97\u0cb3\u0cc1"},ko:{visual_challenge:"\uadf8\ub9bc\uc73c\ub85c \ubcf4\uc548\ubb38\uc790 \ubc1b\uae30",
-audio_challenge:"\uc74c\uc131\uc73c\ub85c \ubcf4\uc548\ubb38\uc790 \ubc1b\uae30",refresh_btn:"\ubcf4\uc548\ubb38\uc790 \uc0c8\ub85c \ubc1b\uae30",instructions_visual:"\ud14d\uc2a4\ud2b8 \uc785\ub825:",instructions_audio:"\uc74c\uc131 \ubcf4\uc548\ubb38\uc790 \uc785\ub825:",help_btn:"\ub3c4\uc6c0\ub9d0",play_again:"\uc74c\uc131 \ub2e4\uc2dc \ub4e3\uae30",cant_hear_this:"\uc74c\uc131\uc744 MP3\ub85c \ub2e4\uc6b4\ub85c\ub4dc",incorrect_try_again:"\ud2c0\ub838\uc2b5\ub2c8\ub2e4. \ub2e4\uc2dc \uc2dc\ub3c4\ud574 \uc8fc\uc138\uc694.",
-image_alt_text:"reCAPTCHA \ubcf4\uc548\ubb38\uc790 \uc774\ubbf8\uc9c0",privacy_and_terms:"\uac1c\uc778\uc815\ubcf4 \ubcf4\ud638 \ubc0f \uc57d\uad00"},ln:ma,lt:{visual_challenge:"Gauti vaizdin\u012f atpa\u017einimo test\u0105",audio_challenge:"Gauti garso atpa\u017einimo test\u0105",refresh_btn:"Gauti nauj\u0105 atpa\u017einimo test\u0105",instructions_visual:"\u012eveskite tekst\u0105:",instructions_audio:"\u012eveskite tai, k\u0105 girdite:",help_btn:"Pagalba",play_again:"Dar kart\u0105 paleisti gars\u0105",
-cant_hear_this:"Atsisi\u0173sti gars\u0105 kaip MP3",incorrect_try_again:"Neteisingai. Bandykite dar kart\u0105.",image_alt_text:"Testo \u201ereCAPTCHA\u201c vaizdas",privacy_and_terms:"Privatumas ir s\u0105lygos"},lv:{visual_challenge:"Sa\u0146emt vizu\u0101lu izaicin\u0101jumu",audio_challenge:"Sa\u0146emt audio izaicin\u0101jumu",refresh_btn:"Sa\u0146emt jaunu izaicin\u0101jumu",instructions_visual:"Ievadiet tekstu:",instructions_audio:"Ierakstiet dzirdamo:",help_btn:"Pal\u012bdz\u012bba",play_again:"V\u0113lreiz atska\u0146ot ska\u0146u",
-cant_hear_this:"Lejupiel\u0101d\u0113t ska\u0146u MP3\u00a0form\u0101t\u0101",incorrect_try_again:"Nepareizi. M\u0113\u0123iniet v\u0113lreiz.",image_alt_text:"reCAPTCHA izaicin\u0101juma att\u0113ls",privacy_and_terms:"Konfidencialit\u0101te un noteikumi"},ml:{visual_challenge:"\u0d12\u0d30\u0d41 \u0d26\u0d43\u0d36\u0d4d\u0d2f \u0d1a\u0d32\u0d1e\u0d4d\u0d1a\u0d4d \u0d28\u0d47\u0d1f\u0d41\u0d15",audio_challenge:"\u0d12\u0d30\u0d41 \u0d13\u0d21\u0d3f\u0d2f\u0d4b \u0d1a\u0d32\u0d1e\u0d4d\u0d1a\u0d4d \u0d28\u0d47\u0d1f\u0d41\u0d15",
-refresh_btn:"\u0d12\u0d30\u0d41 \u0d2a\u0d41\u0d24\u0d3f\u0d2f \u0d1a\u0d32\u0d1e\u0d4d\u0d1a\u0d4d \u0d28\u0d47\u0d1f\u0d41\u0d15",instructions_visual:"",instructions_audio:"\u0d15\u0d47\u0d7e\u0d15\u0d4d\u0d15\u0d41\u0d28\u0d4d\u0d28\u0d24\u0d4d \u0d1f\u0d48\u0d2a\u0d4d\u0d2a\u0d4d \u0d1a\u0d46\u0d2f\u0d4d\u0d2f\u0d42:",help_btn:"\u0d38\u0d39\u0d3e\u0d2f\u0d02",play_again:"\u0d36\u0d2c\u0d4d\u200c\u0d26\u0d02 \u0d35\u0d40\u0d23\u0d4d\u0d1f\u0d41\u0d02 \u0d2a\u0d4d\u0d32\u0d47 \u0d1a\u0d46\u0d2f\u0d4d\u0d2f\u0d41\u0d15",
-cant_hear_this:"\u0d36\u0d2c\u0d4d\u200c\u0d26\u0d02 MP3 \u0d06\u0d2f\u0d3f \u0d21\u0d57\u0d7a\u0d32\u0d4b\u0d21\u0d4d \u0d1a\u0d46\u0d2f\u0d4d\u0d2f\u0d41\u0d15",incorrect_try_again:"\u0d24\u0d46\u0d31\u0d4d\u0d31\u0d3e\u0d23\u0d4d. \u0d35\u0d40\u0d23\u0d4d\u0d1f\u0d41\u0d02 \u0d36\u0d4d\u0d30\u0d2e\u0d3f\u0d15\u0d4d\u0d15\u0d41\u0d15.",image_alt_text:"reCAPTCHA \u0d1a\u0d32\u0d1e\u0d4d\u0d1a\u0d4d \u0d07\u0d2e\u0d47\u0d1c\u0d4d",privacy_and_terms:"\u0d38\u0d4d\u0d35\u0d15\u0d3e\u0d30\u0d4d\u0d2f\u0d24\u0d2f\u0d41\u0d02 \u0d28\u0d3f\u0d2c\u0d28\u0d4d\u0d27\u0d28\u0d15\u0d33\u0d41\u0d02"},
-mr:{visual_challenge:"\u0926\u0943\u0936\u094d\u200d\u092f\u092e\u093e\u0928 \u0906\u0935\u094d\u0939\u093e\u0928 \u092a\u094d\u0930\u093e\u092a\u094d\u0924 \u0915\u0930\u093e",audio_challenge:"\u0911\u0921\u0940\u0913 \u0906\u0935\u094d\u0939\u093e\u0928 \u092a\u094d\u0930\u093e\u092a\u094d\u0924 \u0915\u0930\u093e",refresh_btn:"\u090f\u0915 \u0928\u0935\u0940\u0928 \u0906\u0935\u094d\u0939\u093e\u0928 \u092a\u094d\u0930\u093e\u092a\u094d\u0924 \u0915\u0930\u093e",instructions_visual:"",instructions_audio:"\u0906\u092a\u0932\u094d\u092f\u093e\u0932\u093e \u091c\u0947 \u0910\u0915\u0942 \u092f\u0947\u0908\u0932 \u0924\u0947 \u091f\u093e\u0907\u092a \u0915\u0930\u093e:",
-help_btn:"\u092e\u0926\u0924",play_again:"\u0927\u094d\u200d\u0935\u0928\u0940 \u092a\u0941\u0928\u094d\u0939\u093e \u092a\u094d\u200d\u0932\u0947 \u0915\u0930\u093e",cant_hear_this:"MP3 \u0930\u0941\u092a\u093e\u0924 \u0927\u094d\u200d\u0935\u0928\u0940 \u0921\u093e\u0909\u0928\u0932\u094b\u0921 \u0915\u0930\u093e",incorrect_try_again:"\u0905\u092f\u094b\u0917\u094d\u200d\u092f. \u092a\u0941\u0928\u094d\u200d\u0939\u093e \u092a\u094d\u0930\u092f\u0924\u094d\u200d\u0928 \u0915\u0930\u093e.",image_alt_text:"reCAPTCHA \u0906\u0935\u094d\u200d\u0939\u093e\u0928 \u092a\u094d\u0930\u0924\u093f\u092e\u093e",
-privacy_and_terms:"\u0917\u094b\u092a\u0928\u0940\u092f\u0924\u093e \u0906\u0923\u093f \u0905\u091f\u0940"},ms:{visual_challenge:"Dapatkan cabaran visual",audio_challenge:"Dapatkan cabaran audio",refresh_btn:"Dapatkan cabaran baru",instructions_visual:"Taipkan teksnya:",instructions_audio:"Taip apa yang didengari:",help_btn:"Bantuan",play_again:"Mainkan bunyi sekali lagi",cant_hear_this:"Muat turun bunyi sebagai MP3",incorrect_try_again:"Tidak betul. Cuba lagi.",image_alt_text:"Imej cabaran reCAPTCHA",
-privacy_and_terms:"Privasi & Syarat"},nl:{visual_challenge:"Een visuele uitdaging proberen",audio_challenge:"Een audio-uitdaging proberen",refresh_btn:"Een nieuwe uitdaging proberen",instructions_visual:"Typ de tekst:",instructions_audio:"Typ wat u hoort:",help_btn:"Help",play_again:"Geluid opnieuw afspelen",cant_hear_this:"Geluid downloaden als MP3",incorrect_try_again:"Onjuist. Probeer het opnieuw.",image_alt_text:"reCAPTCHA-uitdagingsafbeelding",privacy_and_terms:"Privacy en voorwaarden"},no:{visual_challenge:"F\u00e5 en bildeutfordring",
-audio_challenge:"F\u00e5 en lydutfordring",refresh_btn:"F\u00e5 en ny utfordring",instructions_visual:"Skriv inn teksten:",instructions_audio:"Skriv inn det du h\u00f8rer:",help_btn:"Hjelp",play_again:"Spill av lyd p\u00e5 nytt",cant_hear_this:"Last ned lyd som MP3",incorrect_try_again:"Feil. Pr\u00f8v p\u00e5 nytt.",image_alt_text:"reCAPTCHA-utfordringsbilde",privacy_and_terms:"Personvern og vilk\u00e5r"},pl:{visual_challenge:"Poka\u017c podpowied\u017a wizualn\u0105",audio_challenge:"Odtw\u00f3rz podpowied\u017a d\u017awi\u0119kow\u0105",
-refresh_btn:"Nowa podpowied\u017a",instructions_visual:"Przepisz tekst:",instructions_audio:"Wpisz us\u0142yszane s\u0142owa:",help_btn:"Pomoc",play_again:"Odtw\u00f3rz d\u017awi\u0119k ponownie",cant_hear_this:"Pobierz d\u017awi\u0119k jako plik MP3",incorrect_try_again:"Nieprawid\u0142owo. Spr\u00f3buj ponownie.",image_alt_text:"Zadanie obrazkowe reCAPTCHA",privacy_and_terms:"Prywatno\u015b\u0107 i warunki"},pt:pa,"pt-BR":pa,"pt-PT":{visual_challenge:"Obter um desafio visual",audio_challenge:"Obter um desafio de \u00e1udio",
-refresh_btn:"Obter um novo desafio",instructions_visual:"Introduza o texto:",instructions_audio:"Escreva o que ouvir:",help_btn:"Ajuda",play_again:"Reproduzir som novamente",cant_hear_this:"Transferir som como MP3",incorrect_try_again:"Incorreto. Tente novamente.",image_alt_text:"Imagem de teste reCAPTCHA",privacy_and_terms:"Privacidade e Termos de Utiliza\u00e7\u00e3o"},ro:qa,ru:{visual_challenge:"\u0412\u0438\u0437\u0443\u0430\u043b\u044c\u043d\u0430\u044f \u043f\u0440\u043e\u0432\u0435\u0440\u043a\u0430",
-audio_challenge:"\u0417\u0432\u0443\u043a\u043e\u0432\u0430\u044f \u043f\u0440\u043e\u0432\u0435\u0440\u043a\u0430",refresh_btn:"\u041e\u0431\u043d\u043e\u0432\u0438\u0442\u044c",instructions_visual:"\u0412\u0432\u0435\u0434\u0438\u0442\u0435 \u0442\u0435\u043a\u0441\u0442:",instructions_audio:"\u0412\u0432\u0435\u0434\u0438\u0442\u0435 \u0442\u043e, \u0447\u0442\u043e \u0441\u043b\u044b\u0448\u0438\u0442\u0435:",help_btn:"\u0421\u043f\u0440\u0430\u0432\u043a\u0430",play_again:"\u041f\u0440\u043e\u0441\u043b\u0443\u0448\u0430\u0442\u044c \u0435\u0449\u0435 \u0440\u0430\u0437",
-cant_hear_this:"\u0417\u0430\u0433\u0440\u0443\u0437\u0438\u0442\u044c MP3-\u0444\u0430\u0439\u043b",incorrect_try_again:"\u041d\u0435\u043f\u0440\u0430\u0432\u0438\u043b\u044c\u043d\u043e. \u041f\u043e\u0432\u0442\u043e\u0440\u0438\u0442\u0435 \u043f\u043e\u043f\u044b\u0442\u043a\u0443.",image_alt_text:"\u041f\u0440\u043e\u0432\u0435\u0440\u043a\u0430 \u043f\u043e \u0441\u043b\u043e\u0432\u0443 reCAPTCHA",privacy_and_terms:"\u041f\u0440\u0430\u0432\u0438\u043b\u0430 \u0438 \u043f\u0440\u0438\u043d\u0446\u0438\u043f\u044b"},
-sk:{visual_challenge:"Zobrazi\u0165 vizu\u00e1lnu podobu",audio_challenge:"Prehra\u0165 zvukov\u00fa podobu",refresh_btn:"Zobrazi\u0165 nov\u00fd v\u00fdraz",instructions_visual:"Zadajte text:",instructions_audio:"Zadajte, \u010do po\u010dujete:",help_btn:"Pomocn\u00edk",play_again:"Znova prehra\u0165 zvuk",cant_hear_this:"Prevzia\u0165 zvuk v podobe s\u00faboru MP3",incorrect_try_again:"Nespr\u00e1vne. Sk\u00faste to znova.",image_alt_text:"Obr\u00e1zok zadania reCAPTCHA",privacy_and_terms:"Ochrana osobn\u00fdch \u00fadajov a Zmluvn\u00e9 podmienky"},
-sl:{visual_challenge:"Vizualni preskus",audio_challenge:"Zvo\u010dni preskus",refresh_btn:"Nov preskus",instructions_visual:"Vnesite besedilo:",instructions_audio:"Natipkajte, kaj sli\u0161ite:",help_btn:"Pomo\u010d",play_again:"Znova predvajaj zvok",cant_hear_this:"Prenesi zvok kot MP3",incorrect_try_again:"Napa\u010dno. Poskusite znova.",image_alt_text:"Slika izziva reCAPTCHA",privacy_and_terms:"Zasebnost in pogoji"},sr:{visual_challenge:"\u041f\u0440\u0438\u043c\u0438\u0442\u0435 \u0432\u0438\u0437\u0443\u0435\u043b\u043d\u0438 \u0443\u043f\u0438\u0442",
-audio_challenge:"\u041f\u0440\u0438\u043c\u0438\u0442\u0435 \u0430\u0443\u0434\u0438\u043e \u0443\u043f\u0438\u0442",refresh_btn:"\u041f\u0440\u0438\u043c\u0438\u0442\u0435 \u043d\u043e\u0432\u0438 \u0443\u043f\u0438\u0442",instructions_visual:"\u0423\u043d\u0435\u0441\u0438\u0442\u0435 \u0442\u0435\u043a\u0441\u0442:",instructions_audio:"\u041e\u0442\u043a\u0443\u0446\u0430\u0458\u0442\u0435 \u043e\u043d\u043e \u0448\u0442\u043e \u0447\u0443\u0458\u0435\u0442\u0435:",help_btn:"\u041f\u043e\u043c\u043e\u045b",
-play_again:"\u041f\u043e\u043d\u043e\u0432\u043e \u043f\u0443\u0441\u0442\u0438 \u0437\u0432\u0443\u043a",cant_hear_this:"\u041f\u0440\u0435\u0443\u0437\u043c\u0438 \u0437\u0432\u0443\u043a \u043a\u0430\u043e MP3 \u0441\u043d\u0438\u043c\u0430\u043a",incorrect_try_again:"\u041d\u0435\u0442\u0430\u0447\u043d\u043e. \u041f\u043e\u043a\u0443\u0448\u0430\u0458\u0442\u0435 \u043f\u043e\u043d\u043e\u0432\u043e.",image_alt_text:"\u0421\u043b\u0438\u043a\u0430 reCAPTCHA \u043f\u0440\u043e\u0432\u0435\u0440\u0435",
-privacy_and_terms:"\u041f\u0440\u0438\u0432\u0430\u0442\u043d\u043e\u0441\u0442 \u0438 \u0443\u0441\u043b\u043e\u0432\u0438"},sv:{visual_challenge:"H\u00e4mta captcha i bildformat",audio_challenge:"H\u00e4mta captcha i ljudformat",refresh_btn:"H\u00e4mta ny captcha",instructions_visual:"Skriv texten:",instructions_audio:"Skriv det du h\u00f6r:",help_btn:"Hj\u00e4lp",play_again:"Spela upp ljudet igen",cant_hear_this:"H\u00e4mta ljud som MP3",incorrect_try_again:"Fel. F\u00f6rs\u00f6k igen.",image_alt_text:"reCAPTCHA-bild",
-privacy_and_terms:"Sekretess och villkor"},sw:{visual_challenge:"Pata herufi za kusoma",audio_challenge:"Pata herufi za kusikiliza",refresh_btn:"Pata herufi mpya",instructions_visual:"",instructions_audio:"Charaza unachosikia:",help_btn:"Usaidizi",play_again:"Cheza sauti tena",cant_hear_this:"Pakua sauti kama MP3",incorrect_try_again:"Sio sahihi. Jaribu tena.",image_alt_text:"picha ya changamoto ya reCAPTCHA",privacy_and_terms:"Faragha & Masharti"},ta:{visual_challenge:"\u0baa\u0bbe\u0bb0\u0bcd\u0bb5\u0bc8 \u0b9a\u0bc7\u0bb2\u0b9e\u0bcd\u0b9a\u0bc8\u0baa\u0bcd \u0baa\u0bc6\u0bb1\u0bc1\u0b95",
-audio_challenge:"\u0b86\u0b9f\u0bbf\u0baf\u0bcb \u0b9a\u0bc7\u0bb2\u0b9e\u0bcd\u0b9a\u0bc8\u0baa\u0bcd \u0baa\u0bc6\u0bb1\u0bc1\u0b95",refresh_btn:"\u0baa\u0bc1\u0ba4\u0bbf\u0baf \u0b9a\u0bc7\u0bb2\u0b9e\u0bcd\u0b9a\u0bc8\u0baa\u0bcd \u0baa\u0bc6\u0bb1\u0bc1\u0b95",instructions_visual:"",instructions_audio:"\u0b95\u0bc7\u0b9f\u0bcd\u0baa\u0ba4\u0bc8 \u0b9f\u0bc8\u0baa\u0bcd \u0b9a\u0bc6\u0baf\u0bcd\u0b95:",help_btn:"\u0b89\u0ba4\u0bb5\u0bbf",play_again:"\u0b92\u0bb2\u0bbf\u0baf\u0bc8 \u0bae\u0bc0\u0ba3\u0bcd\u0b9f\u0bc1\u0bae\u0bcd \u0b87\u0baf\u0b95\u0bcd\u0b95\u0bc1",
-cant_hear_this:"\u0b92\u0bb2\u0bbf\u0baf\u0bc8 MP3 \u0b86\u0b95 \u0baa\u0ba4\u0bbf\u0bb5\u0bbf\u0bb1\u0b95\u0bcd\u0b95\u0bc1\u0b95",incorrect_try_again:"\u0ba4\u0bb5\u0bb1\u0bbe\u0ba9\u0ba4\u0bc1. \u0bae\u0bc0\u0ba3\u0bcd\u0b9f\u0bc1\u0bae\u0bcd \u0bae\u0bc1\u0baf\u0bb2\u0bb5\u0bc1\u0bae\u0bcd.",image_alt_text:"reCAPTCHA \u0b9a\u0bc7\u0bb2\u0b9e\u0bcd\u0b9a\u0bcd \u0baa\u0b9f\u0bae\u0bcd",privacy_and_terms:"\u0ba4\u0ba9\u0bbf\u0baf\u0bc1\u0bb0\u0bbf\u0bae\u0bc8 & \u0bb5\u0bbf\u0ba4\u0bbf\u0bae\u0bc1\u0bb1\u0bc8\u0b95\u0bb3\u0bcd"},
-te:{visual_challenge:"\u0c12\u0c15 \u0c26\u0c43\u0c36\u0c4d\u0c2f\u0c2e\u0c3e\u0c28 \u0c38\u0c35\u0c3e\u0c32\u0c41\u0c28\u0c41 \u0c38\u0c4d\u0c35\u0c40\u0c15\u0c30\u0c3f\u0c02\u0c1a\u0c02\u0c21\u0c3f",audio_challenge:"\u0c12\u0c15 \u0c06\u0c21\u0c3f\u0c2f\u0c4b \u0c38\u0c35\u0c3e\u0c32\u0c41\u0c28\u0c41 \u0c38\u0c4d\u0c35\u0c40\u0c15\u0c30\u0c3f\u0c02\u0c1a\u0c02\u0c21\u0c3f",refresh_btn:"\u0c15\u0c4d\u0c30\u0c4a\u0c24\u0c4d\u0c24 \u0c38\u0c35\u0c3e\u0c32\u0c41\u0c28\u0c41 \u0c38\u0c4d\u0c35\u0c40\u0c15\u0c30\u0c3f\u0c02\u0c1a\u0c02\u0c21\u0c3f",
-instructions_visual:"",instructions_audio:"\u0c2e\u0c40\u0c30\u0c41 \u0c35\u0c3f\u0c28\u0c4d\u0c28\u0c26\u0c3f \u0c1f\u0c48\u0c2a\u0c4d \u0c1a\u0c47\u0c2f\u0c02\u0c21\u0c3f:",help_btn:"\u0c38\u0c39\u0c3e\u0c2f\u0c02",play_again:"\u0c27\u0c4d\u0c35\u0c28\u0c3f\u0c28\u0c3f \u0c2e\u0c33\u0c4d\u0c32\u0c40 \u0c2a\u0c4d\u0c32\u0c47 \u0c1a\u0c47\u0c2f\u0c3f",cant_hear_this:"\u0c27\u0c4d\u0c35\u0c28\u0c3f\u0c28\u0c3f MP3 \u0c35\u0c32\u0c46 \u0c21\u0c4c\u0c28\u0c4d\u200c\u0c32\u0c4b\u0c21\u0c4d \u0c1a\u0c47\u0c2f\u0c3f",
-incorrect_try_again:"\u0c24\u0c2a\u0c4d\u0c2a\u0c41. \u0c2e\u0c33\u0c4d\u0c32\u0c40 \u0c2a\u0c4d\u0c30\u0c2f\u0c24\u0c4d\u0c28\u0c3f\u0c02\u0c1a\u0c02\u0c21\u0c3f.",image_alt_text:"reCAPTCHA \u0c38\u0c35\u0c3e\u0c32\u0c41 \u0c1a\u0c3f\u0c24\u0c4d\u0c30\u0c02",privacy_and_terms:"\u0c17\u0c4b\u0c2a\u0c4d\u0c2f\u0c24 & \u0c28\u0c3f\u0c2c\u0c02\u0c27\u0c28\u0c32\u0c41"},th:{visual_challenge:"\u0e23\u0e31\u0e1a\u0e04\u0e27\u0e32\u0e21\u0e17\u0e49\u0e32\u0e17\u0e32\u0e22\u0e14\u0e49\u0e32\u0e19\u0e20\u0e32\u0e1e",
-audio_challenge:"\u0e23\u0e31\u0e1a\u0e04\u0e27\u0e32\u0e21\u0e17\u0e49\u0e32\u0e17\u0e32\u0e22\u0e14\u0e49\u0e32\u0e19\u0e40\u0e2a\u0e35\u0e22\u0e07",refresh_btn:"\u0e23\u0e31\u0e1a\u0e04\u0e27\u0e32\u0e21\u0e17\u0e49\u0e32\u0e17\u0e32\u0e22\u0e43\u0e2b\u0e21\u0e48",instructions_visual:"\u0e1e\u0e34\u0e21\u0e1e\u0e4c\u0e02\u0e49\u0e2d\u0e04\u0e27\u0e32\u0e21\u0e19\u0e35\u0e49:",instructions_audio:"\u0e1e\u0e34\u0e21\u0e1e\u0e4c\u0e2a\u0e34\u0e48\u0e07\u0e17\u0e35\u0e48\u0e04\u0e38\u0e13\u0e44\u0e14\u0e49\u0e22\u0e34\u0e19:",
-help_btn:"\u0e04\u0e27\u0e32\u0e21\u0e0a\u0e48\u0e27\u0e22\u0e40\u0e2b\u0e25\u0e37\u0e2d",play_again:"\u0e40\u0e25\u0e48\u0e19\u0e40\u0e2a\u0e35\u0e22\u0e07\u0e2d\u0e35\u0e01\u0e04\u0e23\u0e31\u0e49\u0e07",cant_hear_this:"\u0e14\u0e32\u0e27\u0e42\u0e2b\u0e25\u0e14\u0e40\u0e2a\u0e35\u0e22\u0e07\u0e40\u0e1b\u0e47\u0e19 MP3",incorrect_try_again:"\u0e44\u0e21\u0e48\u0e16\u0e39\u0e01\u0e15\u0e49\u0e2d\u0e07 \u0e25\u0e2d\u0e07\u0e2d\u0e35\u0e01\u0e04\u0e23\u0e31\u0e49\u0e07",image_alt_text:"\u0e23\u0e2b\u0e31\u0e2a\u0e20\u0e32\u0e1e reCAPTCHA",
-privacy_and_terms:"\u0e19\u0e42\u0e22\u0e1a\u0e32\u0e22\u0e2a\u0e48\u0e27\u0e19\u0e1a\u0e38\u0e04\u0e04\u0e25\u0e41\u0e25\u0e30\u0e02\u0e49\u0e2d\u0e01\u0e33\u0e2b\u0e19\u0e14"},tr:{visual_challenge:"G\u00f6rsel sorgu al",audio_challenge:"Sesli sorgu al",refresh_btn:"Yeniden y\u00fckle",instructions_visual:"Metni yaz\u0131n:",instructions_audio:"Duydu\u011funuzu yaz\u0131n:",help_btn:"Yard\u0131m",play_again:"Sesi tekrar \u00e7al",cant_hear_this:"Sesi MP3 olarak indir",incorrect_try_again:"Yanl\u0131\u015f. Tekrar deneyin.",
-image_alt_text:"reCAPTCHA sorusu resmi",privacy_and_terms:"Gizlilik ve \u015eartlar"},uk:{visual_challenge:"\u041e\u0442\u0440\u0438\u043c\u0430\u0442\u0438 \u0432\u0456\u0437\u0443\u0430\u043b\u044c\u043d\u0438\u0439 \u0442\u0435\u043a\u0441\u0442",audio_challenge:"\u041e\u0442\u0440\u0438\u043c\u0430\u0442\u0438 \u0430\u0443\u0434\u0456\u043e\u0437\u0430\u043f\u0438\u0441",refresh_btn:"\u041e\u043d\u043e\u0432\u0438\u0442\u0438 \u0442\u0435\u043a\u0441\u0442",instructions_visual:"\u0412\u0432\u0435\u0434\u0456\u0442\u044c \u0442\u0435\u043a\u0441\u0442:",
-instructions_audio:"\u0412\u0432\u0435\u0434\u0456\u0442\u044c \u043f\u043e\u0447\u0443\u0442\u0435:",help_btn:"\u0414\u043e\u0432\u0456\u0434\u043a\u0430",play_again:"\u0412\u0456\u0434\u0442\u0432\u043e\u0440\u0438\u0442\u0438 \u0437\u0430\u043f\u0438\u0441 \u0449\u0435 \u0440\u0430\u0437",cant_hear_this:"\u0417\u0430\u0432\u0430\u043d\u0442\u0430\u0436\u0438\u0442\u0438 \u0437\u0430\u043f\u0438\u0441 \u044f\u043a MP3",incorrect_try_again:"\u041d\u0435\u043f\u0440\u0430\u0432\u0438\u043b\u044c\u043d\u043e. \u0421\u043f\u0440\u043e\u0431\u0443\u0439\u0442\u0435 \u0449\u0435 \u0440\u0430\u0437.",
-image_alt_text:"\u0417\u043e\u0431\u0440\u0430\u0436\u0435\u043d\u043d\u044f \u0437\u0430\u0432\u0434\u0430\u043d\u043d\u044f reCAPTCHA",privacy_and_terms:"\u041a\u043e\u043d\u0444\u0456\u0434\u0435\u043d\u0446\u0456\u0439\u043d\u0456\u0441\u0442\u044c \u0456 \u0443\u043c\u043e\u0432\u0438"},ur:{visual_challenge:"\u0627\u06cc\u06a9 \u0645\u0631\u0626\u06cc \u0686\u06cc\u0644\u0646\u062c \u062d\u0627\u0635\u0644 \u06a9\u0631\u06cc\u06ba",audio_challenge:"\u0627\u06cc\u06a9 \u0622\u0688\u06cc\u0648 \u0686\u06cc\u0644\u0646\u062c \u062d\u0627\u0635\u0644 \u06a9\u0631\u06cc\u06ba",
-refresh_btn:"\u0627\u06cc\u06a9 \u0646\u06cc\u0627 \u0686\u06cc\u0644\u0646\u062c \u062d\u0627\u0635\u0644 \u06a9\u0631\u06cc\u06ba",instructions_visual:"",instructions_audio:"\u062c\u0648 \u0633\u0646\u0627\u0626\u06cc \u062f\u06cc\u062a\u0627 \u06c1\u06d2 \u0648\u06c1 \u0679\u0627\u0626\u067e \u06a9\u0631\u06cc\u06ba:",help_btn:"\u0645\u062f\u062f",play_again:"\u0622\u0648\u0627\u0632 \u062f\u0648\u0628\u0627\u0631\u06c1 \u0686\u0644\u0627\u0626\u06cc\u06ba",cant_hear_this:"\u0622\u0648\u0627\u0632 \u06a9\u0648 MP3 \u06a9\u06d2 \u0628\u0637\u0648\u0631 \u0688\u0627\u0624\u0646 \u0644\u0648\u0688 \u06a9\u0631\u06cc\u06ba",
-incorrect_try_again:"\u063a\u0644\u0637\u06d4 \u062f\u0648\u0628\u0627\u0631\u06c1 \u06a9\u0648\u0634\u0634 \u06a9\u0631\u06cc\u06ba\u06d4",image_alt_text:"reCAPTCHA \u0686\u06cc\u0644\u0646\u062c \u0648\u0627\u0644\u06cc \u0634\u0628\u06cc\u06c1",privacy_and_terms:"\u0631\u0627\u0632\u062f\u0627\u0631\u06cc \u0648 \u0634\u0631\u0627\u0626\u0637"},vi:{visual_challenge:"Nh\u1eadn th\u1eed th\u00e1ch h\u00ecnh \u1ea3nh",audio_challenge:"Nh\u1eadn th\u1eed th\u00e1ch \u00e2m thanh",refresh_btn:"Nh\u1eadn th\u1eed th\u00e1ch m\u1edbi",
-instructions_visual:"Nh\u1eadp v\u0103n b\u1ea3n:",instructions_audio:"Nh\u1eadp n\u1ed9i dung b\u1ea1n nghe th\u1ea5y:",help_btn:"Tr\u1ee3 gi\u00fap",play_again:"Ph\u00e1t l\u1ea1i \u00e2m thanh",cant_hear_this:"T\u1ea3i \u00e2m thanh xu\u1ed1ng d\u01b0\u1edbi d\u1ea1ng MP3",incorrect_try_again:"Kh\u00f4ng ch\u00ednh x\u00e1c. H\u00e3y th\u1eed l\u1ea1i.",image_alt_text:"H\u00ecnh x\u00e1c th\u1ef1c reCAPTCHA",privacy_and_terms:"B\u1ea3o m\u1eadt v\u00e0 \u0111i\u1ec1u kho\u1ea3n"},"zh-CN":ra,"zh-HK":{visual_challenge:"\u56de\u7b54\u5716\u50cf\u9a57\u8b49\u554f\u984c",
-audio_challenge:"\u53d6\u5f97\u8a9e\u97f3\u9a57\u8b49\u554f\u984c",refresh_btn:"\u63db\u4e00\u500b\u9a57\u8b49\u554f\u984c",instructions_visual:"\u8f38\u5165\u6587\u5b57\uff1a",instructions_audio:"\u9375\u5165\u60a8\u6240\u807d\u5230\u7684\uff1a",help_btn:"\u8aaa\u660e",play_again:"\u518d\u6b21\u64ad\u653e\u8072\u97f3",cant_hear_this:"\u5c07\u8072\u97f3\u4e0b\u8f09\u70ba MP3",incorrect_try_again:"\u4e0d\u6b63\u78ba\uff0c\u518d\u8a66\u4e00\u6b21\u3002",image_alt_text:"reCAPTCHA \u9a57\u8b49\u6587\u5b57\u5716\u7247",
-privacy_and_terms:"\u79c1\u96b1\u6b0a\u8207\u689d\u6b3e"},"zh-TW":{visual_challenge:"\u53d6\u5f97\u5716\u7247\u9a57\u8b49\u554f\u984c",audio_challenge:"\u53d6\u5f97\u8a9e\u97f3\u9a57\u8b49\u554f\u984c",refresh_btn:"\u53d6\u5f97\u65b0\u7684\u9a57\u8b49\u554f\u984c",instructions_visual:"\u8acb\u8f38\u5165\u5716\u7247\u4e2d\u7684\u6587\u5b57\uff1a",instructions_audio:"\u8acb\u8f38\u5165\u8a9e\u97f3\u5167\u5bb9\uff1a",help_btn:"\u8aaa\u660e",play_again:"\u518d\u6b21\u64ad\u653e",cant_hear_this:"\u4ee5 MP3 \u683c\u5f0f\u4e0b\u8f09\u8072\u97f3",
-incorrect_try_again:"\u9a57\u8b49\u78bc\u6709\u8aa4\uff0c\u8acb\u518d\u8a66\u4e00\u6b21\u3002",image_alt_text:"reCAPTCHA \u9a57\u8b49\u6587\u5b57\u5716\u7247",privacy_and_terms:"\u96b1\u79c1\u6b0a\u8207\u689d\u6b3e"},zu:{visual_challenge:"Thola inselelo ebonakalayo",audio_challenge:"Thola inselelo yokulalelwayo",refresh_btn:"Thola inselelo entsha",instructions_visual:"",instructions_audio:"Bhala okuzwayo:",help_btn:"Usizo",play_again:"Phinda udlale okulalelwayo futhi",cant_hear_this:"Layisha umsindo njenge-MP3",
-incorrect_try_again:"Akulungile. Zama futhi.",image_alt_text:"umfanekiso oyinselelo we-reCAPTCHA",privacy_and_terms:"Okwangasese kanye nemigomo"},tl:la,he:oa,"in":na,mo:qa,zh:ra};var x=function(a){if(Error.captureStackTrace)Error.captureStackTrace(this,x);else{var b=Error().stack;b&&(this.stack=b)}a&&(this.message=String(a))};u(x,Error);x.prototype.name="CustomError";var ta;var ua=function(a,b){for(var c=a.split("%s"),d="",e=Array.prototype.slice.call(arguments,1);e.length&&1<c.length;)d+=c.shift()+e.shift();return d+c.join("%s")},va=String.prototype.trim?function(a){return a.trim()}:function(a){return a.replace(/^[\s\xa0]+|[\s\xa0]+$/g,"")},Da=function(a){if(!wa.test(a))return a;-1!=a.indexOf("&")&&(a=a.replace(xa,"&amp;"));-1!=a.indexOf("<")&&(a=a.replace(ya,"&lt;"));-1!=a.indexOf(">")&&(a=a.replace(za,"&gt;"));-1!=a.indexOf('"')&&(a=a.replace(Aa,"&quot;"));-1!=a.indexOf("'")&&
-(a=a.replace(Ba,"&#39;"));-1!=a.indexOf("\x00")&&(a=a.replace(Ca,"&#0;"));return a},xa=/&/g,ya=/</g,za=/>/g,Aa=/"/g,Ba=/'/g,Ca=/\x00/g,wa=/[\x00&<>"']/,Ea=function(a,b){return a<b?-1:a>b?1:0},Fa=function(a){return String(a).replace(/\-([a-z])/g,function(a,c){return c.toUpperCase()})},Ga=function(a){var b=q(void 0)?"undefined".replace(/([-()\[\]{}+?*.$\^|,:#<!\\])/g,"\\$1").replace(/\x08/g,"\\x08"):"\\s";return a.replace(new RegExp("(^"+(b?"|["+b+"]+":"")+")([a-z])","g"),function(a,b,e){return b+e.toUpperCase()})};var Ha=function(a,b){b.unshift(a);x.call(this,ua.apply(null,b));b.shift()};u(Ha,x);Ha.prototype.name="AssertionError";
-var Ia=function(a,b,c,d){var e="Assertion failed";if(c)var e=e+(": "+c),g=d;else a&&(e+=": "+a,g=b);throw new Ha(""+e,g||[]);},y=function(a,b,c){a||Ia("",null,b,Array.prototype.slice.call(arguments,2))},Ja=function(a,b){throw new Ha("Failure"+(a?": "+a:""),Array.prototype.slice.call(arguments,1));},Ka=function(a,b,c){q(a)||Ia("Expected string but got %s: %s.",[n(a),a],b,Array.prototype.slice.call(arguments,2));return a},La=function(a,b,c){r(a)||Ia("Expected function but got %s: %s.",[n(a),a],b,Array.prototype.slice.call(arguments,
-2))};var z=Array.prototype,Ma=z.indexOf?function(a,b,c){y(null!=a.length);return z.indexOf.call(a,b,c)}:function(a,b,c){c=null==c?0:0>c?Math.max(0,a.length+c):c;if(q(a))return q(b)&&1==b.length?a.indexOf(b,c):-1;for(;c<a.length;c++)if(c in a&&a[c]===b)return c;return-1},Na=z.forEach?function(a,b,c){y(null!=a.length);z.forEach.call(a,b,c)}:function(a,b,c){for(var d=a.length,e=q(a)?a.split(""):a,g=0;g<d;g++)g in e&&b.call(c,e[g],g,a)},Oa=z.map?function(a,b,c){y(null!=a.length);return z.map.call(a,b,c)}:
-function(a,b,c){for(var d=a.length,e=Array(d),g=q(a)?a.split(""):a,f=0;f<d;f++)f in g&&(e[f]=b.call(c,g[f],f,a));return e},Pa=z.some?function(a,b,c){y(null!=a.length);return z.some.call(a,b,c)}:function(a,b,c){for(var d=a.length,e=q(a)?a.split(""):a,g=0;g<d;g++)if(g in e&&b.call(c,e[g],g,a))return!0;return!1},Qa=function(a,b){var c=Ma(a,b),d;if(d=0<=c)y(null!=a.length),z.splice.call(a,c,1);return d},Ra=function(a){var b=a.length;if(0<b){for(var c=Array(b),d=0;d<b;d++)c[d]=a[d];return c}return[]},
-Sa=function(a,b,c){y(null!=a.length);return 2>=arguments.length?z.slice.call(a,b):z.slice.call(a,b,c)};var Ta=function(a,b){for(var c in a)b.call(void 0,a[c],c,a)},Ua=function(a){var b=[],c=0,d;for(d in a)b[c++]=d;return b},Va=function(a){for(var b in a)return!1;return!0},Xa=function(){var a=Wa()?k.google_ad:null,b={},c;for(c in a)b[c]=a[c];return b},Ya="constructor hasOwnProperty isPrototypeOf propertyIsEnumerable toLocaleString toString valueOf".split(" "),Za=function(a,b){for(var c,d,e=1;e<arguments.length;e++){d=arguments[e];for(c in d)a[c]=d[c];for(var g=0;g<Ya.length;g++)c=Ya[g],Object.prototype.hasOwnProperty.call(d,
-c)&&(a[c]=d[c])}},$a=function(a){var b=arguments.length;if(1==b&&p(arguments[0]))return $a.apply(null,arguments[0]);for(var c={},d=0;d<b;d++)c[arguments[d]]=!0;return c};var A;t:{var ab=k.navigator;if(ab){var bb=ab.userAgent;if(bb){A=bb;break t}}A=""}var B=function(a){return-1!=A.indexOf(a)};var cb=B("Opera")||B("OPR"),C=B("Trident")||B("MSIE"),D=B("Gecko")&&-1==A.toLowerCase().indexOf("webkit")&&!(B("Trident")||B("MSIE")),E=-1!=A.toLowerCase().indexOf("webkit"),db=function(){var a=k.document;return a?a.documentMode:void 0},eb=function(){var a="",b;if(cb&&k.opera)return a=k.opera.version,r(a)?a():a;D?b=/rv\:([^\);]+)(\)|;)/:C?b=/\b(?:MSIE|rv)[: ]([^\);]+)(\)|;)/:E&&(b=/WebKit\/(\S+)/);b&&(a=(a=b.exec(A))?a[1]:"");return C&&(b=db(),b>parseFloat(a))?String(b):a}(),fb={},F=function(a){var b;
-if(!(b=fb[a])){b=0;for(var c=va(String(eb)).split("."),d=va(String(a)).split("."),e=Math.max(c.length,d.length),g=0;0==b&&g<e;g++){var f=c[g]||"",m=d[g]||"",$=RegExp("(\\d*)(\\D*)","g"),K=RegExp("(\\d*)(\\D*)","g");do{var G=$.exec(f)||["","",""],aa=K.exec(m)||["","",""];if(0==G[0].length&&0==aa[0].length)break;b=Ea(0==G[1].length?0:parseInt(G[1],10),0==aa[1].length?0:parseInt(aa[1],10))||Ea(0==G[2].length,0==aa[2].length)||Ea(G[2],aa[2])}while(0==b)}b=fb[a]=0<=b}return b},gb=k.document,hb=gb&&C?db()||
-("CSS1Compat"==gb.compatMode?parseInt(eb,10):5):void 0;var ib=function(a){if(8192>a.length)return String.fromCharCode.apply(null,a);for(var b="",c=0;c<a.length;c+=8192)var d=Sa(a,c,c+8192),b=b+String.fromCharCode.apply(null,d);return b},jb=function(a){return Oa(a,function(a){a=a.toString(16);return 1<a.length?a:"0"+a}).join("")};var kb=null,lb=null,mb=function(a){if(!kb){kb={};lb={};for(var b=0;65>b;b++)kb[b]="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=".charAt(b),lb[kb[b]]=b,62<=b&&(lb["ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_.".charAt(b)]=b)}for(var b=lb,c=[],d=0;d<a.length;){var e=b[a.charAt(d++)],g=d<a.length?b[a.charAt(d)]:0;++d;var f=d<a.length?b[a.charAt(d)]:64;++d;var m=d<a.length?b[a.charAt(d)]:64;++d;if(null==e||null==g||null==f||null==m)throw Error();c.push(e<<2|g>>
-4);64!=f&&(c.push(g<<4&240|f>>2),64!=m&&c.push(f<<6&192|m))}return c};var H=function(){this.disposed_=this.disposed_;this.onDisposeCallbacks_=this.onDisposeCallbacks_};H.prototype.disposed_=!1;H.prototype.dispose=function(){this.disposed_||(this.disposed_=!0,this.disposeInternal())};var nb=function(a,b){a.onDisposeCallbacks_||(a.onDisposeCallbacks_=[]);a.onDisposeCallbacks_.push(l(void 0)?s(b,void 0):b)};H.prototype.disposeInternal=function(){if(this.onDisposeCallbacks_)for(;this.onDisposeCallbacks_.length;)this.onDisposeCallbacks_.shift()()};
-var ob=function(a){a&&"function"==typeof a.dispose&&a.dispose()};var pb=!C||C&&9<=hb;!D&&!C||C&&C&&9<=hb||D&&F("1.9.1");C&&F("9");var sb=function(a){return a?new qb(rb(a)):ta||(ta=new qb)},tb=function(a,b){return q(b)?a.getElementById(b):b},vb=function(a,b){Ta(b,function(b,d){"style"==d?a.style.cssText=b:"class"==d?a.className=b:"for"==d?a.htmlFor=b:d in ub?a.setAttribute(ub[d],b):0==d.lastIndexOf("aria-",0)||0==d.lastIndexOf("data-",0)?a.setAttribute(d,b):a[d]=b})},ub={cellpadding:"cellPadding",cellspacing:"cellSpacing",colspan:"colSpan",frameborder:"frameBorder",height:"height",maxlength:"maxLength",role:"role",rowspan:"rowSpan",
-type:"type",usemap:"useMap",valign:"vAlign",width:"width"},xb=function(a,b,c){return wb(document,arguments)},wb=function(a,b){var c=b[0],d=b[1];if(!pb&&d&&(d.name||d.type)){c=["<",c];d.name&&c.push(' name="',Da(d.name),'"');if(d.type){c.push(' type="',Da(d.type),'"');var e={};Za(e,d);delete e.type;d=e}c.push(">");c=c.join("")}c=a.createElement(c);d&&(q(d)?c.className=d:p(d)?c.className=d.join(" "):vb(c,d));2<b.length&&yb(a,c,b);return c},yb=function(a,b,c){function d(c){c&&b.appendChild(q(c)?a.createTextNode(c):
-c)}for(var e=2;e<c.length;e++){var g=c[e];!ca(g)||da(g)&&0<g.nodeType?d(g):Na(zb(g)?Ra(g):g,d)}},Ab=function(a){for(var b;b=a.firstChild;)a.removeChild(b)},Bb=function(a){a&&a.parentNode&&a.parentNode.removeChild(a)},rb=function(a){y(a,"Node cannot be null or undefined.");return 9==a.nodeType?a:a.ownerDocument||a.document},zb=function(a){if(a&&"number"==typeof a.length){if(da(a))return"function"==typeof a.item||"string"==typeof a.item;if(r(a))return"function"==typeof a.item}return!1},qb=function(a){this.document_=
-a||k.document||document};h=qb.prototype;h.getDomHelper=sb;h.getElement=function(a){return tb(this.document_,a)};h.$=qb.prototype.getElement;h.createDom=function(a,b,c){return wb(this.document_,arguments)};h.createElement=function(a){return this.document_.createElement(a)};h.createTextNode=function(a){return this.document_.createTextNode(String(a))};h.appendChild=function(a,b){a.appendChild(b)};var Cb=function(a){k.setTimeout(function(){throw a;},0)},Db,Eb=function(){var a=k.MessageChannel;"undefined"===typeof a&&"undefined"!==typeof window&&window.postMessage&&window.addEventListener&&(a=function(){var a=document.createElement("iframe");a.style.display="none";a.src="";document.documentElement.appendChild(a);var b=a.contentWindow,a=b.document;a.open();a.write("");a.close();var c="callImmediate"+Math.random(),d="file:"==b.location.protocol?"*":b.location.protocol+"//"+b.location.host,a=s(function(a){if(("*"==
-d||a.origin==d)&&a.data==c)this.port1.onmessage()},this);b.addEventListener("message",a,!1);this.port1={};this.port2={postMessage:function(){b.postMessage(c,d)}}});if("undefined"!==typeof a&&!B("Trident")&&!B("MSIE")){var b=new a,c={},d=c;b.port1.onmessage=function(){if(l(c.next)){c=c.next;var a=c.cb;c.cb=null;a()}};return function(a){d.next={cb:a};d=d.next;b.port2.postMessage(0)}}return"undefined"!==typeof document&&"onreadystatechange"in document.createElement("script")?function(a){var b=document.createElement("script");
-b.onreadystatechange=function(){b.onreadystatechange=null;b.parentNode.removeChild(b);b=null;a();a=null};document.documentElement.appendChild(b)}:function(a){k.setTimeout(a,0)}};var Kb=function(a,b){Fb||Gb();Hb||(Fb(),Hb=!0);Ib.push(new Jb(a,b))},Fb,Gb=function(){if(k.Promise&&k.Promise.resolve){var a=k.Promise.resolve();Fb=function(){a.then(Lb)}}else Fb=function(){var a=Lb;!r(k.setImmediate)||k.Window&&k.Window.prototype.setImmediate==k.setImmediate?(Db||(Db=Eb()),Db(a)):k.setImmediate(a)}},Hb=!1,Ib=[],Lb=function(){for(;Ib.length;){var a=Ib;Ib=[];for(var b=0;b<a.length;b++){var c=a[b];try{c.fn.call(c.scope)}catch(d){Cb(d)}}}Hb=!1},Jb=function(a,b){this.fn=a;this.scope=
-b};var Mb=function(a){a.prototype.then=a.prototype.then;a.prototype.$goog_Thenable=!0},Nb=function(a){if(!a)return!1;try{return!!a.$goog_Thenable}catch(b){return!1}};var L=function(a,b){this.state_=0;this.result_=void 0;this.callbackEntries_=this.parent_=null;this.hadUnhandledRejection_=this.executing_=!1;try{var c=this;a.call(b,function(a){I(c,2,a)},function(a){if(!(a instanceof J))try{if(a instanceof Error)throw a;throw Error("Promise rejected.");}catch(b){}I(c,3,a)})}catch(d){I(this,3,d)}};
-L.prototype.then=function(a,b,c){null!=a&&La(a,"opt_onFulfilled should be a function.");null!=b&&La(b,"opt_onRejected should be a function. Did you pass opt_context as the second argument instead of the third?");return Ob(this,r(a)?a:null,r(b)?b:null,c)};Mb(L);L.prototype.cancel=function(a){0==this.state_&&Kb(function(){var b=new J(a);Pb(this,b)},this)};
-var Pb=function(a,b){if(0==a.state_)if(a.parent_){var c=a.parent_;if(c.callbackEntries_){for(var d=0,e=-1,g=0,f;f=c.callbackEntries_[g];g++)if(f=f.child)if(d++,f==a&&(e=g),0<=e&&1<d)break;0<=e&&(0==c.state_&&1==d?Pb(c,b):(d=c.callbackEntries_.splice(e,1)[0],d.child&&Qb(c),d.onRejected(b)))}}else I(a,3,b)},Sb=function(a,b){a.callbackEntries_&&a.callbackEntries_.length||2!=a.state_&&3!=a.state_||Rb(a);a.callbackEntries_||(a.callbackEntries_=[]);a.callbackEntries_.push(b)},Ob=function(a,b,c,d){var e=
-{child:null,onFulfilled:null,onRejected:null};e.child=new L(function(a,f){e.onFulfilled=b?function(c){try{var e=b.call(d,c);a(e)}catch(K){f(K)}}:a;e.onRejected=c?function(b){try{var e=c.call(d,b);!l(e)&&b instanceof J?f(b):a(e)}catch(K){f(K)}}:f});e.child.parent_=a;Sb(a,e);return e.child};L.prototype.unblockAndFulfill_=function(a){y(1==this.state_);this.state_=0;I(this,2,a)};L.prototype.unblockAndReject_=function(a){y(1==this.state_);this.state_=0;I(this,3,a)};
-var I=function(a,b,c){if(0==a.state_){if(a==c)b=3,c=new TypeError("Promise cannot resolve to itself");else{if(Nb(c)){a.state_=1;c.then(a.unblockAndFulfill_,a.unblockAndReject_,a);return}if(da(c))try{var d=c.then;if(r(d)){Tb(a,c,d);return}}catch(e){b=3,c=e}}a.result_=c;a.state_=b;Rb(a);3!=b||c instanceof J||Ub(a,c)}},Tb=function(a,b,c){a.state_=1;var d=!1,e=function(b){d||(d=!0,a.unblockAndFulfill_(b))},g=function(b){d||(d=!0,a.unblockAndReject_(b))};try{c.call(b,e,g)}catch(f){g(f)}},Rb=function(a){a.executing_||
-(a.executing_=!0,Kb(a.executeCallbacks_,a))};L.prototype.executeCallbacks_=function(){for(;this.callbackEntries_&&this.callbackEntries_.length;){var a=this.callbackEntries_;this.callbackEntries_=[];for(var b=0;b<a.length;b++){var c=a[b],d=this.result_;if(2==this.state_)c.onFulfilled(d);else c.child&&Qb(this),c.onRejected(d)}}this.executing_=!1};
-var Qb=function(a){for(;a&&a.hadUnhandledRejection_;a=a.parent_)a.hadUnhandledRejection_=!1},Ub=function(a,b){a.hadUnhandledRejection_=!0;Kb(function(){a.hadUnhandledRejection_&&Vb.call(null,b)})},Vb=Cb,J=function(a){x.call(this,a)};u(J,x);J.prototype.name="cancel";/*
- Portions of this code are from MochiKit, received by
- The Closure Authors under the MIT license. All other code is Copyright
- 2005-2009 The Closure Authors. All Rights Reserved.
-*/
-var M=function(a,b){this.sequence_=[];this.onCancelFunction_=a;this.defaultScope_=b||null;this.hadError_=this.fired_=!1;this.result_=void 0;this.silentlyCanceled_=this.blocking_=this.blocked_=!1;this.unhandledErrorId_=0;this.parent_=null;this.branches_=0};
-M.prototype.cancel=function(a){if(this.fired_)this.result_ instanceof M&&this.result_.cancel();else{if(this.parent_){var b=this.parent_;delete this.parent_;a?b.cancel(a):(b.branches_--,0>=b.branches_&&b.cancel())}this.onCancelFunction_?this.onCancelFunction_.call(this.defaultScope_,this):this.silentlyCanceled_=!0;this.fired_||Wb(this,new Xb)}};M.prototype.continue_=function(a,b){this.blocked_=!1;Yb(this,a,b)};
-var Yb=function(a,b,c){a.fired_=!0;a.result_=c;a.hadError_=!b;Zb(a)},ac=function(a){if(a.fired_){if(!a.silentlyCanceled_)throw new $b;a.silentlyCanceled_=!1}};M.prototype.callback=function(a){ac(this);bc(a);Yb(this,!0,a)};var Wb=function(a,b){ac(a);bc(b);Yb(a,!1,b)},bc=function(a){y(!(a instanceof M),"An execution sequence may not be initiated with a blocking Deferred.")},cc=function(a,b,c,d){y(!a.blocking_,"Blocking Deferreds can not be re-used");a.sequence_.push([b,c,d]);a.fired_&&Zb(a)};
-M.prototype.then=function(a,b,c){var d,e,g=new L(function(a,b){d=a;e=b});cc(this,d,function(a){a instanceof Xb?g.cancel():e(a)});return g.then(a,b,c)};Mb(M);
-var dc=function(a){return Pa(a.sequence_,function(a){return r(a[1])})},Zb=function(a){if(a.unhandledErrorId_&&a.fired_&&dc(a)){var b=a.unhandledErrorId_,c=ec[b];c&&(k.clearTimeout(c.id_),delete ec[b]);a.unhandledErrorId_=0}a.parent_&&(a.parent_.branches_--,delete a.parent_);for(var b=a.result_,d=c=!1;a.sequence_.length&&!a.blocked_;){var e=a.sequence_.shift(),g=e[0],f=e[1],e=e[2];if(g=a.hadError_?f:g)try{var m=g.call(e||a.defaultScope_,b);l(m)&&(a.hadError_=a.hadError_&&(m==b||m instanceof Error),
-a.result_=b=m);Nb(b)&&(d=!0,a.blocked_=!0)}catch($){b=$,a.hadError_=!0,dc(a)||(c=!0)}}a.result_=b;d&&(m=s(a.continue_,a,!0),d=s(a.continue_,a,!1),b instanceof M?(cc(b,m,d),b.blocking_=!0):b.then(m,d));c&&(b=new fc(b),ec[b.id_]=b,a.unhandledErrorId_=b.id_)},$b=function(){x.call(this)};u($b,x);$b.prototype.message="Deferred has already fired";$b.prototype.name="AlreadyCalledError";var Xb=function(){x.call(this)};u(Xb,x);Xb.prototype.message="Deferred was canceled";Xb.prototype.name="CanceledError";
-var fc=function(a){this.id_=k.setTimeout(s(this.throwError,this),0);this.error_=a};fc.prototype.throwError=function(){y(ec[this.id_],"Cannot throw an error that is not scheduled.");delete ec[this.id_];throw this.error_;};var ec={};var kc=function(a){var b={},c=b.document||document,d=document.createElement("SCRIPT"),e={script_:d,timeout_:void 0},g=new M(gc,e),f=null,m=null!=b.timeout?b.timeout:5E3;0<m&&(f=window.setTimeout(function(){hc(d,!0);Wb(g,new ic(1,"Timeout reached for loading script "+a))},m),e.timeout_=f);d.onload=d.onreadystatechange=function(){d.readyState&&"loaded"!=d.readyState&&"complete"!=d.readyState||(hc(d,b.cleanupWhenDone||!1,f),g.callback(null))};d.onerror=function(){hc(d,!0,f);Wb(g,new ic(0,"Error while loading script "+
-a))};vb(d,{type:"text/javascript",charset:"UTF-8",src:a});jc(c).appendChild(d);return g},jc=function(a){var b=a.getElementsByTagName("HEAD");return b&&0!=b.length?b[0]:a.documentElement},gc=function(){if(this&&this.script_){var a=this.script_;a&&"SCRIPT"==a.tagName&&hc(a,!0,this.timeout_)}},hc=function(a,b,c){null!=c&&k.clearTimeout(c);a.onload=ba;a.onerror=ba;a.onreadystatechange=ba;b&&window.setTimeout(function(){Bb(a)},0)},ic=function(a,b){var c="Jsloader error (code #"+a+")";b&&(c+=": "+b);x.call(this,
-c);this.code=a};u(ic,x);var lc=function(a){lc[" "](a);return a};lc[" "]=ba;var mc=!C||C&&9<=hb,nc=C&&!F("9");!E||F("528");D&&F("1.9b")||C&&F("8")||cb&&F("9.5")||E&&F("528");D&&!F("8")||C&&F("9");var N=function(a,b){this.type=a;this.currentTarget=this.target=b;this.defaultPrevented=this.propagationStopped_=!1;this.returnValue_=!0};N.prototype.disposeInternal=function(){};N.prototype.dispose=function(){};N.prototype.preventDefault=function(){this.defaultPrevented=!0;this.returnValue_=!1};var O=function(a,b){N.call(this,a?a.type:"");this.relatedTarget=this.currentTarget=this.target=null;this.charCode=this.keyCode=this.button=this.screenY=this.screenX=this.clientY=this.clientX=this.offsetY=this.offsetX=0;this.metaKey=this.shiftKey=this.altKey=this.ctrlKey=!1;this.event_=this.state=null;if(a){var c=this.type=a.type;this.target=a.target||a.srcElement;this.currentTarget=b;var d=a.relatedTarget;if(d){if(D){var e;t:{try{lc(d.nodeName);e=!0;break t}catch(g){}e=!1}e||(d=null)}}else"mouseover"==
-c?d=a.fromElement:"mouseout"==c&&(d=a.toElement);this.relatedTarget=d;this.offsetX=E||void 0!==a.offsetX?a.offsetX:a.layerX;this.offsetY=E||void 0!==a.offsetY?a.offsetY:a.layerY;this.clientX=void 0!==a.clientX?a.clientX:a.pageX;this.clientY=void 0!==a.clientY?a.clientY:a.pageY;this.screenX=a.screenX||0;this.screenY=a.screenY||0;this.button=a.button;this.keyCode=a.keyCode||0;this.charCode=a.charCode||("keypress"==c?a.keyCode:0);this.ctrlKey=a.ctrlKey;this.altKey=a.altKey;this.shiftKey=a.shiftKey;this.metaKey=
-a.metaKey;this.state=a.state;this.event_=a;a.defaultPrevented&&this.preventDefault()}};u(O,N);O.prototype.preventDefault=function(){O.superClass_.preventDefault.call(this);var a=this.event_;if(a.preventDefault)a.preventDefault();else if(a.returnValue=!1,nc)try{if(a.ctrlKey||112<=a.keyCode&&123>=a.keyCode)a.keyCode=-1}catch(b){}};O.prototype.disposeInternal=function(){};var oc="closure_listenable_"+(1E6*Math.random()|0),pc=0;var qc=function(a,b,c,d,e){this.listener=a;this.proxy=null;this.src=b;this.type=c;this.capture=!!d;this.handler=e;this.key=++pc;this.removed=this.callOnce=!1},rc=function(a){a.removed=!0;a.listener=null;a.proxy=null;a.src=null;a.handler=null};var P=function(a){this.src=a;this.listeners={};this.typeCount_=0};P.prototype.add=function(a,b,c,d,e){var g=a.toString();a=this.listeners[g];a||(a=this.listeners[g]=[],this.typeCount_++);var f=sc(a,b,d,e);-1<f?(b=a[f],c||(b.callOnce=!1)):(b=new qc(b,this.src,g,!!d,e),b.callOnce=c,a.push(b));return b};
-P.prototype.remove=function(a,b,c,d){a=a.toString();if(!(a in this.listeners))return!1;var e=this.listeners[a];b=sc(e,b,c,d);return-1<b?(rc(e[b]),y(null!=e.length),z.splice.call(e,b,1),0==e.length&&(delete this.listeners[a],this.typeCount_--),!0):!1};var tc=function(a,b){var c=b.type;if(!(c in a.listeners))return!1;var d=Qa(a.listeners[c],b);d&&(rc(b),0==a.listeners[c].length&&(delete a.listeners[c],a.typeCount_--));return d};
-P.prototype.removeAll=function(a){a=a&&a.toString();var b=0,c;for(c in this.listeners)if(!a||c==a){for(var d=this.listeners[c],e=0;e<d.length;e++)++b,rc(d[e]);delete this.listeners[c];this.typeCount_--}return b};P.prototype.getListener=function(a,b,c,d){a=this.listeners[a.toString()];var e=-1;a&&(e=sc(a,b,c,d));return-1<e?a[e]:null};var sc=function(a,b,c,d){for(var e=0;e<a.length;++e){var g=a[e];if(!g.removed&&g.listener==b&&g.capture==!!c&&g.handler==d)return e}return-1};var uc="closure_lm_"+(1E6*Math.random()|0),vc={},wc=0,xc=function(a,b,c,d,e){if(p(b)){for(var g=0;g<b.length;g++)xc(a,b[g],c,d,e);return null}c=yc(c);if(a&&a[oc])a=a.listen(b,c,d,e);else{if(!b)throw Error("Invalid event type");var g=!!d,f=zc(a);f||(a[uc]=f=new P(a));c=f.add(b,c,!1,d,e);c.proxy||(d=Ac(),c.proxy=d,d.src=a,d.listener=c,a.addEventListener?a.addEventListener(b.toString(),d,g):a.attachEvent(Bc(b.toString()),d),wc++);a=c}return a},Ac=function(){var a=Cc,b=mc?function(c){return a.call(b.src,
-b.listener,c)}:function(c){c=a.call(b.src,b.listener,c);if(!c)return c};return b},Dc=function(a,b,c,d,e){if(p(b))for(var g=0;g<b.length;g++)Dc(a,b[g],c,d,e);else c=yc(c),a&&a[oc]?a.unlisten(b,c,d,e):a&&(a=zc(a))&&(b=a.getListener(b,c,!!d,e))&&Ec(b)},Ec=function(a){if("number"==typeof a||!a||a.removed)return!1;var b=a.src;if(b&&b[oc])return tc(b.eventTargetListeners_,a);var c=a.type,d=a.proxy;b.removeEventListener?b.removeEventListener(c,d,a.capture):b.detachEvent&&b.detachEvent(Bc(c),d);wc--;(c=zc(b))?
-(tc(c,a),0==c.typeCount_&&(c.src=null,b[uc]=null)):rc(a);return!0},Bc=function(a){return a in vc?vc[a]:vc[a]="on"+a},Gc=function(a,b,c,d){var e=1;if(a=zc(a))if(b=a.listeners[b.toString()])for(b=b.concat(),a=0;a<b.length;a++){var g=b[a];g&&g.capture==c&&!g.removed&&(e&=!1!==Fc(g,d))}return Boolean(e)},Fc=function(a,b){var c=a.listener,d=a.handler||a.src;a.callOnce&&Ec(a);return c.call(d,b)},Cc=function(a,b){if(a.removed)return!0;if(!mc){var c;if(!(c=b))t:{c=["window","event"];for(var d=k,e;e=c.shift();)if(null!=
-d[e])d=d[e];else{c=null;break t}c=d}e=c;c=new O(e,this);d=!0;if(!(0>e.keyCode||void 0!=e.returnValue)){t:{var g=!1;if(0==e.keyCode)try{e.keyCode=-1;break t}catch(f){g=!0}if(g||void 0==e.returnValue)e.returnValue=!0}e=[];for(g=c.currentTarget;g;g=g.parentNode)e.push(g);for(var g=a.type,m=e.length-1;!c.propagationStopped_&&0<=m;m--)c.currentTarget=e[m],d&=Gc(e[m],g,!0,c);for(m=0;!c.propagationStopped_&&m<e.length;m++)c.currentTarget=e[m],d&=Gc(e[m],g,!1,c)}return d}return Fc(a,new O(b,this))},zc=function(a){a=
-a[uc];return a instanceof P?a:null},Hc="__closure_events_fn_"+(1E9*Math.random()>>>0),yc=function(a){y(a,"Listener can not be null.");if(r(a))return a;y(a.handleEvent,"An object listener must have handleEvent method.");a[Hc]||(a[Hc]=function(b){return a.handleEvent(b)});return a[Hc]};var Q=function(a){H.call(this);this.handler_=a;this.keys_={}};u(Q,H);var Ic=[];h=Q.prototype;h.listen=function(a,b,c,d){p(b)||(b&&(Ic[0]=b.toString()),b=Ic);for(var e=0;e<b.length;e++){var g=xc(a,b[e],c||this.handleEvent,d||!1,this.handler_||this);if(!g)break;this.keys_[g.key]=g}return this};
-h.unlisten=function(a,b,c,d,e){if(p(b))for(var g=0;g<b.length;g++)this.unlisten(a,b[g],c,d,e);else c=c||this.handleEvent,e=e||this.handler_||this,c=yc(c),d=!!d,b=a&&a[oc]?a.getListener(b,c,d,e):a?(a=zc(a))?a.getListener(b,c,d,e):null:null,b&&(Ec(b),delete this.keys_[b.key]);return this};h.removeAll=function(){Ta(this.keys_,Ec);this.keys_={}};h.disposeInternal=function(){Q.superClass_.disposeInternal.call(this);this.removeAll()};
-h.handleEvent=function(){throw Error("EventHandler.handleEvent not implemented");};var R=function(){H.call(this);this.eventTargetListeners_=new P(this);this.actualEventTarget_=this;this.parentEventTarget_=null};u(R,H);R.prototype[oc]=!0;h=R.prototype;h.setParentEventTarget=function(a){this.parentEventTarget_=a};h.addEventListener=function(a,b,c,d){xc(this,a,b,c,d)};h.removeEventListener=function(a,b,c,d){Dc(this,a,b,c,d)};
-h.dispatchEvent=function(a){Jc(this);var b,c=this.parentEventTarget_;if(c){b=[];for(var d=1;c;c=c.parentEventTarget_)b.push(c),y(1E3>++d,"infinite loop")}c=this.actualEventTarget_;d=a.type||a;if(q(a))a=new N(a,c);else if(a instanceof N)a.target=a.target||c;else{var e=a;a=new N(d,c);Za(a,e)}var e=!0,g;if(b)for(var f=b.length-1;!a.propagationStopped_&&0<=f;f--)g=a.currentTarget=b[f],e=Kc(g,d,!0,a)&&e;a.propagationStopped_||(g=a.currentTarget=c,e=Kc(g,d,!0,a)&&e,a.propagationStopped_||(e=Kc(g,d,!1,a)&&
-e));if(b)for(f=0;!a.propagationStopped_&&f<b.length;f++)g=a.currentTarget=b[f],e=Kc(g,d,!1,a)&&e;return e};h.disposeInternal=function(){R.superClass_.disposeInternal.call(this);this.eventTargetListeners_&&this.eventTargetListeners_.removeAll(void 0);this.parentEventTarget_=null};h.listen=function(a,b,c,d){Jc(this);return this.eventTargetListeners_.add(String(a),b,!1,c,d)};h.unlisten=function(a,b,c,d){return this.eventTargetListeners_.remove(String(a),b,c,d)};
-var Kc=function(a,b,c,d){b=a.eventTargetListeners_.listeners[String(b)];if(!b)return!0;b=b.concat();for(var e=!0,g=0;g<b.length;++g){var f=b[g];if(f&&!f.removed&&f.capture==c){var m=f.listener,$=f.handler||f.src;f.callOnce&&tc(a.eventTargetListeners_,f);e=!1!==m.call($,d)&&e}}return e&&0!=d.returnValue_};R.prototype.getListener=function(a,b,c,d){return this.eventTargetListeners_.getListener(String(a),b,c,d)};var Jc=function(a){y(a.eventTargetListeners_,"Event target is not initialized. Did you call the superclass (goog.events.EventTarget) constructor?")};var S=function(a){R.call(this);this.imageIdToRequestMap_={};this.imageIdToImageMap_={};this.handler_=new Q(this);this.parent_=a};u(S,R);var Lc=[C&&!F("11")?"readystatechange":"load","abort","error"],Mc=function(a,b,c){(c=q(c)?c:c.src)&&(a.imageIdToRequestMap_[b]={src:c,corsRequestType:l(void 0)?void 0:null})};
-S.prototype.start=function(){var a=this.imageIdToRequestMap_;Na(Ua(a),function(b){var c=a[b];if(c&&(delete a[b],!this.disposed_)){var d;d=this.parent_?sb(this.parent_).createDom("img"):new Image;c.corsRequestType&&(d.crossOrigin=c.corsRequestType);this.handler_.listen(d,Lc,this.onNetworkEvent_);this.imageIdToImageMap_[b]=d;d.id=b;d.src=c.src}},this)};
-S.prototype.onNetworkEvent_=function(a){var b=a.currentTarget;if(b){if("readystatechange"==a.type)if("complete"==b.readyState)a.type="load";else return;"undefined"==typeof b.naturalWidth&&("load"==a.type?(b.naturalWidth=b.width,b.naturalHeight=b.height):(b.naturalWidth=0,b.naturalHeight=0));this.dispatchEvent({type:a.type,target:b});!this.disposed_&&(a=b.id,delete this.imageIdToRequestMap_[a],b=this.imageIdToImageMap_[a])&&(delete this.imageIdToImageMap_[a],this.handler_.unlisten(b,Lc,this.onNetworkEvent_),
-Va(this.imageIdToImageMap_)&&Va(this.imageIdToRequestMap_)&&this.dispatchEvent("complete"))}};S.prototype.disposeInternal=function(){delete this.imageIdToRequestMap_;delete this.imageIdToImageMap_;ob(this.handler_);S.superClass_.disposeInternal.call(this)};var T=function(){};T.getInstance=function(){return T.instance_?T.instance_:T.instance_=new T};T.prototype.nextId_=0;var U=function(a){R.call(this);this.dom_=a||sb();this.id_=null;this.inDocument_=!1;this.element_=null;this.googUiComponentHandler_=void 0;this.childIndex_=this.children_=this.parent_=null;this.wasDecorated_=!1};u(U,R);h=U.prototype;h.idGenerator_=T.getInstance();h.getElement=function(){return this.element_};h.setParentEventTarget=function(a){if(this.parent_&&this.parent_!=a)throw Error("Method not supported");U.superClass_.setParentEventTarget.call(this,a)};h.getDomHelper=function(){return this.dom_};
-h.createDom=function(){this.element_=this.dom_.createElement("div")};
-var Oc=function(a,b){if(a.inDocument_)throw Error("Component already rendered");a.element_||a.createDom();b?b.insertBefore(a.element_,null):a.dom_.document_.body.appendChild(a.element_);a.parent_&&!a.parent_.inDocument_||Nc(a)},Nc=function(a){a.inDocument_=!0;Pc(a,function(a){!a.inDocument_&&a.getElement()&&Nc(a)})},Qc=function(a){Pc(a,function(a){a.inDocument_&&Qc(a)});a.googUiComponentHandler_&&a.googUiComponentHandler_.removeAll();a.inDocument_=!1};
-U.prototype.disposeInternal=function(){this.inDocument_&&Qc(this);this.googUiComponentHandler_&&(this.googUiComponentHandler_.dispose(),delete this.googUiComponentHandler_);Pc(this,function(a){a.dispose()});!this.wasDecorated_&&this.element_&&Bb(this.element_);this.parent_=this.element_=this.childIndex_=this.children_=null;U.superClass_.disposeInternal.call(this)};var Pc=function(a,b){a.children_&&Na(a.children_,b,void 0)};
-U.prototype.removeChild=function(a,b){if(a){var c=q(a)?a:a.id_||(a.id_=":"+(a.idGenerator_.nextId_++).toString(36)),d;this.childIndex_&&c?(d=this.childIndex_,d=(c in d?d[c]:void 0)||null):d=null;a=d;if(c&&a){d=this.childIndex_;c in d&&delete d[c];Qa(this.children_,a);b&&(Qc(a),a.element_&&Bb(a.element_));c=a;if(null==c)throw Error("Unable to set parent component");c.parent_=null;U.superClass_.setParentEventTarget.call(c,null)}}if(!a)throw Error("Child is not in parent component");return a};var V=function(a,b,c){U.call(this,c);this.captchaImage_=a;this.adImage_=b&&300==b.naturalWidth&&57==b.naturalHeight?b:null};u(V,U);V.prototype.createDom=function(){V.superClass_.createDom.call(this);var a=this.getElement();this.captchaImage_.alt=W.image_alt_text;this.getDomHelper().appendChild(a,this.captchaImage_);this.adImage_&&(this.adImage_.alt=W.image_alt_text,this.getDomHelper().appendChild(a,this.adImage_),this.adImage_&&Rc(this.adImage_)&&(a.innerHTML+='<div id="recaptcha-ad-choices"><div class="recaptcha-ad-choices-collapsed"><img height="15" width="30" alt="AdChoices" border="0" src="//www.gstatic.com/recaptcha/api/img/adicon.png"/></div><div class="recaptcha-ad-choices-expanded"><a href="https://support.google.com/adsense/troubleshooter/1631343" target="_blank"><img height="15" width="75" alt="AdChoices" border="0" src="//www.gstatic.com/recaptcha/api/img/adchoices.png"/></a></div></div>'))};
-var Rc=function(a){var b=Sc(a,"visibility");a=Sc(a,"display");return"hidden"!=b&&"none"!=a},Sc=function(a,b){var c;t:{c=rb(a);if(c.defaultView&&c.defaultView.getComputedStyle&&(c=c.defaultView.getComputedStyle(a,null))){c=c[b]||c.getPropertyValue(b)||"";break t}c=""}if(!(c=c||(a.currentStyle?a.currentStyle[b]:null))&&(c=a.style[Fa(b)],"undefined"===typeof c)){c=a.style;var d;t:if(d=Fa(b),void 0===a.style[d]){var e=(E?"Webkit":D?"Moz":C?"ms":cb?"O":null)+Ga(d);if(void 0!==a.style[e]){d=e;break t}}c=
-c[d]||""}return c};V.prototype.disposeInternal=function(){delete this.captchaImage_;delete this.adImage_;V.superClass_.disposeInternal.call(this)};var Tc=function(a,b,c){H.call(this);this.listener_=a;this.interval_=b||0;this.handler_=c;this.callback_=s(this.doAction_,this)};u(Tc,H);h=Tc.prototype;h.id_=0;h.disposeInternal=function(){Tc.superClass_.disposeInternal.call(this);this.stop();delete this.listener_;delete this.handler_};
-h.start=function(a){this.stop();var b=this.callback_;a=l(a)?a:this.interval_;if(!r(b))if(b&&"function"==typeof b.handleEvent)b=s(b.handleEvent,b);else throw Error("Invalid listener argument");this.id_=2147483647<a?-1:k.setTimeout(b,a||0)};h.stop=function(){this.isActive()&&k.clearTimeout(this.id_);this.id_=0};h.isActive=function(){return 0!=this.id_};h.doAction_=function(){this.id_=0;this.listener_&&this.listener_.call(this.handler_)};var Uc=function(a,b){H.call(this);this.listener_=a;this.handler_=b;this.delay_=new Tc(s(this.onTick_,this),0,this)};u(Uc,H);h=Uc.prototype;h.interval_=0;h.runUntil_=0;h.disposeInternal=function(){this.delay_.dispose();delete this.listener_;delete this.handler_;Uc.superClass_.disposeInternal.call(this)};h.start=function(a,b){this.stop();var c=b||0;this.interval_=Math.max(a||0,0);this.runUntil_=0>c?-1:ha()+c;this.delay_.start(0>c?this.interval_:Math.min(this.interval_,c))};h.stop=function(){this.delay_.stop()};
-h.isActive=function(){return this.delay_.isActive()};h.onSuccess=function(){};h.onFailure=function(){};h.onTick_=function(){if(this.listener_.call(this.handler_))this.onSuccess();else if(0>this.runUntil_)this.delay_.start(this.interval_);else{var a=this.runUntil_-ha();if(0>=a)this.onFailure();else this.delay_.start(Math.min(this.interval_,a))}};$a("area base br col command embed hr img input keygen link meta param source track wbr".split(" "));$a("action","cite","data","formaction","href","manifest","poster","src");$a("link","script","style");var Vc={sanitizedContentKindHtml:!0},Wc={sanitizedContentKindText:!0},Xc=function(){throw Error("Do not instantiate directly");};Xc.prototype.contentDir=null;Xc.prototype.toString=function(){return this.content};var bd=function(a){var b=Yc;y(b,"Soy template may not be null.");var c=sb().createElement("DIV");a=Zc(b(a||$c,void 0,void 0));b=a.match(ad);y(!b,"This template starts with a %s, which cannot be a child of a <div>, as required by soy internals. Consider using goog.soy.renderElement instead.\nTemplate output: %s",b&&b[0],a);c.innerHTML=a;return 1==c.childNodes.length&&(a=c.firstChild,1==a.nodeType)?a:c},Zc=function(a){if(!da(a))return String(a);if(a instanceof Xc){if(a.contentKind===Vc)return Ka(a.content);
-if(a.contentKind===Wc)return Da(a.content)}Ja("Soy template output is unsafe for use as HTML: "+a);return"zSoyz"},ad=/^<(body|caption|col|colgroup|head|html|tr|td|tbody|thead|tfoot)>/i,$c={};C&&F(8);var cd=function(){Xc.call(this)};u(cd,Xc);cd.prototype.contentKind=Vc;var dd=function(a){function b(a){this.content=a}b.prototype=a.prototype;return function(a,d){var e=new b(String(a));void 0!==d&&(e.contentDir=d);return e}}(cd);(function(a){function b(a){this.content=a}b.prototype=a.prototype;return function(a,d){var e=String(a);if(!e)return"";e=new b(e);void 0!==d&&(e.contentDir=d);return e}})(cd);
-var ed={"\x00":"\\x00","\b":"\\x08","\t":"\\t","\n":"\\n","\x0B":"\\x0b","\f":"\\f","\r":"\\r",'"':"\\x22",$:"\\x24","&":"\\x26","'":"\\x27","(":"\\x28",")":"\\x29","*":"\\x2a","+":"\\x2b",",":"\\x2c","-":"\\x2d",".":"\\x2e","/":"\\/",":":"\\x3a","<":"\\x3c","=":"\\x3d",">":"\\x3e","?":"\\x3f","[":"\\x5b","\\":"\\\\","]":"\\x5d","^":"\\x5e","{":"\\x7b","|":"\\x7c","}":"\\x7d","\u0085":"\\x85","\u2028":"\\u2028","\u2029":"\\u2029"},fd=function(a){return ed[a]},gd=/[\x00\x08-\x0d\x22\x26\x27\/\x3c-\x3e\\\x85\u2028\u2029]/g;var Yc=function(a){return dd('<script type="text/javascript">var challenge = \''+String(a.challenge).replace(gd,fd)+"'; var publisherId = '"+String(a.publisherId).replace(gd,fd)+"';"+("ca-mongoogle"==a.publisherId?'google_page_url = "3pcerttesting.com/dab/recaptcha.html";':"")+"\n    google_ad_client = publisherId;\n    google_ad_type = 'html';\n    google_ad_output = 'js';\n    google_image_size = '300x57';\n    google_captcha_token = challenge;\n    google_ad_request_done = function(ad) {\n      window.parent.recaptcha.ads.adutils.googleAdRequestDone(ad);\n    };\n    \x3c/script><script type=\"text/javascript\" src=\"//pagead2.googlesyndication.com/pagead/show_ads.js\">\x3c/script>")};
-Yc.soyTemplateName="recaptcha.soy.ads.iframeAdsLoader.main";var Wa=function(){var a=k.google_ad;return!!(a&&a.token&&a.imageAdUrl&&a.hashedAnswer&&a.salt&&a.delayedImpressionUrl&&a.engagementUrl)},hd=function(){k.google_ad&&(k.google_ad=null)},id=function(a){a=a||document.body;var b=k.google_ad;b&&b.searchUpliftUrl&&(b=xb("iframe",{src:'data:text/html;charset=utf-8,<body><img src="https://'+b.searchUpliftUrl+'"></img></body>',style:"display:none"}),a.appendChild(b))},jd=0,kd=function(a){var b=new S;Mc(b,"recaptcha-url-"+jd++,a);b.start()},ld=function(a,b){var c=
-RecaptchaState.publisher_id;hd();var d=xb("iframe",{id:"recaptcha-loader-"+jd++,style:"display: none"});document.body.appendChild(d);var e=d.contentWindow?d.contentWindow.document:d.contentDocument;e.open("text/html","replace");e.write(bd({challenge:a,publisherId:c}).innerHTML);e.close();c=new Uc(function(){return!!k.google_ad});c.onSuccess=function(){Bb(d);b()};c.onFailure=function(){Bb(d);b()};c.start(50,2E3)};t("recaptcha.ads.adutils.googleAdRequestDone",function(a){k.google_ad=a});var md=function(){this.blockSize=-1};var nd=function(){this.blockSize=-1;this.blockSize=64;this.chain_=Array(4);this.block_=Array(this.blockSize);this.totalLength_=this.blockLength_=0;this.reset()};u(nd,md);nd.prototype.reset=function(){this.chain_[0]=1732584193;this.chain_[1]=4023233417;this.chain_[2]=2562383102;this.chain_[3]=271733878;this.totalLength_=this.blockLength_=0};
-var od=function(a,b,c){c||(c=0);var d=Array(16);if(q(b))for(var e=0;16>e;++e)d[e]=b.charCodeAt(c++)|b.charCodeAt(c++)<<8|b.charCodeAt(c++)<<16|b.charCodeAt(c++)<<24;else for(e=0;16>e;++e)d[e]=b[c++]|b[c++]<<8|b[c++]<<16|b[c++]<<24;b=a.chain_[0];c=a.chain_[1];var e=a.chain_[2],g=a.chain_[3],f=0,f=b+(g^c&(e^g))+d[0]+3614090360&4294967295;b=c+(f<<7&4294967295|f>>>25);f=g+(e^b&(c^e))+d[1]+3905402710&4294967295;g=b+(f<<12&4294967295|f>>>20);f=e+(c^g&(b^c))+d[2]+606105819&4294967295;e=g+(f<<17&4294967295|
-f>>>15);f=c+(b^e&(g^b))+d[3]+3250441966&4294967295;c=e+(f<<22&4294967295|f>>>10);f=b+(g^c&(e^g))+d[4]+4118548399&4294967295;b=c+(f<<7&4294967295|f>>>25);f=g+(e^b&(c^e))+d[5]+1200080426&4294967295;g=b+(f<<12&4294967295|f>>>20);f=e+(c^g&(b^c))+d[6]+2821735955&4294967295;e=g+(f<<17&4294967295|f>>>15);f=c+(b^e&(g^b))+d[7]+4249261313&4294967295;c=e+(f<<22&4294967295|f>>>10);f=b+(g^c&(e^g))+d[8]+1770035416&4294967295;b=c+(f<<7&4294967295|f>>>25);f=g+(e^b&(c^e))+d[9]+2336552879&4294967295;g=b+(f<<12&4294967295|
-f>>>20);f=e+(c^g&(b^c))+d[10]+4294925233&4294967295;e=g+(f<<17&4294967295|f>>>15);f=c+(b^e&(g^b))+d[11]+2304563134&4294967295;c=e+(f<<22&4294967295|f>>>10);f=b+(g^c&(e^g))+d[12]+1804603682&4294967295;b=c+(f<<7&4294967295|f>>>25);f=g+(e^b&(c^e))+d[13]+4254626195&4294967295;g=b+(f<<12&4294967295|f>>>20);f=e+(c^g&(b^c))+d[14]+2792965006&4294967295;e=g+(f<<17&4294967295|f>>>15);f=c+(b^e&(g^b))+d[15]+1236535329&4294967295;c=e+(f<<22&4294967295|f>>>10);f=b+(e^g&(c^e))+d[1]+4129170786&4294967295;b=c+(f<<
-5&4294967295|f>>>27);f=g+(c^e&(b^c))+d[6]+3225465664&4294967295;g=b+(f<<9&4294967295|f>>>23);f=e+(b^c&(g^b))+d[11]+643717713&4294967295;e=g+(f<<14&4294967295|f>>>18);f=c+(g^b&(e^g))+d[0]+3921069994&4294967295;c=e+(f<<20&4294967295|f>>>12);f=b+(e^g&(c^e))+d[5]+3593408605&4294967295;b=c+(f<<5&4294967295|f>>>27);f=g+(c^e&(b^c))+d[10]+38016083&4294967295;g=b+(f<<9&4294967295|f>>>23);f=e+(b^c&(g^b))+d[15]+3634488961&4294967295;e=g+(f<<14&4294967295|f>>>18);f=c+(g^b&(e^g))+d[4]+3889429448&4294967295;c=
-e+(f<<20&4294967295|f>>>12);f=b+(e^g&(c^e))+d[9]+568446438&4294967295;b=c+(f<<5&4294967295|f>>>27);f=g+(c^e&(b^c))+d[14]+3275163606&4294967295;g=b+(f<<9&4294967295|f>>>23);f=e+(b^c&(g^b))+d[3]+4107603335&4294967295;e=g+(f<<14&4294967295|f>>>18);f=c+(g^b&(e^g))+d[8]+1163531501&4294967295;c=e+(f<<20&4294967295|f>>>12);f=b+(e^g&(c^e))+d[13]+2850285829&4294967295;b=c+(f<<5&4294967295|f>>>27);f=g+(c^e&(b^c))+d[2]+4243563512&4294967295;g=b+(f<<9&4294967295|f>>>23);f=e+(b^c&(g^b))+d[7]+1735328473&4294967295;
-e=g+(f<<14&4294967295|f>>>18);f=c+(g^b&(e^g))+d[12]+2368359562&4294967295;c=e+(f<<20&4294967295|f>>>12);f=b+(c^e^g)+d[5]+4294588738&4294967295;b=c+(f<<4&4294967295|f>>>28);f=g+(b^c^e)+d[8]+2272392833&4294967295;g=b+(f<<11&4294967295|f>>>21);f=e+(g^b^c)+d[11]+1839030562&4294967295;e=g+(f<<16&4294967295|f>>>16);f=c+(e^g^b)+d[14]+4259657740&4294967295;c=e+(f<<23&4294967295|f>>>9);f=b+(c^e^g)+d[1]+2763975236&4294967295;b=c+(f<<4&4294967295|f>>>28);f=g+(b^c^e)+d[4]+1272893353&4294967295;g=b+(f<<11&4294967295|
-f>>>21);f=e+(g^b^c)+d[7]+4139469664&4294967295;e=g+(f<<16&4294967295|f>>>16);f=c+(e^g^b)+d[10]+3200236656&4294967295;c=e+(f<<23&4294967295|f>>>9);f=b+(c^e^g)+d[13]+681279174&4294967295;b=c+(f<<4&4294967295|f>>>28);f=g+(b^c^e)+d[0]+3936430074&4294967295;g=b+(f<<11&4294967295|f>>>21);f=e+(g^b^c)+d[3]+3572445317&4294967295;e=g+(f<<16&4294967295|f>>>16);f=c+(e^g^b)+d[6]+76029189&4294967295;c=e+(f<<23&4294967295|f>>>9);f=b+(c^e^g)+d[9]+3654602809&4294967295;b=c+(f<<4&4294967295|f>>>28);f=g+(b^c^e)+d[12]+
-3873151461&4294967295;g=b+(f<<11&4294967295|f>>>21);f=e+(g^b^c)+d[15]+530742520&4294967295;e=g+(f<<16&4294967295|f>>>16);f=c+(e^g^b)+d[2]+3299628645&4294967295;c=e+(f<<23&4294967295|f>>>9);f=b+(e^(c|~g))+d[0]+4096336452&4294967295;b=c+(f<<6&4294967295|f>>>26);f=g+(c^(b|~e))+d[7]+1126891415&4294967295;g=b+(f<<10&4294967295|f>>>22);f=e+(b^(g|~c))+d[14]+2878612391&4294967295;e=g+(f<<15&4294967295|f>>>17);f=c+(g^(e|~b))+d[5]+4237533241&4294967295;c=e+(f<<21&4294967295|f>>>11);f=b+(e^(c|~g))+d[12]+1700485571&
-4294967295;b=c+(f<<6&4294967295|f>>>26);f=g+(c^(b|~e))+d[3]+2399980690&4294967295;g=b+(f<<10&4294967295|f>>>22);f=e+(b^(g|~c))+d[10]+4293915773&4294967295;e=g+(f<<15&4294967295|f>>>17);f=c+(g^(e|~b))+d[1]+2240044497&4294967295;c=e+(f<<21&4294967295|f>>>11);f=b+(e^(c|~g))+d[8]+1873313359&4294967295;b=c+(f<<6&4294967295|f>>>26);f=g+(c^(b|~e))+d[15]+4264355552&4294967295;g=b+(f<<10&4294967295|f>>>22);f=e+(b^(g|~c))+d[6]+2734768916&4294967295;e=g+(f<<15&4294967295|f>>>17);f=c+(g^(e|~b))+d[13]+1309151649&
-4294967295;c=e+(f<<21&4294967295|f>>>11);f=b+(e^(c|~g))+d[4]+4149444226&4294967295;b=c+(f<<6&4294967295|f>>>26);f=g+(c^(b|~e))+d[11]+3174756917&4294967295;g=b+(f<<10&4294967295|f>>>22);f=e+(b^(g|~c))+d[2]+718787259&4294967295;e=g+(f<<15&4294967295|f>>>17);f=c+(g^(e|~b))+d[9]+3951481745&4294967295;a.chain_[0]=a.chain_[0]+b&4294967295;a.chain_[1]=a.chain_[1]+(e+(f<<21&4294967295|f>>>11))&4294967295;a.chain_[2]=a.chain_[2]+e&4294967295;a.chain_[3]=a.chain_[3]+g&4294967295};
-nd.prototype.update=function(a,b){l(b)||(b=a.length);for(var c=b-this.blockSize,d=this.block_,e=this.blockLength_,g=0;g<b;){if(0==e)for(;g<=c;)od(this,a,g),g+=this.blockSize;if(q(a))for(;g<b;){if(d[e++]=a.charCodeAt(g++),e==this.blockSize){od(this,d);e=0;break}}else for(;g<b;)if(d[e++]=a[g++],e==this.blockSize){od(this,d);e=0;break}}this.blockLength_=e;this.totalLength_+=b};var X=function(){Q.call(this);this.callback_=this.element_=null;this.md5_=new nd};u(X,Q);var pd=function(a,b,c,d,e){a.unwatch();a.element_=b;a.callback_=e;a.listen(b,"keyup",s(a.onChanged_,a,c,d))};X.prototype.unwatch=function(){this.element_&&this.callback_&&(this.removeAll(),this.callback_=this.element_=null)};
-X.prototype.onChanged_=function(a,b){var c;c=(c=this.element_.value)?c.replace(/[\s\xa0]+/g,"").toLowerCase():"";this.md5_.reset();this.md5_.update(c+"."+b);c=this.md5_;var d=Array((56>c.blockLength_?c.blockSize:2*c.blockSize)-c.blockLength_);d[0]=128;for(var e=1;e<d.length-8;++e)d[e]=0;for(var g=8*c.totalLength_,e=d.length-8;e<d.length;++e)d[e]=g&255,g/=256;c.update(d);d=Array(16);for(e=g=0;4>e;++e)for(var f=0;32>f;f+=8)d[g++]=c.chain_[e]>>>f&255;jb(d).toLowerCase()==a.toLowerCase()&&this.callback_()};
-X.prototype.disposeInternal=function(){this.element_=null;X.superClass_.disposeInternal.call(this)};var rd=function(a,b,c){this.adObject_=a;this.captchaImageUrl_=b;this.opt_successCallback_=c||null;qd(this)};u(rd,H);var qd=function(a){var b=new S;nb(a,ga(ob,b));Mc(b,"recaptcha_challenge_image",a.captchaImageUrl_);Mc(b,"recaptcha_ad_image",a.adObject_.imageAdUrl);var c={};xc(b,"load",s(function(a,b){a[b.target.id]=b.target},a,c));xc(b,"complete",s(a.handleImagesLoaded_,a,c));b.start()};
-rd.prototype.handleImagesLoaded_=function(a){a=new V(a.recaptcha_challenge_image,a.recaptcha_ad_image);nb(this,ga(ob,a));var b=tb(document,"recaptcha_image");Ab(b);Oc(a,b);a.adImage_&&Rc(a.adImage_)&&(kd(this.adObject_.delayedImpressionUrl),a=new X,nb(this,ga(ob,a)),pd(a,tb(document,"recaptcha_response_field"),this.adObject_.hashedAnswer,this.adObject_.salt,s(function(a,b){a.unwatch();kd(b)},this,a,this.adObject_.engagementUrl)),this.opt_successCallback_&&this.opt_successCallback_("04"+this.adObject_.token))};var W=w;t("RecaptchaStr",W);var Y=k.RecaptchaOptions;t("RecaptchaOptions",Y);var sd={tabindex:0,theme:"red",callback:null,lang:null,custom_theme_widget:null,custom_translations:null};t("RecaptchaDefaultOptions",sd);
-var Z={widget:null,timer_id:-1,style_set:!1,theme:null,type:"image",ajax_verify_cb:null,th1:null,th2:null,th3:null,element:"",ad_captcha_plugin:null,reload_timeout:-1,force_reload:!1,$:function(a){return"string"==typeof a?document.getElementById(a):a},attachEvent:function(a,b,c){a&&a.addEventListener?a.addEventListener(b,c,!1):a&&a.attachEvent&&a.attachEvent("on"+b,c)},create:function(a,b,c){Z.destroy();b&&(Z.widget=Z.$(b),Z.element=b);Z._init_options(c);Z._call_challenge(a)},destroy:function(){var a=
-Z.$("recaptcha_challenge_field");a&&a.parentNode.removeChild(a);-1!=Z.timer_id&&clearInterval(Z.timer_id);Z.timer_id=-1;if(a=Z.$("recaptcha_image"))a.innerHTML="";Z.update_widget();Z.widget&&("custom"!=Z.theme?Z.widget.innerHTML="":Z.widget.style.display="none",Z.widget=null)},focus_response_field:function(){var a=Z.$("recaptcha_response_field");a&&a.focus()},get_challenge:function(){return"undefined"==typeof RecaptchaState?null:RecaptchaState.challenge},get_response:function(){var a=Z.$("recaptcha_response_field");
-return a?a.value:null},ajax_verify:function(a){Z.ajax_verify_cb=a;a=Z.get_challenge()||"";var b=Z.get_response()||"";a=Z._get_api_server()+"/ajaxverify?c="+encodeURIComponent(a)+"&response="+encodeURIComponent(b);Z._add_script(a)},_ajax_verify_callback:function(a){Z.ajax_verify_cb(a)},_get_overridable_url:function(a){var b=window.location.protocol;if("undefined"!=typeof _RecaptchaOverrideApiServer)a=_RecaptchaOverrideApiServer;else if("undefined"!=typeof RecaptchaState&&"string"==typeof RecaptchaState.server&&
-0<RecaptchaState.server.length)return RecaptchaState.server.replace(/\/+$/,"");return b+"//"+a},_get_api_server:function(){return Z._get_overridable_url("www.google.com/recaptcha/api")},_get_static_url_root:function(){return Z._get_overridable_url("www.gstatic.com/recaptcha/api")},_call_challenge:function(a){a=Z._get_api_server()+"/challenge?k="+a+"&ajax=1&cachestop="+Math.random();Z.getLang_()&&(a+="&lang="+Z.getLang_());"undefined"!=typeof Y.extra_challenge_params&&(a+="&"+Y.extra_challenge_params);
-Z._add_script(a)},_add_script:function(a){var b=document.createElement("script");b.type="text/javascript";b.src=a;Z._get_script_area().appendChild(b)},_get_script_area:function(){var a=document.getElementsByTagName("head");return a=!a||1>a.length?document.body:a[0]},_hash_merge:function(a){for(var b={},c=0;c<a.length;c++)for(var d in a[c])b[d]=a[c][d];return b},_init_options:function(a){Y=Z._hash_merge([sd,a||{}])},challenge_callback_internal:function(){Z.update_widget();Z._reset_timer();W=Z._hash_merge([w,
-sa[Z.getLang_()]||{},Y.custom_translations||{}]);window.addEventListener&&window.addEventListener("unload",function(){Z.destroy()},!1);Z._is_ie()&&window.attachEvent&&window.attachEvent("onbeforeunload",function(){});if(0<navigator.userAgent.indexOf("KHTML")){var a=document.createElement("iframe");a.src="about:blank";a.style.height="0px";a.style.width="0px";a.style.visibility="hidden";a.style.border="none";a.appendChild(document.createTextNode("This frame prevents back/forward cache problems in Safari."));
-document.body.appendChild(a)}Z._finish_widget()},_add_css:function(a){if(-1!=navigator.appVersion.indexOf("MSIE 5"))document.write('<style type="text/css">'+a+"</style>");else{var b=document.createElement("style");b.type="text/css";b.styleSheet?b.styleSheet.cssText=a:b.appendChild(document.createTextNode(a));Z._get_script_area().appendChild(b)}},_set_style:function(a){Z.style_set||(Z.style_set=!0,Z._add_css(a+"\n\n.recaptcha_is_showing_audio .recaptcha_only_if_image,.recaptcha_isnot_showing_audio .recaptcha_only_if_audio,.recaptcha_had_incorrect_sol .recaptcha_only_if_no_incorrect_sol,.recaptcha_nothad_incorrect_sol .recaptcha_only_if_incorrect_sol{display:none !important}"))},
-_init_builtin_theme:function(){var a=Z.$,b=Z._get_static_url_root(),c=v.VertCss,d=v.VertHtml,e=b+"/img/"+Z.theme,g="gif",b=Z.theme;"clean"==b&&(c=v.CleanCss,d=v.CleanHtml,g="png");c=c.replace(/IMGROOT/g,e);Z._set_style(c);Z.update_widget();Z.widget.innerHTML='<div id="recaptcha_area">'+d+"</div>";c=Z.getLang_();a("recaptcha_privacy")&&null!=c&&"en"==c.substring(0,2).toLowerCase()&&null!=W.privacy_and_terms&&0<W.privacy_and_terms.length&&(c=document.createElement("a"),c.href="http://www.google.com/intl/en/policies/",
-c.target="_blank",c.innerHTML=W.privacy_and_terms,a("recaptcha_privacy").appendChild(c));c=function(b,c,d,K){var G=a(b);G.src=e+"/"+c+"."+g;c=W[d];G.alt=c;b=a(b+"_btn");b.title=c;Z.attachEvent(b,"click",K)};c("recaptcha_reload","refresh","refresh_btn",function(){Z.reload_internal("r")});c("recaptcha_switch_audio","audio","audio_challenge",function(){Z.switch_type("audio")});c("recaptcha_switch_img","text","visual_challenge",function(){Z.switch_type("image")});c("recaptcha_whatsthis","help","help_btn",
-Z.showhelp);"clean"==b&&(a("recaptcha_logo").src=e+"/logo."+g);a("recaptcha_table").className="recaptchatable recaptcha_theme_"+Z.theme;b=function(b,c){var d=a(b);d&&(RecaptchaState.rtl&&"span"==d.tagName.toLowerCase()&&(d.dir="rtl"),d.appendChild(document.createTextNode(W[c])))};b("recaptcha_instructions_image","instructions_visual");b("recaptcha_instructions_audio","instructions_audio");b("recaptcha_instructions_error","incorrect_try_again");a("recaptcha_instructions_image")||a("recaptcha_instructions_audio")||
-(b="audio"==Z.type?W.instructions_audio:W.instructions_visual,b=b.replace(/:$/,""),a("recaptcha_response_field").setAttribute("placeholder",b))},_finish_widget:function(){var a=Z.$,b=Y,c=b.theme;c in{blackglass:1,clean:1,custom:1,red:1,white:1}||(c="red");Z.theme||(Z.theme=c);"custom"!=Z.theme?Z._init_builtin_theme():Z._set_style("");c=document.createElement("span");c.id="recaptcha_challenge_field_holder";c.style.display="none";a("recaptcha_response_field").parentNode.insertBefore(c,a("recaptcha_response_field"));
-a("recaptcha_response_field").setAttribute("autocomplete","off");a("recaptcha_image").style.width="300px";a("recaptcha_image").style.height="57px";a("recaptcha_challenge_field_holder").innerHTML='<input type="hidden" name="recaptcha_challenge_field" id="recaptcha_challenge_field" value=""/>';Z.th_init();Z.should_focus=!1;Z.th3||Z.force_reload?(Z._set_challenge(RecaptchaState.challenge,"image",!0),setTimeout(function(){Z.reload_internal("i")},100)):Z._set_challenge(RecaptchaState.challenge,"image",
-!1);Z.updateTabIndexes_();Z.update_widget();Z.widget&&(Z.widget.style.display="");b.callback&&b.callback()},updateTabIndexes_:function(){var a=Z.$,b=Y;b.tabindex&&(b=b.tabindex,a("recaptcha_response_field").tabIndex=b++,"audio"==Z.type&&a("recaptcha_audio_play_again")&&(a("recaptcha_audio_play_again").tabIndex=b++,a("recaptcha_audio_download"),a("recaptcha_audio_download").tabIndex=b++),"custom"!=Z.theme&&(a("recaptcha_reload_btn").tabIndex=b++,a("recaptcha_switch_audio_btn").tabIndex=b++,a("recaptcha_switch_img_btn").tabIndex=
-b++,a("recaptcha_whatsthis_btn").tabIndex=b,a("recaptcha_privacy").tabIndex=b++))},switch_type:function(a){if(!((new Date).getTime()<Z.reload_timeout)&&(Z.type=a,Z.reload_internal("audio"==Z.type?"a":"v"),"custom"!=Z.theme)){a=Z.$;var b="audio"==Z.type?W.instructions_audio:W.instructions_visual,b=b.replace(/:$/,"");a("recaptcha_response_field").setAttribute("placeholder",b)}},reload:function(){Z.reload_internal("r")},reload_internal:function(a){var b=Y,c=RecaptchaState,d=(new Date).getTime();d<Z.reload_timeout||
-(Z.reload_timeout=d+1E3,"undefined"==typeof a&&(a="r"),d=Z._get_api_server()+"/reload?c="+c.challenge+"&k="+c.site+"&reason="+a+"&type="+Z.type,Z.getLang_()&&(d+="&lang="+Z.getLang_()),"undefined"!=typeof b.extra_challenge_params&&(d+="&"+b.extra_challenge_params),Z.th_callback_invoke(),Z.th1&&(d+="&th="+Z.th1,Z.th1=""),"audio"==Z.type&&(d=b.audio_beta_12_08?d+"&audio_beta_12_08=1":d+"&new_audio_default=1"),Z.should_focus="t"!=a&&"i"!=a,Z._add_script(d),ob(Z.ad_captcha_plugin),c.publisher_id=null)},
-th_callback_invoke:function(){if(Z.th3)try{var a=Z.th3.exec();a&&1600>a.length&&(Z.th1=a)}catch(b){Z.th1=""}},finish_reload:function(a,b,c,d){RecaptchaState.payload_url=c;RecaptchaState.is_incorrect=!1;RecaptchaState.publisher_id=d;Z._set_challenge(a,b,!1);Z.updateTabIndexes_()},_set_challenge:function(a,b,c){"image"==b&&RecaptchaState.publisher_id?ld(a,function(){Z._set_challenge_internal(a,b,c)}):Z._set_challenge_internal(a,b,c)},_set_challenge_internal:function(a,b,c){var d=Z.$,e=RecaptchaState;
-e.challenge=a;Z.type=b;d("recaptcha_challenge_field").value=e.challenge;c||("audio"==b?(d("recaptcha_image").innerHTML=Z.getAudioCaptchaHtml(),Z._loop_playback()):"image"==b&&(a=e.payload_url,a||(a=Z._get_api_server()+"/image?c="+e.challenge,Z.th_callback_invoke(),Z.th1&&(a+="&th="+Z.th1,Z.th1="")),id(d("recaptcha_widget_div")),Wa()?Z.ad_captcha_plugin=new rd(Xa(),a,function(a){RecaptchaState.challenge=a;d("recaptcha_challenge_field").value=a}):d("recaptcha_image").innerHTML='<img id="recaptcha_challenge_image" alt="'+
-W.image_alt_text+'" height="57" width="300" src="'+a+'" />',hd()));Z._css_toggle("recaptcha_had_incorrect_sol","recaptcha_nothad_incorrect_sol",e.is_incorrect);Z._css_toggle("recaptcha_is_showing_audio","recaptcha_isnot_showing_audio","audio"==b);Z._clear_input();Z.should_focus&&Z.focus_response_field();Z._reset_timer()},_reset_timer:function(){clearInterval(Z.timer_id);var a=Math.max(1E3*(RecaptchaState.timeout-60),6E4);Z.timer_id=setInterval(function(){Z.reload_internal("t")},a);return a},showhelp:function(){window.open(Z._get_help_link(),
-"recaptcha_popup","width=460,height=580,location=no,menubar=no,status=no,toolbar=no,scrollbars=yes,resizable=yes")},_clear_input:function(){Z.$("recaptcha_response_field").value=""},_displayerror:function(a){var b=Z.$;b("recaptcha_image").innerHTML="";b("recaptcha_image").appendChild(document.createTextNode(a))},reloaderror:function(a){Z._displayerror(a)},_is_ie:function(){return 0<navigator.userAgent.indexOf("MSIE")&&!window.opera},_css_toggle:function(a,b,c){Z.update_widget();var d=Z.widget;d||
-(d=document.body);var e=d.className,e=e.replace(new RegExp("(^|\\s+)"+a+"(\\s+|$)")," "),e=e.replace(new RegExp("(^|\\s+)"+b+"(\\s+|$)")," ");d.className=e+(" "+(c?a:b))},_get_help_link:function(){var a="https://support.google.com/recaptcha/";Z.getLang_()&&(a+="?hl="+Z.getLang_());return a},playAgain:function(){Z.$("recaptcha_image").innerHTML=Z.getAudioCaptchaHtml();Z._loop_playback()},_loop_playback:function(){var a=Z.$("recaptcha_audio_play_again");a&&Z.attachEvent(a,"click",function(){Z.playAgain();
-return!1})},getAudioCaptchaHtml:function(){var a=RecaptchaState.payload_url;a||(a=Z._get_api_server()+"/audio.mp3?c="+RecaptchaState.challenge,Z.th_callback_invoke(),Z.th1&&(a+="&th="+Z.th1,Z.th1=""));var b=Z._get_api_server()+"/swf/audiocaptcha.swf?v2",b=Z._is_ie()?'<object classid="clsid:D27CDB6E-AE6D-11cf-96B8-444553540000" id="audiocaptcha" width="0" height="0" codebase="https://fpdownload.macromedia.com/get/flashplayer/current/swflash.cab"><param name="movie" value="'+b+'" /><param name="quality" value="high" /><param name="bgcolor" value="#869ca7" /><param name="allowScriptAccess" value="always" /></object><br/>':
-'<embed src="'+b+'" quality="high" bgcolor="#869ca7" width="0" height="0" name="audiocaptcha" align="middle" play="true" loop="false" quality="high" allowScriptAccess="always" type="application/x-shockwave-flash" pluginspage="http://www.adobe.com/go/getflashplayer" /></embed>',c="";Z.checkFlashVer()&&(c="<br/>"+Z.getSpan_('<a id="recaptcha_audio_play_again" class="recaptcha_audio_cant_hear_link">'+W.play_again+"</a>"));c+="<br/>"+Z.getSpan_('<a id="recaptcha_audio_download" class="recaptcha_audio_cant_hear_link" target="_blank" href="'+
-a+'">'+W.cant_hear_this+"</a>");return b+c},getSpan_:function(a){return"<span"+(RecaptchaState&&RecaptchaState.rtl?' dir="rtl"':"")+">"+a+"</span>"},gethttpwavurl:function(){if("audio"!=Z.type)return"";var a=RecaptchaState.payload_url;a||(a=Z._get_api_server()+"/image?c="+RecaptchaState.challenge,Z.th_callback_invoke(),Z.th1&&(a+="&th="+Z.th1,Z.th1=""));return a},checkFlashVer:function(){var a=-1!=navigator.appVersion.indexOf("MSIE"),b=-1!=navigator.appVersion.toLowerCase().indexOf("win"),c=-1!=navigator.userAgent.indexOf("Opera"),
-d=-1;if(null!=navigator.plugins&&0<navigator.plugins.length){if(navigator.plugins["Shockwave Flash 2.0"]||navigator.plugins["Shockwave Flash"])d=navigator.plugins["Shockwave Flash"+(navigator.plugins["Shockwave Flash 2.0"]?" 2.0":"")].description.split(" ")[2].split(".")[0]}else if(a&&b&&!c)try{d=(new ActiveXObject("ShockwaveFlash.ShockwaveFlash.7")).GetVariable("$version").split(" ")[1].split(",")[0]}catch(e){}return 9<=d},getLang_:function(){return Y.lang?Y.lang:"undefined"!=typeof RecaptchaState&&
-RecaptchaState.lang?RecaptchaState.lang:null},challenge_callback:function(){Z.force_reload=!!RecaptchaState.force_reload;if(RecaptchaState.t3){var a=RecaptchaState.t1?ib(mb(RecaptchaState.t1)):"",b=RecaptchaState.t2?ib(mb(RecaptchaState.t2)):"",c=RecaptchaState.t3?ib(mb(RecaptchaState.t3)):"";Z.th2=c;if(a)b=kc(a),cc(b,Z.challenge_callback_internal,null,void 0),cc(b,null,Z.challenge_callback_internal,void 0);else{if(k.execScript)k.execScript(b,"JavaScript");else if(k.eval)null==ia&&(k.eval("var _et_ = 1;"),
-"undefined"!=typeof k._et_?(delete k._et_,ia=!0):ia=!1),ia?k.eval(b):(a=k.document,c=a.createElement("script"),c.type="text/javascript",c.defer=!1,c.appendChild(a.createTextNode(b)),a.body.appendChild(c),a.body.removeChild(c));else throw Error("goog.globalEval not available");Z.challenge_callback_internal()}}else Z.challenge_callback_internal()},th_init:function(){try{k.thintinel&&k.thintinel.th&&(Z.th3=new k.thintinel.th(Z.th2),Z.th2="")}catch(a){}},update_widget:function(){Z.element&&(Z.widget=
-Z.$(Z.element))}};t("Recaptcha",Z);})()

From b37c4724191995eec4f5bc9d64f176b2a315f777 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Fri, 10 Aug 2018 23:50:21 +1000
Subject: [PATCH 338/830] Rename async to async_helpers because `async` is a
 keyword on Python 3.7 (#3678)

---
 changelog.d/3678.misc                         | 1 +
 synapse/app/federation_sender.py              | 2 +-
 synapse/federation/federation_server.py       | 8 ++++----
 synapse/handlers/device.py                    | 2 +-
 synapse/handlers/federation.py                | 2 +-
 synapse/handlers/initial_sync.py              | 2 +-
 synapse/handlers/message.py                   | 2 +-
 synapse/handlers/pagination.py                | 2 +-
 synapse/handlers/presence.py                  | 2 +-
 synapse/handlers/read_marker.py               | 2 +-
 synapse/handlers/register.py                  | 2 +-
 synapse/handlers/room_list.py                 | 2 +-
 synapse/handlers/room_member.py               | 2 +-
 synapse/handlers/sync.py                      | 2 +-
 synapse/http/client.py                        | 2 +-
 synapse/http/matrixfederationclient.py        | 2 +-
 synapse/notifier.py                           | 2 +-
 synapse/push/bulk_push_rule_evaluator.py      | 2 +-
 synapse/push/mailer.py                        | 2 +-
 synapse/rest/client/transactions.py           | 2 +-
 synapse/rest/media/v1/media_repository.py     | 2 +-
 synapse/rest/media/v1/preview_url_resource.py | 2 +-
 synapse/state.py                              | 2 +-
 synapse/storage/events.py                     | 2 +-
 synapse/storage/roommember.py                 | 2 +-
 synapse/util/{async.py => async_helpers.py}   | 0
 synapse/util/caches/descriptors.py            | 2 +-
 synapse/util/caches/response_cache.py         | 2 +-
 synapse/util/caches/snapshot_cache.py         | 2 +-
 synapse/util/logcontext.py                    | 2 +-
 tests/storage/test__base.py                   | 2 +-
 tests/util/test_linearizer.py                 | 2 +-
 tests/util/test_rwlock.py                     | 2 +-
 33 files changed, 35 insertions(+), 34 deletions(-)
 create mode 100644 changelog.d/3678.misc
 rename synapse/util/{async.py => async_helpers.py} (100%)

diff --git a/changelog.d/3678.misc b/changelog.d/3678.misc
new file mode 100644
index 000000000000..0d7c8da64aa7
--- /dev/null
+++ b/changelog.d/3678.misc
@@ -0,0 +1 @@
+Rename synapse.util.async to synapse.util.async_helpers to mitigate async becoming a keyword on Python 3.7.
diff --git a/synapse/app/federation_sender.py b/synapse/app/federation_sender.py
index e082d45c94e9..7bbf0ad0826c 100644
--- a/synapse/app/federation_sender.py
+++ b/synapse/app/federation_sender.py
@@ -40,7 +40,7 @@
 from synapse.replication.tcp.client import ReplicationClientHandler
 from synapse.server import HomeServer
 from synapse.storage.engines import create_engine
-from synapse.util.async import Linearizer
+from synapse.util.async_helpers import Linearizer
 from synapse.util.httpresourcetree import create_resource_tree
 from synapse.util.logcontext import LoggingContext, run_in_background
 from synapse.util.manhole import manhole
diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index 2b62f687b683..a23136784a39 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -40,7 +40,7 @@
 from synapse.federation.units import Edu, Transaction
 from synapse.http.endpoint import parse_server_name
 from synapse.types import get_domain_from_id
-from synapse.util import async
+from synapse.util.async_helpers import Linearizer, concurrently_execute
 from synapse.util.caches.response_cache import ResponseCache
 from synapse.util.logutils import log_function
 
@@ -67,8 +67,8 @@ def __init__(self, hs):
         self.auth = hs.get_auth()
         self.handler = hs.get_handlers().federation_handler
 
-        self._server_linearizer = async.Linearizer("fed_server")
-        self._transaction_linearizer = async.Linearizer("fed_txn_handler")
+        self._server_linearizer = Linearizer("fed_server")
+        self._transaction_linearizer = Linearizer("fed_txn_handler")
 
         self.transaction_actions = TransactionActions(self.store)
 
@@ -200,7 +200,7 @@ def process_pdus_for_room(room_id):
                         event_id, f.getTraceback().rstrip(),
                     )
 
-        yield async.concurrently_execute(
+        yield concurrently_execute(
             process_pdus_for_room, pdus_by_room.keys(),
             TRANSACTION_CONCURRENCY_LIMIT,
         )
diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py
index 2d44f15da306..9e017116a904 100644
--- a/synapse/handlers/device.py
+++ b/synapse/handlers/device.py
@@ -23,7 +23,7 @@
 from synapse.api.errors import FederationDeniedError
 from synapse.types import RoomStreamToken, get_domain_from_id
 from synapse.util import stringutils
-from synapse.util.async import Linearizer
+from synapse.util.async_helpers import Linearizer
 from synapse.util.caches.expiringcache import ExpiringCache
 from synapse.util.metrics import measure_func
 from synapse.util.retryutils import NotRetryingDestination
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 0dffd44e22e0..2380d17f4e99 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -52,7 +52,7 @@
 from synapse.state import resolve_events_with_factory
 from synapse.types import UserID, get_domain_from_id
 from synapse.util import logcontext, unwrapFirstError
-from synapse.util.async import Linearizer
+from synapse.util.async_helpers import Linearizer
 from synapse.util.distributor import user_joined_room
 from synapse.util.frozenutils import unfreeze
 from synapse.util.logutils import log_function
diff --git a/synapse/handlers/initial_sync.py b/synapse/handlers/initial_sync.py
index 40e7580a61dd..1fb17fd9a5b1 100644
--- a/synapse/handlers/initial_sync.py
+++ b/synapse/handlers/initial_sync.py
@@ -25,7 +25,7 @@
 from synapse.streams.config import PaginationConfig
 from synapse.types import StreamToken, UserID
 from synapse.util import unwrapFirstError
-from synapse.util.async import concurrently_execute
+from synapse.util.async_helpers import concurrently_execute
 from synapse.util.caches.snapshot_cache import SnapshotCache
 from synapse.util.logcontext import make_deferred_yieldable, run_in_background
 from synapse.visibility import filter_events_for_client
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index bcb093ba3ea7..01a362360e92 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -32,7 +32,7 @@
 from synapse.events.validator import EventValidator
 from synapse.replication.http.send_event import ReplicationSendEventRestServlet
 from synapse.types import RoomAlias, UserID
-from synapse.util.async import Linearizer
+from synapse.util.async_helpers import Linearizer
 from synapse.util.frozenutils import frozendict_json_encoder
 from synapse.util.logcontext import run_in_background
 from synapse.util.metrics import measure_func
diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py
index b2849783ed1b..a97d43550f6c 100644
--- a/synapse/handlers/pagination.py
+++ b/synapse/handlers/pagination.py
@@ -22,7 +22,7 @@
 from synapse.api.errors import SynapseError
 from synapse.events.utils import serialize_event
 from synapse.types import RoomStreamToken
-from synapse.util.async import ReadWriteLock
+from synapse.util.async_helpers import ReadWriteLock
 from synapse.util.logcontext import run_in_background
 from synapse.util.stringutils import random_string
 from synapse.visibility import filter_events_for_client
diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index 3732830194ee..20fc3b03230d 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -36,7 +36,7 @@
 from synapse.metrics import LaterGauge
 from synapse.storage.presence import UserPresenceState
 from synapse.types import UserID, get_domain_from_id
-from synapse.util.async import Linearizer
+from synapse.util.async_helpers import Linearizer
 from synapse.util.caches.descriptors import cachedInlineCallbacks
 from synapse.util.logcontext import run_in_background
 from synapse.util.logutils import log_function
diff --git a/synapse/handlers/read_marker.py b/synapse/handlers/read_marker.py
index 995460f82a30..32108568c640 100644
--- a/synapse/handlers/read_marker.py
+++ b/synapse/handlers/read_marker.py
@@ -17,7 +17,7 @@
 
 from twisted.internet import defer
 
-from synapse.util.async import Linearizer
+from synapse.util.async_helpers import Linearizer
 
 from ._base import BaseHandler
 
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index 0e16bbe0ee7a..3526b20d5a57 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -28,7 +28,7 @@
 )
 from synapse.http.client import CaptchaServerHttpClient
 from synapse.types import RoomAlias, RoomID, UserID, create_requester
-from synapse.util.async import Linearizer
+from synapse.util.async_helpers import Linearizer
 from synapse.util.threepids import check_3pid_allowed
 
 from ._base import BaseHandler
diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index 828229f5c3e3..37e41afd61db 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -26,7 +26,7 @@
 
 from synapse.api.constants import EventTypes, JoinRules
 from synapse.types import ThirdPartyInstanceID
-from synapse.util.async import concurrently_execute
+from synapse.util.async_helpers import concurrently_execute
 from synapse.util.caches.descriptors import cachedInlineCallbacks
 from synapse.util.caches.response_cache import ResponseCache
 
diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 0d4a3f4677e5..fb94b5d7d4a1 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -30,7 +30,7 @@
 from synapse.api.constants import EventTypes, Membership
 from synapse.api.errors import AuthError, Codes, SynapseError
 from synapse.types import RoomID, UserID
-from synapse.util.async import Linearizer
+from synapse.util.async_helpers import Linearizer
 from synapse.util.distributor import user_joined_room, user_left_room
 
 logger = logging.getLogger(__name__)
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index dff1f67dcb45..6393a9674b72 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -25,7 +25,7 @@
 from synapse.api.constants import EventTypes, Membership
 from synapse.push.clientformat import format_push_rules_for_user
 from synapse.types import RoomStreamToken
-from synapse.util.async import concurrently_execute
+from synapse.util.async_helpers import concurrently_execute
 from synapse.util.caches.expiringcache import ExpiringCache
 from synapse.util.caches.lrucache import LruCache
 from synapse.util.caches.response_cache import ResponseCache
diff --git a/synapse/http/client.py b/synapse/http/client.py
index 3771e0b3f627..ab4fbf59b28d 100644
--- a/synapse/http/client.py
+++ b/synapse/http/client.py
@@ -42,7 +42,7 @@
 from synapse.api.errors import Codes, HttpResponseException, SynapseError
 from synapse.http import cancelled_to_request_timed_out_error, redact_uri
 from synapse.http.endpoint import SpiderEndpoint
-from synapse.util.async import add_timeout_to_deferred
+from synapse.util.async_helpers import add_timeout_to_deferred
 from synapse.util.caches import CACHE_SIZE_FACTOR
 from synapse.util.logcontext import make_deferred_yieldable
 
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 762273f59b4b..44b61e70a49c 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -43,7 +43,7 @@
 from synapse.http import cancelled_to_request_timed_out_error
 from synapse.http.endpoint import matrix_federation_endpoint
 from synapse.util import logcontext
-from synapse.util.async import add_timeout_to_deferred
+from synapse.util.async_helpers import add_timeout_to_deferred
 from synapse.util.logcontext import make_deferred_yieldable
 
 logger = logging.getLogger(__name__)
diff --git a/synapse/notifier.py b/synapse/notifier.py
index e650c3e4941a..82f391481c2c 100644
--- a/synapse/notifier.py
+++ b/synapse/notifier.py
@@ -25,7 +25,7 @@
 from synapse.handlers.presence import format_user_presence_state
 from synapse.metrics import LaterGauge
 from synapse.types import StreamToken
-from synapse.util.async import (
+from synapse.util.async_helpers import (
     DeferredTimeoutError,
     ObservableDeferred,
     add_timeout_to_deferred,
diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py
index 1d14d3639c1d..8f9a76147f97 100644
--- a/synapse/push/bulk_push_rule_evaluator.py
+++ b/synapse/push/bulk_push_rule_evaluator.py
@@ -26,7 +26,7 @@
 from synapse.api.constants import EventTypes, Membership
 from synapse.event_auth import get_user_power_level
 from synapse.state import POWER_KEY
-from synapse.util.async import Linearizer
+from synapse.util.async_helpers import Linearizer
 from synapse.util.caches import register_cache
 from synapse.util.caches.descriptors import cached
 
diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py
index 9d601208fd62..bfa6df7b68b8 100644
--- a/synapse/push/mailer.py
+++ b/synapse/push/mailer.py
@@ -35,7 +35,7 @@
     name_from_member_event,
 )
 from synapse.types import UserID
-from synapse.util.async import concurrently_execute
+from synapse.util.async_helpers import concurrently_execute
 from synapse.visibility import filter_events_for_client
 
 logger = logging.getLogger(__name__)
diff --git a/synapse/rest/client/transactions.py b/synapse/rest/client/transactions.py
index 00b1b3066e5d..511e96ab00d9 100644
--- a/synapse/rest/client/transactions.py
+++ b/synapse/rest/client/transactions.py
@@ -17,7 +17,7 @@
 to ensure idempotency when performing PUTs using the REST API."""
 import logging
 
-from synapse.util.async import ObservableDeferred
+from synapse.util.async_helpers import ObservableDeferred
 from synapse.util.logcontext import make_deferred_yieldable, run_in_background
 
 logger = logging.getLogger(__name__)
diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py
index 8fb413d8253b..4c589e05e07d 100644
--- a/synapse/rest/media/v1/media_repository.py
+++ b/synapse/rest/media/v1/media_repository.py
@@ -36,7 +36,7 @@
 )
 from synapse.http.matrixfederationclient import MatrixFederationHttpClient
 from synapse.metrics.background_process_metrics import run_as_background_process
-from synapse.util.async import Linearizer
+from synapse.util.async_helpers import Linearizer
 from synapse.util.logcontext import make_deferred_yieldable
 from synapse.util.retryutils import NotRetryingDestination
 from synapse.util.stringutils import is_ascii, random_string
diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py
index 27aa0def2f02..778ef9733741 100644
--- a/synapse/rest/media/v1/preview_url_resource.py
+++ b/synapse/rest/media/v1/preview_url_resource.py
@@ -42,7 +42,7 @@
 )
 from synapse.http.servlet import parse_integer, parse_string
 from synapse.metrics.background_process_metrics import run_as_background_process
-from synapse.util.async import ObservableDeferred
+from synapse.util.async_helpers import ObservableDeferred
 from synapse.util.caches.expiringcache import ExpiringCache
 from synapse.util.logcontext import make_deferred_yieldable, run_in_background
 from synapse.util.stringutils import is_ascii, random_string
diff --git a/synapse/state.py b/synapse/state.py
index e1092b97a9b4..8b92d4057a2a 100644
--- a/synapse/state.py
+++ b/synapse/state.py
@@ -28,7 +28,7 @@
 from synapse.api.constants import EventTypes
 from synapse.api.errors import AuthError
 from synapse.events.snapshot import EventContext
-from synapse.util.async import Linearizer
+from synapse.util.async_helpers import Linearizer
 from synapse.util.caches import CACHE_SIZE_FACTOR
 from synapse.util.caches.expiringcache import ExpiringCache
 from synapse.util.logutils import log_function
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index ce32e8fefd23..d4aa192a0a49 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -38,7 +38,7 @@
 from synapse.storage.event_federation import EventFederationStore
 from synapse.storage.events_worker import EventsWorkerStore
 from synapse.types import RoomStreamToken, get_domain_from_id
-from synapse.util.async import ObservableDeferred
+from synapse.util.async_helpers import ObservableDeferred
 from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
 from synapse.util.frozenutils import frozendict_json_encoder
 from synapse.util.logcontext import PreserveLoggingContext, make_deferred_yieldable
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index 10dce21cea19..9b4e6d6aa820 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -26,7 +26,7 @@
 from synapse.api.constants import EventTypes, Membership
 from synapse.storage.events_worker import EventsWorkerStore
 from synapse.types import get_domain_from_id
-from synapse.util.async import Linearizer
+from synapse.util.async_helpers import Linearizer
 from synapse.util.caches import intern_string
 from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
 from synapse.util.stringutils import to_ascii
diff --git a/synapse/util/async.py b/synapse/util/async_helpers.py
similarity index 100%
rename from synapse/util/async.py
rename to synapse/util/async_helpers.py
diff --git a/synapse/util/caches/descriptors.py b/synapse/util/caches/descriptors.py
index 861c24809c31..187510576a36 100644
--- a/synapse/util/caches/descriptors.py
+++ b/synapse/util/caches/descriptors.py
@@ -25,7 +25,7 @@
 from twisted.internet import defer
 
 from synapse.util import logcontext, unwrapFirstError
-from synapse.util.async import ObservableDeferred
+from synapse.util.async_helpers import ObservableDeferred
 from synapse.util.caches import get_cache_factor_for
 from synapse.util.caches.lrucache import LruCache
 from synapse.util.caches.treecache import TreeCache, iterate_tree_cache_entry
diff --git a/synapse/util/caches/response_cache.py b/synapse/util/caches/response_cache.py
index a8491b42d57b..afb03b2e1b7c 100644
--- a/synapse/util/caches/response_cache.py
+++ b/synapse/util/caches/response_cache.py
@@ -16,7 +16,7 @@
 
 from twisted.internet import defer
 
-from synapse.util.async import ObservableDeferred
+from synapse.util.async_helpers import ObservableDeferred
 from synapse.util.caches import register_cache
 from synapse.util.logcontext import make_deferred_yieldable, run_in_background
 
diff --git a/synapse/util/caches/snapshot_cache.py b/synapse/util/caches/snapshot_cache.py
index d03678b8c886..8318db8d2c76 100644
--- a/synapse/util/caches/snapshot_cache.py
+++ b/synapse/util/caches/snapshot_cache.py
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from synapse.util.async import ObservableDeferred
+from synapse.util.async_helpers import ObservableDeferred
 
 
 class SnapshotCache(object):
diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py
index 8dcae50b3936..07e83fadda0c 100644
--- a/synapse/util/logcontext.py
+++ b/synapse/util/logcontext.py
@@ -526,7 +526,7 @@ def _set_context_cb(result, context):
     "synapse.util.logcontext",
     "synapse.http.server",
     "synapse.storage._base",
-    "synapse.util.async",
+    "synapse.util.async_helpers",
 ]
 
 
diff --git a/tests/storage/test__base.py b/tests/storage/test__base.py
index 6d6f00c5c5cc..52eff2a10494 100644
--- a/tests/storage/test__base.py
+++ b/tests/storage/test__base.py
@@ -18,7 +18,7 @@
 
 from twisted.internet import defer
 
-from synapse.util.async import ObservableDeferred
+from synapse.util.async_helpers import ObservableDeferred
 from synapse.util.caches.descriptors import Cache, cached
 
 from tests import unittest
diff --git a/tests/util/test_linearizer.py b/tests/util/test_linearizer.py
index 4729bd5a0a15..7f48a72de88f 100644
--- a/tests/util/test_linearizer.py
+++ b/tests/util/test_linearizer.py
@@ -20,7 +20,7 @@
 from twisted.internet.defer import CancelledError
 
 from synapse.util import Clock, logcontext
-from synapse.util.async import Linearizer
+from synapse.util.async_helpers import Linearizer
 
 from tests import unittest
 
diff --git a/tests/util/test_rwlock.py b/tests/util/test_rwlock.py
index 24194e3b2599..7cd470be6743 100644
--- a/tests/util/test_rwlock.py
+++ b/tests/util/test_rwlock.py
@@ -14,7 +14,7 @@
 # limitations under the License.
 
 
-from synapse.util.async import ReadWriteLock
+from synapse.util.async_helpers import ReadWriteLock
 
 from tests import unittest
 

From 8b3d9b6b199abb87246f982d5db356f1966db925 Mon Sep 17 00:00:00 2001
From: black <bot@matrix.org>
Date: Fri, 10 Aug 2018 23:54:09 +1000
Subject: [PATCH 339/830] Run black.

---
 tests/api/test_auth.py                        |  76 ++--
 tests/api/test_filtering.py                   | 361 +++++-------------
 tests/api/test_ratelimiting.py                |   7 +-
 tests/appservice/test_appservice.py           |  86 ++---
 tests/appservice/test_scheduler.py            |  28 +-
 tests/config/test_generate.py                 |  38 +-
 tests/config/test_load.py                     |  38 +-
 tests/crypto/test_event_signing.py            |  19 +-
 tests/crypto/test_keyring.py                  |  46 +--
 tests/events/test_utils.py                    | 135 ++-----
 tests/federation/test_federation_server.py    |  26 +-
 tests/handlers/test_appservice.py             |  39 +-
 tests/handlers/test_auth.py                   |  16 +-
 tests/handlers/test_device.py                 |  98 ++---
 tests/handlers/test_directory.py              |  22 +-
 tests/handlers/test_e2e_keys.py               |  99 ++---
 tests/handlers/test_presence.py               | 216 +++++------
 tests/handlers/test_profile.py                |  25 +-
 tests/handlers/test_register.py               |  12 +-
 tests/handlers/test_typing.py                 | 231 ++++++-----
 tests/http/test_endpoint.py                   |   6 +-
 tests/replication/slave/storage/_base.py      |   5 +-
 .../slave/storage/test_account_data.py        |  14 +-
 .../replication/slave/storage/test_events.py  | 105 ++---
 .../slave/storage/test_receipts.py            |   6 +-
 tests/rest/client/test_transactions.py        |  30 +-
 tests/rest/client/v1/test_admin.py            |   5 +-
 tests/rest/client/v1/test_events.py           |  33 +-
 tests/rest/client/v1/test_profile.py          |  42 +-
 tests/rest/client/v1/test_register.py         |   1 +
 tests/rest/client/v1/test_typing.py           |  47 ++-
 tests/rest/client/v1/utils.py                 |  70 ++--
 tests/rest/client/v2_alpha/test_register.py   |  17 +-
 tests/rest/media/v1/test_media_storage.py     |   6 +-
 tests/server.py                               |   2 +
 tests/storage/test__base.py                   |   5 +-
 tests/storage/test_appservice.py              | 178 +++------
 tests/storage/test_background_update.py       |  17 +-
 tests/storage/test_base.py                    |  46 +--
 tests/storage/test_client_ips.py              |  15 +-
 tests/storage/test_devices.py                 |  69 ++--
 tests/storage/test_directory.py               |  22 +-
 tests/storage/test_end_to_end_keys.py         |  61 +--
 tests/storage/test_event_federation.py        |  39 +-
 tests/storage/test_event_push_actions.py      |  78 ++--
 tests/storage/test_keys.py                    |  11 +-
 tests/storage/test_monthly_active_users.py    |  12 +-
 tests/storage/test_presence.py                |  69 ++--
 tests/storage/test_profile.py                 |  18 +-
 tests/storage/test_redaction.py               |  70 ++--
 tests/storage/test_registration.py            |  38 +-
 tests/storage/test_room.py                    |  47 +--
 tests/storage/test_roommember.py              |  31 +-
 tests/storage/test_state.py                   | 249 ++++++------
 tests/storage/test_user_directory.py          |  40 +-
 tests/test_distributor.py                     |  10 +-
 tests/test_dns.py                             |  22 +-
 tests/test_event_auth.py                      |  98 +++--
 tests/test_preview.py                         |  55 +--
 tests/test_state.py                           | 245 +++++-------
 tests/test_test_utils.py                      |   5 +-
 tests/test_types.py                           |   5 +-
 tests/test_visibility.py                      | 145 +++----
 tests/unittest.py                             |   7 +-
 tests/util/caches/test_descriptors.py         |  46 +--
 tests/util/test_dict_cache.py                 |  19 +-
 tests/util/test_expiring_cache.py             |   1 -
 tests/util/test_file_consumer.py              |   5 +-
 tests/util/test_linearizer.py                 |   7 +-
 tests/util/test_logcontext.py                 |  14 +-
 tests/util/test_lrucache.py                   |   2 -
 tests/util/test_rwlock.py                     |   9 +-
 tests/util/test_snapshot_cache.py             |   1 -
 tests/util/test_stream_change_cache.py        |  13 +-
 tests/utils.py                                |  78 ++--
 75 files changed, 1629 insertions(+), 2280 deletions(-)

diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py
index fbb96361a8bc..f8e28876bb52 100644
--- a/tests/api/test_auth.py
+++ b/tests/api/test_auth.py
@@ -34,7 +34,6 @@ def __init__(self, hs):
 
 
 class AuthTestCase(unittest.TestCase):
-
     @defer.inlineCallbacks
     def setUp(self):
         self.state_handler = Mock()
@@ -53,11 +52,7 @@ def setUp(self):
 
     @defer.inlineCallbacks
     def test_get_user_by_req_user_valid_token(self):
-        user_info = {
-            "name": self.test_user,
-            "token_id": "ditto",
-            "device_id": "device",
-        }
+        user_info = {"name": self.test_user, "token_id": "ditto", "device_id": "device"}
         self.store.get_user_by_access_token = Mock(return_value=user_info)
 
         request = Mock(args={})
@@ -76,10 +71,7 @@ def test_get_user_by_req_user_bad_token(self):
         self.failureResultOf(d, AuthError)
 
     def test_get_user_by_req_user_missing_token(self):
-        user_info = {
-            "name": self.test_user,
-            "token_id": "ditto",
-        }
+        user_info = {"name": self.test_user, "token_id": "ditto"}
         self.store.get_user_by_access_token = Mock(return_value=user_info)
 
         request = Mock(args={})
@@ -90,8 +82,7 @@ def test_get_user_by_req_user_missing_token(self):
     @defer.inlineCallbacks
     def test_get_user_by_req_appservice_valid_token(self):
         app_service = Mock(
-            token="foobar", url="a_url", sender=self.test_user,
-            ip_range_whitelist=None,
+            token="foobar", url="a_url", sender=self.test_user, ip_range_whitelist=None
         )
         self.store.get_app_service_by_token = Mock(return_value=app_service)
         self.store.get_user_by_access_token = Mock(return_value=None)
@@ -106,8 +97,11 @@ def test_get_user_by_req_appservice_valid_token(self):
     @defer.inlineCallbacks
     def test_get_user_by_req_appservice_valid_token_good_ip(self):
         from netaddr import IPSet
+
         app_service = Mock(
-            token="foobar", url="a_url", sender=self.test_user,
+            token="foobar",
+            url="a_url",
+            sender=self.test_user,
             ip_range_whitelist=IPSet(["192.168/16"]),
         )
         self.store.get_app_service_by_token = Mock(return_value=app_service)
@@ -122,8 +116,11 @@ def test_get_user_by_req_appservice_valid_token_good_ip(self):
 
     def test_get_user_by_req_appservice_valid_token_bad_ip(self):
         from netaddr import IPSet
+
         app_service = Mock(
-            token="foobar", url="a_url", sender=self.test_user,
+            token="foobar",
+            url="a_url",
+            sender=self.test_user,
             ip_range_whitelist=IPSet(["192.168/16"]),
         )
         self.store.get_app_service_by_token = Mock(return_value=app_service)
@@ -160,8 +157,7 @@ def test_get_user_by_req_appservice_missing_token(self):
     def test_get_user_by_req_appservice_valid_token_valid_user_id(self):
         masquerading_user_id = b"@doppelganger:matrix.org"
         app_service = Mock(
-            token="foobar", url="a_url", sender=self.test_user,
-            ip_range_whitelist=None,
+            token="foobar", url="a_url", sender=self.test_user, ip_range_whitelist=None
         )
         app_service.is_interested_in_user = Mock(return_value=True)
         self.store.get_app_service_by_token = Mock(return_value=app_service)
@@ -174,15 +170,13 @@ def test_get_user_by_req_appservice_valid_token_valid_user_id(self):
         request.requestHeaders.getRawHeaders = mock_getRawHeaders()
         requester = yield self.auth.get_user_by_req(request)
         self.assertEquals(
-            requester.user.to_string(),
-            masquerading_user_id.decode('utf8')
+            requester.user.to_string(), masquerading_user_id.decode('utf8')
         )
 
     def test_get_user_by_req_appservice_valid_token_bad_user_id(self):
         masquerading_user_id = b"@doppelganger:matrix.org"
         app_service = Mock(
-            token="foobar", url="a_url", sender=self.test_user,
-            ip_range_whitelist=None,
+            token="foobar", url="a_url", sender=self.test_user, ip_range_whitelist=None
         )
         app_service.is_interested_in_user = Mock(return_value=False)
         self.store.get_app_service_by_token = Mock(return_value=app_service)
@@ -201,17 +195,15 @@ def test_get_user_from_macaroon(self):
         # TODO(danielwh): Remove this mock when we remove the
         # get_user_by_access_token fallback.
         self.store.get_user_by_access_token = Mock(
-            return_value={
-                "name": "@baldrick:matrix.org",
-                "device_id": "device",
-            }
+            return_value={"name": "@baldrick:matrix.org", "device_id": "device"}
         )
 
         user_id = "@baldrick:matrix.org"
         macaroon = pymacaroons.Macaroon(
             location=self.hs.config.server_name,
             identifier="key",
-            key=self.hs.config.macaroon_secret_key)
+            key=self.hs.config.macaroon_secret_key,
+        )
         macaroon.add_first_party_caveat("gen = 1")
         macaroon.add_first_party_caveat("type = access")
         macaroon.add_first_party_caveat("user_id = %s" % (user_id,))
@@ -225,15 +217,14 @@ def test_get_user_from_macaroon(self):
 
     @defer.inlineCallbacks
     def test_get_guest_user_from_macaroon(self):
-        self.store.get_user_by_id = Mock(return_value={
-            "is_guest": True,
-        })
+        self.store.get_user_by_id = Mock(return_value={"is_guest": True})
 
         user_id = "@baldrick:matrix.org"
         macaroon = pymacaroons.Macaroon(
             location=self.hs.config.server_name,
             identifier="key",
-            key=self.hs.config.macaroon_secret_key)
+            key=self.hs.config.macaroon_secret_key,
+        )
         macaroon.add_first_party_caveat("gen = 1")
         macaroon.add_first_party_caveat("type = access")
         macaroon.add_first_party_caveat("user_id = %s" % (user_id,))
@@ -257,7 +248,8 @@ def test_get_user_from_macaroon_user_db_mismatch(self):
         macaroon = pymacaroons.Macaroon(
             location=self.hs.config.server_name,
             identifier="key",
-            key=self.hs.config.macaroon_secret_key)
+            key=self.hs.config.macaroon_secret_key,
+        )
         macaroon.add_first_party_caveat("gen = 1")
         macaroon.add_first_party_caveat("type = access")
         macaroon.add_first_party_caveat("user_id = %s" % (user,))
@@ -277,7 +269,8 @@ def test_get_user_from_macaroon_missing_caveat(self):
         macaroon = pymacaroons.Macaroon(
             location=self.hs.config.server_name,
             identifier="key",
-            key=self.hs.config.macaroon_secret_key)
+            key=self.hs.config.macaroon_secret_key,
+        )
         macaroon.add_first_party_caveat("gen = 1")
         macaroon.add_first_party_caveat("type = access")
 
@@ -298,7 +291,8 @@ def test_get_user_from_macaroon_wrong_key(self):
         macaroon = pymacaroons.Macaroon(
             location=self.hs.config.server_name,
             identifier="key",
-            key=self.hs.config.macaroon_secret_key + "wrong")
+            key=self.hs.config.macaroon_secret_key + "wrong",
+        )
         macaroon.add_first_party_caveat("gen = 1")
         macaroon.add_first_party_caveat("type = access")
         macaroon.add_first_party_caveat("user_id = %s" % (user,))
@@ -320,7 +314,8 @@ def test_get_user_from_macaroon_unknown_caveat(self):
         macaroon = pymacaroons.Macaroon(
             location=self.hs.config.server_name,
             identifier="key",
-            key=self.hs.config.macaroon_secret_key)
+            key=self.hs.config.macaroon_secret_key,
+        )
         macaroon.add_first_party_caveat("gen = 1")
         macaroon.add_first_party_caveat("type = access")
         macaroon.add_first_party_caveat("user_id = %s" % (user,))
@@ -347,7 +342,8 @@ def test_get_user_from_macaroon_expired(self):
         macaroon = pymacaroons.Macaroon(
             location=self.hs.config.server_name,
             identifier="key",
-            key=self.hs.config.macaroon_secret_key)
+            key=self.hs.config.macaroon_secret_key,
+        )
         macaroon.add_first_party_caveat("gen = 1")
         macaroon.add_first_party_caveat("type = access")
         macaroon.add_first_party_caveat("user_id = %s" % (user,))
@@ -380,7 +376,8 @@ def test_get_user_from_macaroon_with_valid_duration(self):
         macaroon = pymacaroons.Macaroon(
             location=self.hs.config.server_name,
             identifier="key",
-            key=self.hs.config.macaroon_secret_key)
+            key=self.hs.config.macaroon_secret_key,
+        )
         macaroon.add_first_party_caveat("gen = 1")
         macaroon.add_first_party_caveat("type = access")
         macaroon.add_first_party_caveat("user_id = %s" % (user_id,))
@@ -401,9 +398,7 @@ def test_cannot_use_regular_token_as_guest(self):
         token = yield self.hs.handlers.auth_handler.issue_access_token(
             USER_ID, "DEVICE"
         )
-        self.store.add_access_token_to_user.assert_called_with(
-            USER_ID, token, "DEVICE"
-        )
+        self.store.add_access_token_to_user.assert_called_with(USER_ID, token, "DEVICE")
 
         def get_user(tok):
             if token != tok:
@@ -414,10 +409,9 @@ def get_user(tok):
                 "token_id": 1234,
                 "device_id": "DEVICE",
             }
+
         self.store.get_user_by_access_token = get_user
-        self.store.get_user_by_id = Mock(return_value={
-            "is_guest": False,
-        })
+        self.store.get_user_by_id = Mock(return_value={"is_guest": False})
 
         # check the token works
         request = Mock(args={})
diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py
index 836a23fb54bf..1c2d71052cd0 100644
--- a/tests/api/test_filtering.py
+++ b/tests/api/test_filtering.py
@@ -38,7 +38,6 @@ def MockEvent(**kwargs):
 
 
 class FilteringTestCase(unittest.TestCase):
-
     @defer.inlineCallbacks
     def setUp(self):
         self.mock_federation_resource = MockHttpResource()
@@ -47,9 +46,7 @@ def setUp(self):
         self.mock_http_client.put_json = DeferredMockCallable()
 
         hs = yield setup_test_homeserver(
-            handlers=None,
-            http_client=self.mock_http_client,
-            keyring=Mock(),
+            handlers=None, http_client=self.mock_http_client, keyring=Mock()
         )
 
         self.filtering = hs.get_filtering()
@@ -64,7 +61,7 @@ def test_errors_on_invalid_filters(self):
             {"room": {"timeline": {"limit": 0}, "state": {"not_bars": ["*"]}}},
             {"event_format": "other"},
             {"room": {"not_rooms": ["#foo:pik-test"]}},
-            {"presence": {"senders": ["@bar;pik.test.com"]}}
+            {"presence": {"senders": ["@bar;pik.test.com"]}},
         ]
         for filter in invalid_filters:
             with self.assertRaises(SynapseError) as check_filter_error:
@@ -81,34 +78,34 @@ def test_valid_filters(self):
                     "include_leave": False,
                     "rooms": ["!dee:pik-test"],
                     "not_rooms": ["!gee:pik-test"],
-                    "account_data": {"limit": 0, "types": ["*"]}
+                    "account_data": {"limit": 0, "types": ["*"]},
                 }
             },
             {
                 "room": {
                     "state": {
                         "types": ["m.room.*"],
-                        "not_rooms": ["!726s6s6q:example.com"]
+                        "not_rooms": ["!726s6s6q:example.com"],
                     },
                     "timeline": {
                         "limit": 10,
                         "types": ["m.room.message"],
                         "not_rooms": ["!726s6s6q:example.com"],
-                        "not_senders": ["@spam:example.com"]
+                        "not_senders": ["@spam:example.com"],
                     },
                     "ephemeral": {
                         "types": ["m.receipt", "m.typing"],
                         "not_rooms": ["!726s6s6q:example.com"],
-                        "not_senders": ["@spam:example.com"]
-                    }
+                        "not_senders": ["@spam:example.com"],
+                    },
                 },
                 "presence": {
                     "types": ["m.presence"],
-                    "not_senders": ["@alice:example.com"]
+                    "not_senders": ["@alice:example.com"],
                 },
                 "event_format": "client",
-                "event_fields": ["type", "content", "sender"]
-            }
+                "event_fields": ["type", "content", "sender"],
+            },
         ]
         for filter in valid_filters:
             try:
@@ -121,229 +118,131 @@ def test_limits_are_applied(self):
         pass
 
     def test_definition_types_works_with_literals(self):
-        definition = {
-            "types": ["m.room.message", "org.matrix.foo.bar"]
-        }
-        event = MockEvent(
-            sender="@foo:bar",
-            type="m.room.message",
-            room_id="!foo:bar"
-        )
+        definition = {"types": ["m.room.message", "org.matrix.foo.bar"]}
+        event = MockEvent(sender="@foo:bar", type="m.room.message", room_id="!foo:bar")
 
-        self.assertTrue(
-            Filter(definition).check(event)
-        )
+        self.assertTrue(Filter(definition).check(event))
 
     def test_definition_types_works_with_wildcards(self):
-        definition = {
-            "types": ["m.*", "org.matrix.foo.bar"]
-        }
-        event = MockEvent(
-            sender="@foo:bar",
-            type="m.room.message",
-            room_id="!foo:bar"
-        )
-        self.assertTrue(
-            Filter(definition).check(event)
-        )
+        definition = {"types": ["m.*", "org.matrix.foo.bar"]}
+        event = MockEvent(sender="@foo:bar", type="m.room.message", room_id="!foo:bar")
+        self.assertTrue(Filter(definition).check(event))
 
     def test_definition_types_works_with_unknowns(self):
-        definition = {
-            "types": ["m.room.message", "org.matrix.foo.bar"]
-        }
+        definition = {"types": ["m.room.message", "org.matrix.foo.bar"]}
         event = MockEvent(
             sender="@foo:bar",
             type="now.for.something.completely.different",
-            room_id="!foo:bar"
-        )
-        self.assertFalse(
-            Filter(definition).check(event)
+            room_id="!foo:bar",
         )
+        self.assertFalse(Filter(definition).check(event))
 
     def test_definition_not_types_works_with_literals(self):
-        definition = {
-            "not_types": ["m.room.message", "org.matrix.foo.bar"]
-        }
-        event = MockEvent(
-            sender="@foo:bar",
-            type="m.room.message",
-            room_id="!foo:bar"
-        )
-        self.assertFalse(
-            Filter(definition).check(event)
-        )
+        definition = {"not_types": ["m.room.message", "org.matrix.foo.bar"]}
+        event = MockEvent(sender="@foo:bar", type="m.room.message", room_id="!foo:bar")
+        self.assertFalse(Filter(definition).check(event))
 
     def test_definition_not_types_works_with_wildcards(self):
-        definition = {
-            "not_types": ["m.room.message", "org.matrix.*"]
-        }
+        definition = {"not_types": ["m.room.message", "org.matrix.*"]}
         event = MockEvent(
-            sender="@foo:bar",
-            type="org.matrix.custom.event",
-            room_id="!foo:bar"
-        )
-        self.assertFalse(
-            Filter(definition).check(event)
+            sender="@foo:bar", type="org.matrix.custom.event", room_id="!foo:bar"
         )
+        self.assertFalse(Filter(definition).check(event))
 
     def test_definition_not_types_works_with_unknowns(self):
-        definition = {
-            "not_types": ["m.*", "org.*"]
-        }
-        event = MockEvent(
-            sender="@foo:bar",
-            type="com.nom.nom.nom",
-            room_id="!foo:bar"
-        )
-        self.assertTrue(
-            Filter(definition).check(event)
-        )
+        definition = {"not_types": ["m.*", "org.*"]}
+        event = MockEvent(sender="@foo:bar", type="com.nom.nom.nom", room_id="!foo:bar")
+        self.assertTrue(Filter(definition).check(event))
 
     def test_definition_not_types_takes_priority_over_types(self):
         definition = {
             "not_types": ["m.*", "org.*"],
-            "types": ["m.room.message", "m.room.topic"]
+            "types": ["m.room.message", "m.room.topic"],
         }
-        event = MockEvent(
-            sender="@foo:bar",
-            type="m.room.topic",
-            room_id="!foo:bar"
-        )
-        self.assertFalse(
-            Filter(definition).check(event)
-        )
+        event = MockEvent(sender="@foo:bar", type="m.room.topic", room_id="!foo:bar")
+        self.assertFalse(Filter(definition).check(event))
 
     def test_definition_senders_works_with_literals(self):
-        definition = {
-            "senders": ["@flibble:wibble"]
-        }
+        definition = {"senders": ["@flibble:wibble"]}
         event = MockEvent(
-            sender="@flibble:wibble",
-            type="com.nom.nom.nom",
-            room_id="!foo:bar"
-        )
-        self.assertTrue(
-            Filter(definition).check(event)
+            sender="@flibble:wibble", type="com.nom.nom.nom", room_id="!foo:bar"
         )
+        self.assertTrue(Filter(definition).check(event))
 
     def test_definition_senders_works_with_unknowns(self):
-        definition = {
-            "senders": ["@flibble:wibble"]
-        }
+        definition = {"senders": ["@flibble:wibble"]}
         event = MockEvent(
-            sender="@challenger:appears",
-            type="com.nom.nom.nom",
-            room_id="!foo:bar"
-        )
-        self.assertFalse(
-            Filter(definition).check(event)
+            sender="@challenger:appears", type="com.nom.nom.nom", room_id="!foo:bar"
         )
+        self.assertFalse(Filter(definition).check(event))
 
     def test_definition_not_senders_works_with_literals(self):
-        definition = {
-            "not_senders": ["@flibble:wibble"]
-        }
+        definition = {"not_senders": ["@flibble:wibble"]}
         event = MockEvent(
-            sender="@flibble:wibble",
-            type="com.nom.nom.nom",
-            room_id="!foo:bar"
-        )
-        self.assertFalse(
-            Filter(definition).check(event)
+            sender="@flibble:wibble", type="com.nom.nom.nom", room_id="!foo:bar"
         )
+        self.assertFalse(Filter(definition).check(event))
 
     def test_definition_not_senders_works_with_unknowns(self):
-        definition = {
-            "not_senders": ["@flibble:wibble"]
-        }
+        definition = {"not_senders": ["@flibble:wibble"]}
         event = MockEvent(
-            sender="@challenger:appears",
-            type="com.nom.nom.nom",
-            room_id="!foo:bar"
-        )
-        self.assertTrue(
-            Filter(definition).check(event)
+            sender="@challenger:appears", type="com.nom.nom.nom", room_id="!foo:bar"
         )
+        self.assertTrue(Filter(definition).check(event))
 
     def test_definition_not_senders_takes_priority_over_senders(self):
         definition = {
             "not_senders": ["@misspiggy:muppets"],
-            "senders": ["@kermit:muppets", "@misspiggy:muppets"]
+            "senders": ["@kermit:muppets", "@misspiggy:muppets"],
         }
         event = MockEvent(
-            sender="@misspiggy:muppets",
-            type="m.room.topic",
-            room_id="!foo:bar"
-        )
-        self.assertFalse(
-            Filter(definition).check(event)
+            sender="@misspiggy:muppets", type="m.room.topic", room_id="!foo:bar"
         )
+        self.assertFalse(Filter(definition).check(event))
 
     def test_definition_rooms_works_with_literals(self):
-        definition = {
-            "rooms": ["!secretbase:unknown"]
-        }
+        definition = {"rooms": ["!secretbase:unknown"]}
         event = MockEvent(
-            sender="@foo:bar",
-            type="m.room.message",
-            room_id="!secretbase:unknown"
-        )
-        self.assertTrue(
-            Filter(definition).check(event)
+            sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown"
         )
+        self.assertTrue(Filter(definition).check(event))
 
     def test_definition_rooms_works_with_unknowns(self):
-        definition = {
-            "rooms": ["!secretbase:unknown"]
-        }
+        definition = {"rooms": ["!secretbase:unknown"]}
         event = MockEvent(
             sender="@foo:bar",
             type="m.room.message",
-            room_id="!anothersecretbase:unknown"
-        )
-        self.assertFalse(
-            Filter(definition).check(event)
+            room_id="!anothersecretbase:unknown",
         )
+        self.assertFalse(Filter(definition).check(event))
 
     def test_definition_not_rooms_works_with_literals(self):
-        definition = {
-            "not_rooms": ["!anothersecretbase:unknown"]
-        }
+        definition = {"not_rooms": ["!anothersecretbase:unknown"]}
         event = MockEvent(
             sender="@foo:bar",
             type="m.room.message",
-            room_id="!anothersecretbase:unknown"
-        )
-        self.assertFalse(
-            Filter(definition).check(event)
+            room_id="!anothersecretbase:unknown",
         )
+        self.assertFalse(Filter(definition).check(event))
 
     def test_definition_not_rooms_works_with_unknowns(self):
-        definition = {
-            "not_rooms": ["!secretbase:unknown"]
-        }
+        definition = {"not_rooms": ["!secretbase:unknown"]}
         event = MockEvent(
             sender="@foo:bar",
             type="m.room.message",
-            room_id="!anothersecretbase:unknown"
-        )
-        self.assertTrue(
-            Filter(definition).check(event)
+            room_id="!anothersecretbase:unknown",
         )
+        self.assertTrue(Filter(definition).check(event))
 
     def test_definition_not_rooms_takes_priority_over_rooms(self):
         definition = {
             "not_rooms": ["!secretbase:unknown"],
-            "rooms": ["!secretbase:unknown"]
+            "rooms": ["!secretbase:unknown"],
         }
         event = MockEvent(
-            sender="@foo:bar",
-            type="m.room.message",
-            room_id="!secretbase:unknown"
-        )
-        self.assertFalse(
-            Filter(definition).check(event)
+            sender="@foo:bar", type="m.room.message", room_id="!secretbase:unknown"
         )
+        self.assertFalse(Filter(definition).check(event))
 
     def test_definition_combined_event(self):
         definition = {
@@ -352,16 +251,14 @@ def test_definition_combined_event(self):
             "rooms": ["!stage:unknown"],
             "not_rooms": ["!piggyshouse:muppets"],
             "types": ["m.room.message", "muppets.kermit.*"],
-            "not_types": ["muppets.misspiggy.*"]
+            "not_types": ["muppets.misspiggy.*"],
         }
         event = MockEvent(
             sender="@kermit:muppets",  # yup
             type="m.room.message",  # yup
-            room_id="!stage:unknown"  # yup
-        )
-        self.assertTrue(
-            Filter(definition).check(event)
+            room_id="!stage:unknown",  # yup
         )
+        self.assertTrue(Filter(definition).check(event))
 
     def test_definition_combined_event_bad_sender(self):
         definition = {
@@ -370,16 +267,14 @@ def test_definition_combined_event_bad_sender(self):
             "rooms": ["!stage:unknown"],
             "not_rooms": ["!piggyshouse:muppets"],
             "types": ["m.room.message", "muppets.kermit.*"],
-            "not_types": ["muppets.misspiggy.*"]
+            "not_types": ["muppets.misspiggy.*"],
         }
         event = MockEvent(
             sender="@misspiggy:muppets",  # nope
             type="m.room.message",  # yup
-            room_id="!stage:unknown"  # yup
-        )
-        self.assertFalse(
-            Filter(definition).check(event)
+            room_id="!stage:unknown",  # yup
         )
+        self.assertFalse(Filter(definition).check(event))
 
     def test_definition_combined_event_bad_room(self):
         definition = {
@@ -388,16 +283,14 @@ def test_definition_combined_event_bad_room(self):
             "rooms": ["!stage:unknown"],
             "not_rooms": ["!piggyshouse:muppets"],
             "types": ["m.room.message", "muppets.kermit.*"],
-            "not_types": ["muppets.misspiggy.*"]
+            "not_types": ["muppets.misspiggy.*"],
         }
         event = MockEvent(
             sender="@kermit:muppets",  # yup
             type="m.room.message",  # yup
-            room_id="!piggyshouse:muppets"  # nope
-        )
-        self.assertFalse(
-            Filter(definition).check(event)
+            room_id="!piggyshouse:muppets",  # nope
         )
+        self.assertFalse(Filter(definition).check(event))
 
     def test_definition_combined_event_bad_type(self):
         definition = {
@@ -406,37 +299,26 @@ def test_definition_combined_event_bad_type(self):
             "rooms": ["!stage:unknown"],
             "not_rooms": ["!piggyshouse:muppets"],
             "types": ["m.room.message", "muppets.kermit.*"],
-            "not_types": ["muppets.misspiggy.*"]
+            "not_types": ["muppets.misspiggy.*"],
         }
         event = MockEvent(
             sender="@kermit:muppets",  # yup
             type="muppets.misspiggy.kisses",  # nope
-            room_id="!stage:unknown"  # yup
-        )
-        self.assertFalse(
-            Filter(definition).check(event)
+            room_id="!stage:unknown",  # yup
         )
+        self.assertFalse(Filter(definition).check(event))
 
     @defer.inlineCallbacks
     def test_filter_presence_match(self):
-        user_filter_json = {
-            "presence": {
-                "types": ["m.*"]
-            }
-        }
+        user_filter_json = {"presence": {"types": ["m.*"]}}
         filter_id = yield self.datastore.add_user_filter(
-            user_localpart=user_localpart,
-            user_filter=user_filter_json,
-        )
-        event = MockEvent(
-            sender="@foo:bar",
-            type="m.profile",
+            user_localpart=user_localpart, user_filter=user_filter_json
         )
+        event = MockEvent(sender="@foo:bar", type="m.profile")
         events = [event]
 
         user_filter = yield self.filtering.get_user_filter(
-            user_localpart=user_localpart,
-            filter_id=filter_id,
+            user_localpart=user_localpart, filter_id=filter_id
         )
 
         results = user_filter.filter_presence(events=events)
@@ -444,15 +326,10 @@ def test_filter_presence_match(self):
 
     @defer.inlineCallbacks
     def test_filter_presence_no_match(self):
-        user_filter_json = {
-            "presence": {
-                "types": ["m.*"]
-            }
-        }
+        user_filter_json = {"presence": {"types": ["m.*"]}}
 
         filter_id = yield self.datastore.add_user_filter(
-            user_localpart=user_localpart + "2",
-            user_filter=user_filter_json,
+            user_localpart=user_localpart + "2", user_filter=user_filter_json
         )
         event = MockEvent(
             event_id="$asdasd:localhost",
@@ -462,8 +339,7 @@ def test_filter_presence_no_match(self):
         events = [event]
 
         user_filter = yield self.filtering.get_user_filter(
-            user_localpart=user_localpart + "2",
-            filter_id=filter_id,
+            user_localpart=user_localpart + "2", filter_id=filter_id
         )
 
         results = user_filter.filter_presence(events=events)
@@ -471,27 +347,15 @@ def test_filter_presence_no_match(self):
 
     @defer.inlineCallbacks
     def test_filter_room_state_match(self):
-        user_filter_json = {
-            "room": {
-                "state": {
-                    "types": ["m.*"]
-                }
-            }
-        }
+        user_filter_json = {"room": {"state": {"types": ["m.*"]}}}
         filter_id = yield self.datastore.add_user_filter(
-            user_localpart=user_localpart,
-            user_filter=user_filter_json,
-        )
-        event = MockEvent(
-            sender="@foo:bar",
-            type="m.room.topic",
-            room_id="!foo:bar"
+            user_localpart=user_localpart, user_filter=user_filter_json
         )
+        event = MockEvent(sender="@foo:bar", type="m.room.topic", room_id="!foo:bar")
         events = [event]
 
         user_filter = yield self.filtering.get_user_filter(
-            user_localpart=user_localpart,
-            filter_id=filter_id,
+            user_localpart=user_localpart, filter_id=filter_id
         )
 
         results = user_filter.filter_room_state(events=events)
@@ -499,27 +363,17 @@ def test_filter_room_state_match(self):
 
     @defer.inlineCallbacks
     def test_filter_room_state_no_match(self):
-        user_filter_json = {
-            "room": {
-                "state": {
-                    "types": ["m.*"]
-                }
-            }
-        }
+        user_filter_json = {"room": {"state": {"types": ["m.*"]}}}
         filter_id = yield self.datastore.add_user_filter(
-            user_localpart=user_localpart,
-            user_filter=user_filter_json,
+            user_localpart=user_localpart, user_filter=user_filter_json
         )
         event = MockEvent(
-            sender="@foo:bar",
-            type="org.matrix.custom.event",
-            room_id="!foo:bar"
+            sender="@foo:bar", type="org.matrix.custom.event", room_id="!foo:bar"
         )
         events = [event]
 
         user_filter = yield self.filtering.get_user_filter(
-            user_localpart=user_localpart,
-            filter_id=filter_id,
+            user_localpart=user_localpart, filter_id=filter_id
         )
 
         results = user_filter.filter_room_state(events)
@@ -543,45 +397,32 @@ def test_filter_rooms(self):
 
     @defer.inlineCallbacks
     def test_add_filter(self):
-        user_filter_json = {
-            "room": {
-                "state": {
-                    "types": ["m.*"]
-                }
-            }
-        }
+        user_filter_json = {"room": {"state": {"types": ["m.*"]}}}
 
         filter_id = yield self.filtering.add_user_filter(
-            user_localpart=user_localpart,
-            user_filter=user_filter_json,
+            user_localpart=user_localpart, user_filter=user_filter_json
         )
 
         self.assertEquals(filter_id, 0)
-        self.assertEquals(user_filter_json, (
-            yield self.datastore.get_user_filter(
-                user_localpart=user_localpart,
-                filter_id=0,
-            )
-        ))
+        self.assertEquals(
+            user_filter_json,
+            (
+                yield self.datastore.get_user_filter(
+                    user_localpart=user_localpart, filter_id=0
+                )
+            ),
+        )
 
     @defer.inlineCallbacks
     def test_get_filter(self):
-        user_filter_json = {
-            "room": {
-                "state": {
-                    "types": ["m.*"]
-                }
-            }
-        }
+        user_filter_json = {"room": {"state": {"types": ["m.*"]}}}
 
         filter_id = yield self.datastore.add_user_filter(
-            user_localpart=user_localpart,
-            user_filter=user_filter_json,
+            user_localpart=user_localpart, user_filter=user_filter_json
         )
 
         filter = yield self.filtering.get_user_filter(
-            user_localpart=user_localpart,
-            filter_id=filter_id,
+            user_localpart=user_localpart, filter_id=filter_id
         )
 
         self.assertEquals(filter.get_filter_json(), user_filter_json)
diff --git a/tests/api/test_ratelimiting.py b/tests/api/test_ratelimiting.py
index c45b59b36c37..8933fe3b721f 100644
--- a/tests/api/test_ratelimiting.py
+++ b/tests/api/test_ratelimiting.py
@@ -4,17 +4,16 @@
 
 
 class TestRatelimiter(unittest.TestCase):
-
     def test_allowed(self):
         limiter = Ratelimiter()
         allowed, time_allowed = limiter.send_message(
-            user_id="test_id", time_now_s=0, msg_rate_hz=0.1, burst_count=1,
+            user_id="test_id", time_now_s=0, msg_rate_hz=0.1, burst_count=1
         )
         self.assertTrue(allowed)
         self.assertEquals(10., time_allowed)
 
         allowed, time_allowed = limiter.send_message(
-            user_id="test_id", time_now_s=5, msg_rate_hz=0.1, burst_count=1,
+            user_id="test_id", time_now_s=5, msg_rate_hz=0.1, burst_count=1
         )
         self.assertFalse(allowed)
         self.assertEquals(10., time_allowed)
@@ -28,7 +27,7 @@ def test_allowed(self):
     def test_pruning(self):
         limiter = Ratelimiter()
         allowed, time_allowed = limiter.send_message(
-            user_id="test_id_1", time_now_s=0, msg_rate_hz=0.1, burst_count=1,
+            user_id="test_id_1", time_now_s=0, msg_rate_hz=0.1, burst_count=1
         )
 
         self.assertIn("test_id_1", limiter.message_counts)
diff --git a/tests/appservice/test_appservice.py b/tests/appservice/test_appservice.py
index 891e0cc97301..4003869ed665 100644
--- a/tests/appservice/test_appservice.py
+++ b/tests/appservice/test_appservice.py
@@ -24,14 +24,10 @@
 
 
 def _regex(regex, exclusive=True):
-    return {
-        "regex": re.compile(regex),
-        "exclusive": exclusive
-    }
+    return {"regex": re.compile(regex), "exclusive": exclusive}
 
 
 class ApplicationServiceTestCase(unittest.TestCase):
-
     def setUp(self):
         self.service = ApplicationService(
             id="unique_identifier",
@@ -41,8 +37,8 @@ def setUp(self):
             namespaces={
                 ApplicationService.NS_USERS: [],
                 ApplicationService.NS_ROOMS: [],
-                ApplicationService.NS_ALIASES: []
-            }
+                ApplicationService.NS_ALIASES: [],
+            },
         )
         self.event = Mock(
             type="m.something", room_id="!foo:bar", sender="@someone:somewhere"
@@ -52,25 +48,19 @@ def setUp(self):
 
     @defer.inlineCallbacks
     def test_regex_user_id_prefix_match(self):
-        self.service.namespaces[ApplicationService.NS_USERS].append(
-            _regex("@irc_.*")
-        )
+        self.service.namespaces[ApplicationService.NS_USERS].append(_regex("@irc_.*"))
         self.event.sender = "@irc_foobar:matrix.org"
         self.assertTrue((yield self.service.is_interested(self.event)))
 
     @defer.inlineCallbacks
     def test_regex_user_id_prefix_no_match(self):
-        self.service.namespaces[ApplicationService.NS_USERS].append(
-            _regex("@irc_.*")
-        )
+        self.service.namespaces[ApplicationService.NS_USERS].append(_regex("@irc_.*"))
         self.event.sender = "@someone_else:matrix.org"
         self.assertFalse((yield self.service.is_interested(self.event)))
 
     @defer.inlineCallbacks
     def test_regex_room_member_is_checked(self):
-        self.service.namespaces[ApplicationService.NS_USERS].append(
-            _regex("@irc_.*")
-        )
+        self.service.namespaces[ApplicationService.NS_USERS].append(_regex("@irc_.*"))
         self.event.sender = "@someone_else:matrix.org"
         self.event.type = "m.room.member"
         self.event.state_key = "@irc_foobar:matrix.org"
@@ -98,60 +88,47 @@ def test_regex_alias_match(self):
             _regex("#irc_.*:matrix.org")
         )
         self.store.get_aliases_for_room.return_value = [
-            "#irc_foobar:matrix.org", "#athing:matrix.org"
+            "#irc_foobar:matrix.org",
+            "#athing:matrix.org",
         ]
         self.store.get_users_in_room.return_value = []
-        self.assertTrue((yield self.service.is_interested(
-            self.event, self.store
-        )))
+        self.assertTrue((yield self.service.is_interested(self.event, self.store)))
 
     def test_non_exclusive_alias(self):
         self.service.namespaces[ApplicationService.NS_ALIASES].append(
             _regex("#irc_.*:matrix.org", exclusive=False)
         )
-        self.assertFalse(self.service.is_exclusive_alias(
-            "#irc_foobar:matrix.org"
-        ))
+        self.assertFalse(self.service.is_exclusive_alias("#irc_foobar:matrix.org"))
 
     def test_non_exclusive_room(self):
         self.service.namespaces[ApplicationService.NS_ROOMS].append(
             _regex("!irc_.*:matrix.org", exclusive=False)
         )
-        self.assertFalse(self.service.is_exclusive_room(
-            "!irc_foobar:matrix.org"
-        ))
+        self.assertFalse(self.service.is_exclusive_room("!irc_foobar:matrix.org"))
 
     def test_non_exclusive_user(self):
         self.service.namespaces[ApplicationService.NS_USERS].append(
             _regex("@irc_.*:matrix.org", exclusive=False)
         )
-        self.assertFalse(self.service.is_exclusive_user(
-            "@irc_foobar:matrix.org"
-        ))
+        self.assertFalse(self.service.is_exclusive_user("@irc_foobar:matrix.org"))
 
     def test_exclusive_alias(self):
         self.service.namespaces[ApplicationService.NS_ALIASES].append(
             _regex("#irc_.*:matrix.org", exclusive=True)
         )
-        self.assertTrue(self.service.is_exclusive_alias(
-            "#irc_foobar:matrix.org"
-        ))
+        self.assertTrue(self.service.is_exclusive_alias("#irc_foobar:matrix.org"))
 
     def test_exclusive_user(self):
         self.service.namespaces[ApplicationService.NS_USERS].append(
             _regex("@irc_.*:matrix.org", exclusive=True)
         )
-        self.assertTrue(self.service.is_exclusive_user(
-            "@irc_foobar:matrix.org"
-        ))
+        self.assertTrue(self.service.is_exclusive_user("@irc_foobar:matrix.org"))
 
     def test_exclusive_room(self):
         self.service.namespaces[ApplicationService.NS_ROOMS].append(
             _regex("!irc_.*:matrix.org", exclusive=True)
         )
-        self.assertTrue(self.service.is_exclusive_room(
-            "!irc_foobar:matrix.org"
-        ))
+        self.assertTrue(self.service.is_exclusive_room("!irc_foobar:matrix.org"))
 
     @defer.inlineCallbacks
     def test_regex_alias_no_match(self):
@@ -159,47 +136,36 @@ def test_regex_alias_no_match(self):
             _regex("#irc_.*:matrix.org")
         )
         self.store.get_aliases_for_room.return_value = [
-            "#xmpp_foobar:matrix.org", "#athing:matrix.org"
+            "#xmpp_foobar:matrix.org",
+            "#athing:matrix.org",
         ]
         self.store.get_users_in_room.return_value = []
-        self.assertFalse((yield self.service.is_interested(
-            self.event, self.store
-        )))
+        self.assertFalse((yield self.service.is_interested(self.event, self.store)))
 
     @defer.inlineCallbacks
     def test_regex_multiple_matches(self):
         self.service.namespaces[ApplicationService.NS_ALIASES].append(
             _regex("#irc_.*:matrix.org")
         )
-        self.service.namespaces[ApplicationService.NS_USERS].append(
-            _regex("@irc_.*")
-        )
+        self.service.namespaces[ApplicationService.NS_USERS].append(_regex("@irc_.*"))
         self.event.sender = "@irc_foobar:matrix.org"
         self.store.get_aliases_for_room.return_value = ["#irc_barfoo:matrix.org"]
         self.store.get_users_in_room.return_value = []
-        self.assertTrue((yield self.service.is_interested(
-            self.event, self.store
-        )))
+        self.assertTrue((yield self.service.is_interested(self.event, self.store)))
 
     @defer.inlineCallbacks
     def test_interested_in_self(self):
         # make sure invites get through
         self.service.sender = "@appservice:name"
-        self.service.namespaces[ApplicationService.NS_USERS].append(
-            _regex("@irc_.*")
-        )
+        self.service.namespaces[ApplicationService.NS_USERS].append(_regex("@irc_.*"))
         self.event.type = "m.room.member"
-        self.event.content = {
-            "membership": "invite"
-        }
+        self.event.content = {"membership": "invite"}
         self.event.state_key = self.service.sender
         self.assertTrue((yield self.service.is_interested(self.event)))
 
     @defer.inlineCallbacks
     def test_member_list_match(self):
-        self.service.namespaces[ApplicationService.NS_USERS].append(
-            _regex("@irc_.*")
-        )
+        self.service.namespaces[ApplicationService.NS_USERS].append(_regex("@irc_.*"))
         self.store.get_users_in_room.return_value = [
             "@alice:here",
             "@irc_fo:here",  # AS user
@@ -208,6 +174,6 @@ def test_member_list_match(self):
         self.store.get_aliases_for_room.return_value = []
 
         self.event.sender = "@xmpp_foobar:matrix.org"
-        self.assertTrue((yield self.service.is_interested(
-            event=self.event, store=self.store
-        )))
+        self.assertTrue(
+            (yield self.service.is_interested(event=self.event, store=self.store))
+        )
diff --git a/tests/appservice/test_scheduler.py b/tests/appservice/test_scheduler.py
index b9f4863e9ace..db9f86bdacf0 100644
--- a/tests/appservice/test_scheduler.py
+++ b/tests/appservice/test_scheduler.py
@@ -30,7 +30,6 @@
 
 
 class ApplicationServiceSchedulerTransactionCtrlTestCase(unittest.TestCase):
-
     def setUp(self):
         self.clock = MockClock()
         self.store = Mock()
@@ -38,8 +37,10 @@ def setUp(self):
         self.recoverer = Mock()
         self.recoverer_fn = Mock(return_value=self.recoverer)
         self.txnctrl = _TransactionController(
-            clock=self.clock, store=self.store, as_api=self.as_api,
-            recoverer_fn=self.recoverer_fn
+            clock=self.clock,
+            store=self.store,
+            as_api=self.as_api,
+            recoverer_fn=self.recoverer_fn,
         )
 
     def test_single_service_up_txn_sent(self):
@@ -54,9 +55,7 @@ def test_single_service_up_txn_sent(self):
             return_value=defer.succeed(ApplicationServiceState.UP)
         )
         txn.send = Mock(return_value=defer.succeed(True))
-        self.store.create_appservice_txn = Mock(
-            return_value=defer.succeed(txn)
-        )
+        self.store.create_appservice_txn = Mock(return_value=defer.succeed(txn))
 
         # actual call
         self.txnctrl.send(service, events)
@@ -77,9 +76,7 @@ def test_single_service_down(self):
         self.store.get_appservice_state = Mock(
             return_value=defer.succeed(ApplicationServiceState.DOWN)
         )
-        self.store.create_appservice_txn = Mock(
-            return_value=defer.succeed(txn)
-        )
+        self.store.create_appservice_txn = Mock(return_value=defer.succeed(txn))
 
         # actual call
         self.txnctrl.send(service, events)
@@ -104,9 +101,7 @@ def test_single_service_up_txn_not_sent(self):
         )
         self.store.set_appservice_state = Mock(return_value=defer.succeed(True))
         txn.send = Mock(return_value=defer.succeed(False))  # fails to send
-        self.store.create_appservice_txn = Mock(
-            return_value=defer.succeed(txn)
-        )
+        self.store.create_appservice_txn = Mock(return_value=defer.succeed(txn))
 
         # actual call
         self.txnctrl.send(service, events)
@@ -124,7 +119,6 @@ def test_single_service_up_txn_not_sent(self):
 
 
 class ApplicationServiceSchedulerRecovererTestCase(unittest.TestCase):
-
     def setUp(self):
         self.clock = MockClock()
         self.as_api = Mock()
@@ -146,6 +140,7 @@ def test_recover_single_txn(self):
 
         def take_txn(*args, **kwargs):
             return defer.succeed(txns.pop(0))
+
         self.store.get_oldest_unsent_txn = Mock(side_effect=take_txn)
 
         self.recoverer.recover()
@@ -171,6 +166,7 @@ def take_txn(*args, **kwargs):
                 return defer.succeed(txns.pop(0))
             else:
                 return defer.succeed(txn)
+
         self.store.get_oldest_unsent_txn = Mock(side_effect=take_txn)
 
         self.recoverer.recover()
@@ -197,7 +193,6 @@ def take_txn(*args, **kwargs):
 
 
 class ApplicationServiceSchedulerQueuerTestCase(unittest.TestCase):
-
     def setUp(self):
         self.txn_ctrl = Mock()
         self.queuer = _ServiceQueuer(self.txn_ctrl, MockClock())
@@ -211,9 +206,7 @@ def test_send_single_event_no_queue(self):
 
     def test_send_single_event_with_queue(self):
         d = defer.Deferred()
-        self.txn_ctrl.send = Mock(
-            side_effect=lambda x, y: make_deferred_yieldable(d),
-        )
+        self.txn_ctrl.send = Mock(side_effect=lambda x, y: make_deferred_yieldable(d))
         service = Mock(id=4)
         event = Mock(event_id="first")
         event2 = Mock(event_id="second")
@@ -247,6 +240,7 @@ def test_multiple_service_queues(self):
 
         def do_send(x, y):
             return make_deferred_yieldable(send_return_list.pop(0))
+
         self.txn_ctrl.send = Mock(side_effect=do_send)
 
         # send events for different ASes and make sure they are sent
diff --git a/tests/config/test_generate.py b/tests/config/test_generate.py
index eb7f0ab12a89..f88d28a19dce 100644
--- a/tests/config/test_generate.py
+++ b/tests/config/test_generate.py
@@ -24,7 +24,6 @@
 
 
 class ConfigGenerationTestCase(unittest.TestCase):
-
     def setUp(self):
         self.dir = tempfile.mkdtemp()
         self.file = os.path.join(self.dir, "homeserver.yaml")
@@ -33,23 +32,30 @@ def tearDown(self):
         shutil.rmtree(self.dir)
 
     def test_generate_config_generates_files(self):
-        HomeServerConfig.load_or_generate_config("", [
-            "--generate-config",
-            "-c", self.file,
-            "--report-stats=yes",
-            "-H", "lemurs.win"
-        ])
+        HomeServerConfig.load_or_generate_config(
+            "",
+            [
+                "--generate-config",
+                "-c",
+                self.file,
+                "--report-stats=yes",
+                "-H",
+                "lemurs.win",
+            ],
+        )
 
         self.assertSetEqual(
-            set([
-                "homeserver.yaml",
-                "lemurs.win.log.config",
-                "lemurs.win.signing.key",
-                "lemurs.win.tls.crt",
-                "lemurs.win.tls.dh",
-                "lemurs.win.tls.key",
-            ]),
-            set(os.listdir(self.dir))
+            set(
+                [
+                    "homeserver.yaml",
+                    "lemurs.win.log.config",
+                    "lemurs.win.signing.key",
+                    "lemurs.win.tls.crt",
+                    "lemurs.win.tls.dh",
+                    "lemurs.win.tls.key",
+                ]
+            ),
+            set(os.listdir(self.dir)),
         )
 
         self.assert_log_filename_is(
diff --git a/tests/config/test_load.py b/tests/config/test_load.py
index 5c422eff388a..d5f177709350 100644
--- a/tests/config/test_load.py
+++ b/tests/config/test_load.py
@@ -24,7 +24,6 @@
 
 
 class ConfigLoadingTestCase(unittest.TestCase):
-
     def setUp(self):
         self.dir = tempfile.mkdtemp()
         print(self.dir)
@@ -43,15 +42,14 @@ def test_load_fails_if_server_name_missing(self):
     def test_generates_and_loads_macaroon_secret_key(self):
         self.generate_config()
 
-        with open(self.file,
-                  "r") as f:
+        with open(self.file, "r") as f:
             raw = yaml.load(f)
         self.assertIn("macaroon_secret_key", raw)
 
         config = HomeServerConfig.load_config("", ["-c", self.file])
         self.assertTrue(
             hasattr(config, "macaroon_secret_key"),
-            "Want config to have attr macaroon_secret_key"
+            "Want config to have attr macaroon_secret_key",
         )
         if len(config.macaroon_secret_key) < 5:
             self.fail(
@@ -62,7 +60,7 @@ def test_generates_and_loads_macaroon_secret_key(self):
         config = HomeServerConfig.load_or_generate_config("", ["-c", self.file])
         self.assertTrue(
             hasattr(config, "macaroon_secret_key"),
-            "Want config to have attr macaroon_secret_key"
+            "Want config to have attr macaroon_secret_key",
         )
         if len(config.macaroon_secret_key) < 5:
             self.fail(
@@ -80,10 +78,9 @@ def test_load_succeeds_if_macaroon_secret_key_missing(self):
 
     def test_disable_registration(self):
         self.generate_config()
-        self.add_lines_to_config([
-            "enable_registration: true",
-            "disable_registration: true",
-        ])
+        self.add_lines_to_config(
+            ["enable_registration: true", "disable_registration: true"]
+        )
         # Check that disable_registration clobbers enable_registration.
         config = HomeServerConfig.load_config("", ["-c", self.file])
         self.assertFalse(config.enable_registration)
@@ -92,18 +89,23 @@ def test_disable_registration(self):
         self.assertFalse(config.enable_registration)
 
         # Check that either config value is clobbered by the command line.
-        config = HomeServerConfig.load_or_generate_config("", [
-            "-c", self.file, "--enable-registration"
-        ])
+        config = HomeServerConfig.load_or_generate_config(
+            "", ["-c", self.file, "--enable-registration"]
+        )
         self.assertTrue(config.enable_registration)
 
     def generate_config(self):
-        HomeServerConfig.load_or_generate_config("", [
-            "--generate-config",
-            "-c", self.file,
-            "--report-stats=yes",
-            "-H", "lemurs.win"
-        ])
+        HomeServerConfig.load_or_generate_config(
+            "",
+            [
+                "--generate-config",
+                "-c",
+                self.file,
+                "--report-stats=yes",
+                "-H",
+                "lemurs.win",
+            ],
+        )
 
     def generate_config_and_remove_lines_containing(self, needle):
         self.generate_config()
diff --git a/tests/crypto/test_event_signing.py b/tests/crypto/test_event_signing.py
index cd11871b8059..b2536c1e6914 100644
--- a/tests/crypto/test_event_signing.py
+++ b/tests/crypto/test_event_signing.py
@@ -24,9 +24,7 @@
 
 # Perform these tests using given secret key so we get entirely deterministic
 # signatures output that we can test against.
-SIGNING_KEY_SEED = decode_base64(
-    "YJDBA9Xnr2sVqXD9Vj7XVUnmFZcZrlw8Md7kMW+3XA1"
-)
+SIGNING_KEY_SEED = decode_base64("YJDBA9Xnr2sVqXD9Vj7XVUnmFZcZrlw8Md7kMW+3XA1")
 
 KEY_ALG = "ed25519"
 KEY_VER = 1
@@ -36,7 +34,6 @@
 
 
 class EventSigningTestCase(unittest.TestCase):
-
     def setUp(self):
         self.signing_key = nacl.signing.SigningKey(SIGNING_KEY_SEED)
         self.signing_key.alg = KEY_ALG
@@ -51,7 +48,7 @@ def test_sign_minimal(self):
                 'signatures': {},
                 'type': "X",
                 'unsigned': {'age_ts': 1000000},
-            },
+            }
         )
 
         add_hashes_and_signatures(builder, HOSTNAME, self.signing_key)
@@ -61,8 +58,7 @@ def test_sign_minimal(self):
         self.assertTrue(hasattr(event, 'hashes'))
         self.assertIn('sha256', event.hashes)
         self.assertEquals(
-            event.hashes['sha256'],
-            "6tJjLpXtggfke8UxFhAKg82QVkJzvKOVOOSjUDK4ZSI",
+            event.hashes['sha256'], "6tJjLpXtggfke8UxFhAKg82QVkJzvKOVOOSjUDK4ZSI"
         )
 
         self.assertTrue(hasattr(event, 'signatures'))
@@ -77,9 +73,7 @@ def test_sign_minimal(self):
     def test_sign_message(self):
         builder = EventBuilder(
             {
-                'content': {
-                    'body': "Here is the message content",
-                },
+                'content': {'body': "Here is the message content"},
                 'event_id': "$0:domain",
                 'origin': "domain",
                 'origin_server_ts': 1000000,
@@ -98,8 +92,7 @@ def test_sign_message(self):
         self.assertTrue(hasattr(event, 'hashes'))
         self.assertIn('sha256', event.hashes)
         self.assertEquals(
-            event.hashes['sha256'],
-            "onLKD1bGljeBWQhWZ1kaP9SorVmRQNdN5aM2JYU2n/g",
+            event.hashes['sha256'], "onLKD1bGljeBWQhWZ1kaP9SorVmRQNdN5aM2JYU2n/g"
         )
 
         self.assertTrue(hasattr(event, 'signatures'))
@@ -108,5 +101,5 @@ def test_sign_message(self):
         self.assertEquals(
             event.signatures[HOSTNAME][KEY_NAME],
             "Wm+VzmOUOz08Ds+0NTWb1d4CZrVsJSikkeRxh6aCcUw"
-            "u6pNC78FunoD7KNWzqFn241eYHYMGCA5McEiVPdhzBA"
+            "u6pNC78FunoD7KNWzqFn241eYHYMGCA5McEiVPdhzBA",
         )
diff --git a/tests/crypto/test_keyring.py b/tests/crypto/test_keyring.py
index a9d37fe0845f..e40681ed1e71 100644
--- a/tests/crypto/test_keyring.py
+++ b/tests/crypto/test_keyring.py
@@ -36,9 +36,7 @@ def __init__(self):
 
     def get_verify_keys(self):
         vk = signedjson.key.get_verify_key(self.key)
-        return {
-            "%s:%s" % (vk.alg, vk.version): vk,
-        }
+        return {"%s:%s" % (vk.alg, vk.version): vk}
 
     def get_signed_key(self, server_name, verify_key):
         key_id = "%s:%s" % (verify_key.alg, verify_key.version)
@@ -47,10 +45,8 @@ def get_signed_key(self, server_name, verify_key):
             "old_verify_keys": {},
             "valid_until_ts": time.time() * 1000 + 3600,
             "verify_keys": {
-                key_id: {
-                    "key": signedjson.key.encode_verify_key_base64(verify_key)
-                }
-            }
+                key_id: {"key": signedjson.key.encode_verify_key_base64(verify_key)}
+            },
         }
         signedjson.sign.sign_json(res, self.server_name, self.key)
         return res
@@ -62,18 +58,15 @@ def setUp(self):
         self.mock_perspective_server = MockPerspectiveServer()
         self.http_client = Mock()
         self.hs = yield utils.setup_test_homeserver(
-            handlers=None,
-            http_client=self.http_client,
+            handlers=None, http_client=self.http_client
         )
         self.hs.config.perspectives = {
-            self.mock_perspective_server.server_name:
-                self.mock_perspective_server.get_verify_keys()
+            self.mock_perspective_server.server_name: self.mock_perspective_server.get_verify_keys()
         }
 
     def check_context(self, _, expected):
         self.assertEquals(
-            getattr(LoggingContext.current_context(), "request", None),
-            expected
+            getattr(LoggingContext.current_context(), "request", None), expected
         )
 
     @defer.inlineCallbacks
@@ -89,8 +82,7 @@ def test_wait_for_previous_lookups(self):
             context_one.request = "one"
 
             wait_1_deferred = kr.wait_for_previous_lookups(
-                ["server1"],
-                {"server1": lookup_1_deferred},
+                ["server1"], {"server1": lookup_1_deferred}
             )
 
             # there were no previous lookups, so the deferred should be ready
@@ -105,8 +97,7 @@ def test_wait_for_previous_lookups(self):
             # set off another wait. It should block because the first lookup
             # hasn't yet completed.
             wait_2_deferred = kr.wait_for_previous_lookups(
-                ["server1"],
-                {"server1": lookup_2_deferred},
+                ["server1"], {"server1": lookup_2_deferred}
             )
             self.assertFalse(wait_2_deferred.called)
             # ... so we should have reset the LoggingContext.
@@ -132,21 +123,19 @@ def test_verify_json_objects_for_server_awaits_previous_requests(self):
         persp_resp = {
             "server_keys": [
                 self.mock_perspective_server.get_signed_key(
-                    "server10",
-                    signedjson.key.get_verify_key(key1)
-                ),
+                    "server10", signedjson.key.get_verify_key(key1)
+                )
             ]
         }
         persp_deferred = defer.Deferred()
 
         @defer.inlineCallbacks
         def get_perspectives(**kwargs):
-            self.assertEquals(
-                LoggingContext.current_context().request, "11",
-            )
+            self.assertEquals(LoggingContext.current_context().request, "11")
             with logcontext.PreserveLoggingContext():
                 yield persp_deferred
             defer.returnValue(persp_resp)
+
         self.http_client.post_json.side_effect = get_perspectives
 
         with LoggingContext("11") as context_11:
@@ -154,9 +143,7 @@ def get_perspectives(**kwargs):
 
             # start off a first set of lookups
             res_deferreds = kr.verify_json_objects_for_server(
-                [("server10", json1),
-                 ("server11", {})
-                 ]
+                [("server10", json1), ("server11", {})]
             )
 
             # the unsigned json should be rejected pretty quickly
@@ -172,7 +159,7 @@ def get_perspectives(**kwargs):
 
             # wait a tick for it to send the request to the perspectives server
             # (it first tries the datastore)
-            yield clock.sleep(1)   # XXX find out why this takes so long!
+            yield clock.sleep(1)  # XXX find out why this takes so long!
             self.http_client.post_json.assert_called_once()
 
             self.assertIs(LoggingContext.current_context(), context_11)
@@ -186,7 +173,7 @@ def get_perspectives(**kwargs):
                 self.http_client.post_json.return_value = defer.Deferred()
 
                 res_deferreds_2 = kr.verify_json_objects_for_server(
-                    [("server10", json1)],
+                    [("server10", json1)]
                 )
                 yield clock.sleep(1)
                 self.http_client.post_json.assert_not_called()
@@ -207,8 +194,7 @@ def test_verify_json_for_server(self):
 
         key1 = signedjson.key.generate_signing_key(1)
         yield self.hs.datastore.store_server_verify_key(
-            "server9", "", time.time() * 1000,
-            signedjson.key.get_verify_key(key1),
+            "server9", "", time.time() * 1000, signedjson.key.get_verify_key(key1)
         )
         json1 = {}
         signedjson.sign.sign_json(json1, "server9", key1)
diff --git a/tests/events/test_utils.py b/tests/events/test_utils.py
index f51d99419e5d..ff217ca8b989 100644
--- a/tests/events/test_utils.py
+++ b/tests/events/test_utils.py
@@ -31,25 +31,20 @@ def MockEvent(**kwargs):
 class PruneEventTestCase(unittest.TestCase):
     """ Asserts that a new event constructed with `evdict` will look like
     `matchdict` when it is redacted. """
+
     def run_test(self, evdict, matchdict):
-        self.assertEquals(
-            prune_event(FrozenEvent(evdict)).get_dict(),
-            matchdict
-        )
+        self.assertEquals(prune_event(FrozenEvent(evdict)).get_dict(), matchdict)
 
     def test_minimal(self):
         self.run_test(
-            {
-                'type': 'A',
-                'event_id': '$test:domain',
-            },
+            {'type': 'A', 'event_id': '$test:domain'},
             {
                 'type': 'A',
                 'event_id': '$test:domain',
                 'content': {},
                 'signatures': {},
                 'unsigned': {},
-            }
+            },
         )
 
     def test_basic_keys(self):
@@ -70,23 +65,19 @@ def test_basic_keys(self):
                 'content': {},
                 'signatures': {},
                 'unsigned': {},
-            }
+            },
         )
 
     def test_unsigned_age_ts(self):
         self.run_test(
-            {
-                'type': 'B',
-                'event_id': '$test:domain',
-                'unsigned': {'age_ts': 20},
-            },
+            {'type': 'B', 'event_id': '$test:domain', 'unsigned': {'age_ts': 20}},
             {
                 'type': 'B',
                 'event_id': '$test:domain',
                 'content': {},
                 'signatures': {},
                 'unsigned': {'age_ts': 20},
-            }
+            },
         )
 
         self.run_test(
@@ -101,23 +92,19 @@ def test_unsigned_age_ts(self):
                 'content': {},
                 'signatures': {},
                 'unsigned': {},
-            }
+            },
         )
 
     def test_content(self):
         self.run_test(
-            {
-                'type': 'C',
-                'event_id': '$test:domain',
-                'content': {'things': 'here'},
-            },
+            {'type': 'C', 'event_id': '$test:domain', 'content': {'things': 'here'}},
             {
                 'type': 'C',
                 'event_id': '$test:domain',
                 'content': {},
                 'signatures': {},
                 'unsigned': {},
-            }
+            },
         )
 
         self.run_test(
@@ -132,27 +119,20 @@ def test_content(self):
                 'content': {'creator': '@2:domain'},
                 'signatures': {},
                 'unsigned': {},
-            }
+            },
         )
 
 
 class SerializeEventTestCase(unittest.TestCase):
-
     def serialize(self, ev, fields):
         return serialize_event(ev, 1479807801915, only_event_fields=fields)
 
     def test_event_fields_works_with_keys(self):
         self.assertEquals(
             self.serialize(
-                MockEvent(
-                    sender="@alice:localhost",
-                    room_id="!foo:bar"
-                ),
-                ["room_id"]
+                MockEvent(sender="@alice:localhost", room_id="!foo:bar"), ["room_id"]
             ),
-            {
-                "room_id": "!foo:bar",
-            }
+            {"room_id": "!foo:bar"},
         )
 
     def test_event_fields_works_with_nested_keys(self):
@@ -161,17 +141,11 @@ def test_event_fields_works_with_nested_keys(self):
                 MockEvent(
                     sender="@alice:localhost",
                     room_id="!foo:bar",
-                    content={
-                        "body": "A message",
-                    },
+                    content={"body": "A message"},
                 ),
-                ["content.body"]
+                ["content.body"],
             ),
-            {
-                "content": {
-                    "body": "A message",
-                }
-            }
+            {"content": {"body": "A message"}},
         )
 
     def test_event_fields_works_with_dot_keys(self):
@@ -180,17 +154,11 @@ def test_event_fields_works_with_dot_keys(self):
                 MockEvent(
                     sender="@alice:localhost",
                     room_id="!foo:bar",
-                    content={
-                        "key.with.dots": {},
-                    },
+                    content={"key.with.dots": {}},
                 ),
-                ["content.key\.with\.dots"]
+                ["content.key\.with\.dots"],
             ),
-            {
-                "content": {
-                    "key.with.dots": {},
-                }
-            }
+            {"content": {"key.with.dots": {}}},
         )
 
     def test_event_fields_works_with_nested_dot_keys(self):
@@ -201,21 +169,12 @@ def test_event_fields_works_with_nested_dot_keys(self):
                     room_id="!foo:bar",
                     content={
                         "not_me": 1,
-                        "nested.dot.key": {
-                            "leaf.key": 42,
-                            "not_me_either": 1,
-                        },
+                        "nested.dot.key": {"leaf.key": 42, "not_me_either": 1},
                     },
                 ),
-                ["content.nested\.dot\.key.leaf\.key"]
+                ["content.nested\.dot\.key.leaf\.key"],
             ),
-            {
-                "content": {
-                    "nested.dot.key": {
-                        "leaf.key": 42,
-                    },
-                }
-            }
+            {"content": {"nested.dot.key": {"leaf.key": 42}}},
         )
 
     def test_event_fields_nops_with_unknown_keys(self):
@@ -224,17 +183,11 @@ def test_event_fields_nops_with_unknown_keys(self):
                 MockEvent(
                     sender="@alice:localhost",
                     room_id="!foo:bar",
-                    content={
-                        "foo": "bar",
-                    },
+                    content={"foo": "bar"},
                 ),
-                ["content.foo", "content.notexists"]
+                ["content.foo", "content.notexists"],
             ),
-            {
-                "content": {
-                    "foo": "bar",
-                }
-            }
+            {"content": {"foo": "bar"}},
         )
 
     def test_event_fields_nops_with_non_dict_keys(self):
@@ -243,13 +196,11 @@ def test_event_fields_nops_with_non_dict_keys(self):
                 MockEvent(
                     sender="@alice:localhost",
                     room_id="!foo:bar",
-                    content={
-                        "foo": ["I", "am", "an", "array"],
-                    },
+                    content={"foo": ["I", "am", "an", "array"]},
                 ),
-                ["content.foo.am"]
+                ["content.foo.am"],
             ),
-            {}
+            {},
         )
 
     def test_event_fields_nops_with_array_keys(self):
@@ -258,13 +209,11 @@ def test_event_fields_nops_with_array_keys(self):
                 MockEvent(
                     sender="@alice:localhost",
                     room_id="!foo:bar",
-                    content={
-                        "foo": ["I", "am", "an", "array"],
-                    },
+                    content={"foo": ["I", "am", "an", "array"]},
                 ),
-                ["content.foo.1"]
+                ["content.foo.1"],
             ),
-            {}
+            {},
         )
 
     def test_event_fields_all_fields_if_empty(self):
@@ -274,31 +223,21 @@ def test_event_fields_all_fields_if_empty(self):
                     type="foo",
                     event_id="test",
                     room_id="!foo:bar",
-                    content={
-                        "foo": "bar",
-                    },
+                    content={"foo": "bar"},
                 ),
-                []
+                [],
             ),
             {
                 "type": "foo",
                 "event_id": "test",
                 "room_id": "!foo:bar",
-                "content": {
-                    "foo": "bar",
-                },
-                "unsigned": {}
-            }
+                "content": {"foo": "bar"},
+                "unsigned": {},
+            },
         )
 
     def test_event_fields_fail_if_fields_not_str(self):
         with self.assertRaises(TypeError):
             self.serialize(
-                MockEvent(
-                    room_id="!foo:bar",
-                    content={
-                        "foo": "bar",
-                    },
-                ),
-                ["room_id", 4]
+                MockEvent(room_id="!foo:bar", content={"foo": "bar"}), ["room_id", 4]
             )
diff --git a/tests/federation/test_federation_server.py b/tests/federation/test_federation_server.py
index c91e25f54ff0..af15f4cc5a81 100644
--- a/tests/federation/test_federation_server.py
+++ b/tests/federation/test_federation_server.py
@@ -23,10 +23,7 @@
 @unittest.DEBUG
 class ServerACLsTestCase(unittest.TestCase):
     def test_blacklisted_server(self):
-        e = _create_acl_event({
-            "allow": ["*"],
-            "deny": ["evil.com"],
-        })
+        e = _create_acl_event({"allow": ["*"], "deny": ["evil.com"]})
         logging.info("ACL event: %s", e.content)
 
         self.assertFalse(server_matches_acl_event("evil.com", e))
@@ -36,10 +33,7 @@ def test_blacklisted_server(self):
         self.assertTrue(server_matches_acl_event("honestly.not.evil.com", e))
 
     def test_block_ip_literals(self):
-        e = _create_acl_event({
-            "allow_ip_literals": False,
-            "allow": ["*"],
-        })
+        e = _create_acl_event({"allow_ip_literals": False, "allow": ["*"]})
         logging.info("ACL event: %s", e.content)
 
         self.assertFalse(server_matches_acl_event("1.2.3.4", e))
@@ -49,10 +43,12 @@ def test_block_ip_literals(self):
 
 
 def _create_acl_event(content):
-    return FrozenEvent({
-        "room_id": "!a:b",
-        "event_id": "$a:b",
-        "type": "m.room.server_acls",
-        "sender": "@a:b",
-        "content": content
-    })
+    return FrozenEvent(
+        {
+            "room_id": "!a:b",
+            "event_id": "$a:b",
+            "type": "m.room.server_acls",
+            "sender": "@a:b",
+            "content": content,
+        }
+    )
diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py
index 57c0771cf33d..ba7148ec0181 100644
--- a/tests/handlers/test_appservice.py
+++ b/tests/handlers/test_appservice.py
@@ -45,20 +45,18 @@ def test_notify_interested_services(self):
         services = [
             self._mkservice(is_interested=False),
             interested_service,
-            self._mkservice(is_interested=False)
+            self._mkservice(is_interested=False),
         ]
 
         self.mock_store.get_app_services = Mock(return_value=services)
         self.mock_store.get_user_by_id = Mock(return_value=[])
 
         event = Mock(
-            sender="@someone:anywhere",
-            type="m.room.message",
-            room_id="!foo:bar"
+            sender="@someone:anywhere", type="m.room.message", room_id="!foo:bar"
         )
         self.mock_store.get_new_events_for_appservice.side_effect = [
             (0, [event]),
-            (0, [])
+            (0, []),
         ]
         self.mock_as_api.push = Mock()
         yield self.handler.notify_interested_services(0)
@@ -74,21 +72,15 @@ def test_query_user_exists_unknown_user(self):
         self.mock_store.get_app_services = Mock(return_value=services)
         self.mock_store.get_user_by_id = Mock(return_value=None)
 
-        event = Mock(
-            sender=user_id,
-            type="m.room.message",
-            room_id="!foo:bar"
-        )
+        event = Mock(sender=user_id, type="m.room.message", room_id="!foo:bar")
         self.mock_as_api.push = Mock()
         self.mock_as_api.query_user = Mock()
         self.mock_store.get_new_events_for_appservice.side_effect = [
             (0, [event]),
-            (0, [])
+            (0, []),
         ]
         yield self.handler.notify_interested_services(0)
-        self.mock_as_api.query_user.assert_called_once_with(
-            services[0], user_id
-        )
+        self.mock_as_api.query_user.assert_called_once_with(services[0], user_id)
 
     @defer.inlineCallbacks
     def test_query_user_exists_known_user(self):
@@ -96,25 +88,19 @@ def test_query_user_exists_known_user(self):
         services = [self._mkservice(is_interested=True)]
         services[0].is_interested_in_user = Mock(return_value=True)
         self.mock_store.get_app_services = Mock(return_value=services)
-        self.mock_store.get_user_by_id = Mock(return_value={
-            "name": user_id
-        })
+        self.mock_store.get_user_by_id = Mock(return_value={"name": user_id})
 
-        event = Mock(
-            sender=user_id,
-            type="m.room.message",
-            room_id="!foo:bar"
-        )
+        event = Mock(sender=user_id, type="m.room.message", room_id="!foo:bar")
         self.mock_as_api.push = Mock()
         self.mock_as_api.query_user = Mock()
         self.mock_store.get_new_events_for_appservice.side_effect = [
             (0, [event]),
-            (0, [])
+            (0, []),
         ]
         yield self.handler.notify_interested_services(0)
         self.assertFalse(
             self.mock_as_api.query_user.called,
-            "query_user called when it shouldn't have been."
+            "query_user called when it shouldn't have been.",
         )
 
     @defer.inlineCallbacks
@@ -129,7 +115,7 @@ def test_query_room_alias_exists(self):
         services = [
             self._mkservice_alias(is_interested_in_alias=False),
             interested_service,
-            self._mkservice_alias(is_interested_in_alias=False)
+            self._mkservice_alias(is_interested_in_alias=False),
         ]
 
         self.mock_store.get_app_services = Mock(return_value=services)
@@ -140,8 +126,7 @@ def test_query_room_alias_exists(self):
         result = yield self.handler.query_room_alias_exists(room_alias)
 
         self.mock_as_api.query_alias.assert_called_once_with(
-            interested_service,
-            room_alias_str
+            interested_service, room_alias_str
         )
         self.assertEquals(result.room_id, room_id)
         self.assertEquals(result.servers, servers)
diff --git a/tests/handlers/test_auth.py b/tests/handlers/test_auth.py
index 8a9bf2d5fd02..ede01f8099a6 100644
--- a/tests/handlers/test_auth.py
+++ b/tests/handlers/test_auth.py
@@ -81,9 +81,7 @@ def verify_nonce(caveat):
     def test_short_term_login_token_gives_user_id(self):
         self.hs.clock.now = 1000
 
-        token = self.macaroon_generator.generate_short_term_login_token(
-            "a_user", 5000
-        )
+        token = self.macaroon_generator.generate_short_term_login_token("a_user", 5000)
         user_id = yield self.auth_handler.validate_short_term_login_token_and_get_user_id(
             token
         )
@@ -98,17 +96,13 @@ def test_short_term_login_token_gives_user_id(self):
 
     @defer.inlineCallbacks
     def test_short_term_login_token_cannot_replace_user_id(self):
-        token = self.macaroon_generator.generate_short_term_login_token(
-            "a_user", 5000
-        )
+        token = self.macaroon_generator.generate_short_term_login_token("a_user", 5000)
         macaroon = pymacaroons.Macaroon.deserialize(token)
 
         user_id = yield self.auth_handler.validate_short_term_login_token_and_get_user_id(
             macaroon.serialize()
         )
-        self.assertEqual(
-            "a_user", user_id
-        )
+        self.assertEqual("a_user", user_id)
 
         # add another "user_id" caveat, which might allow us to override the
         # user_id.
@@ -165,7 +159,5 @@ def test_mau_limits_not_exceeded(self):
         )
 
     def _get_macaroon(self):
-        token = self.macaroon_generator.generate_short_term_login_token(
-            "user_a", 5000
-        )
+        token = self.macaroon_generator.generate_short_term_login_token("user_a", 5000)
         return pymacaroons.Macaroon.deserialize(token)
diff --git a/tests/handlers/test_device.py b/tests/handlers/test_device.py
index 633a0b7f361f..d70d64550403 100644
--- a/tests/handlers/test_device.py
+++ b/tests/handlers/test_device.py
@@ -28,9 +28,9 @@
 class DeviceTestCase(unittest.TestCase):
     def __init__(self, *args, **kwargs):
         super(DeviceTestCase, self).__init__(*args, **kwargs)
-        self.store = None    # type: synapse.storage.DataStore
+        self.store = None  # type: synapse.storage.DataStore
         self.handler = None  # type: synapse.handlers.device.DeviceHandler
-        self.clock = None    # type: utils.MockClock
+        self.clock = None  # type: utils.MockClock
 
     @defer.inlineCallbacks
     def setUp(self):
@@ -44,7 +44,7 @@ def test_device_is_created_if_doesnt_exist(self):
         res = yield self.handler.check_device_registered(
             user_id="@boris:foo",
             device_id="fco",
-            initial_device_display_name="display name"
+            initial_device_display_name="display name",
         )
         self.assertEqual(res, "fco")
 
@@ -56,14 +56,14 @@ def test_device_is_preserved_if_exists(self):
         res1 = yield self.handler.check_device_registered(
             user_id="@boris:foo",
             device_id="fco",
-            initial_device_display_name="display name"
+            initial_device_display_name="display name",
         )
         self.assertEqual(res1, "fco")
 
         res2 = yield self.handler.check_device_registered(
             user_id="@boris:foo",
             device_id="fco",
-            initial_device_display_name="new display name"
+            initial_device_display_name="new display name",
         )
         self.assertEqual(res2, "fco")
 
@@ -75,7 +75,7 @@ def test_device_id_is_made_up_if_unspecified(self):
         device_id = yield self.handler.check_device_registered(
             user_id="@theresa:foo",
             device_id=None,
-            initial_device_display_name="display"
+            initial_device_display_name="display",
         )
 
         dev = yield self.handler.store.get_device("@theresa:foo", device_id)
@@ -87,43 +87,53 @@ def test_get_devices_by_user(self):
 
         res = yield self.handler.get_devices_by_user(user1)
         self.assertEqual(3, len(res))
-        device_map = {
-            d["device_id"]: d for d in res
-        }
-        self.assertDictContainsSubset({
-            "user_id": user1,
-            "device_id": "xyz",
-            "display_name": "display 0",
-            "last_seen_ip": None,
-            "last_seen_ts": None,
-        }, device_map["xyz"])
-        self.assertDictContainsSubset({
-            "user_id": user1,
-            "device_id": "fco",
-            "display_name": "display 1",
-            "last_seen_ip": "ip1",
-            "last_seen_ts": 1000000,
-        }, device_map["fco"])
-        self.assertDictContainsSubset({
-            "user_id": user1,
-            "device_id": "abc",
-            "display_name": "display 2",
-            "last_seen_ip": "ip3",
-            "last_seen_ts": 3000000,
-        }, device_map["abc"])
+        device_map = {d["device_id"]: d for d in res}
+        self.assertDictContainsSubset(
+            {
+                "user_id": user1,
+                "device_id": "xyz",
+                "display_name": "display 0",
+                "last_seen_ip": None,
+                "last_seen_ts": None,
+            },
+            device_map["xyz"],
+        )
+        self.assertDictContainsSubset(
+            {
+                "user_id": user1,
+                "device_id": "fco",
+                "display_name": "display 1",
+                "last_seen_ip": "ip1",
+                "last_seen_ts": 1000000,
+            },
+            device_map["fco"],
+        )
+        self.assertDictContainsSubset(
+            {
+                "user_id": user1,
+                "device_id": "abc",
+                "display_name": "display 2",
+                "last_seen_ip": "ip3",
+                "last_seen_ts": 3000000,
+            },
+            device_map["abc"],
+        )
 
     @defer.inlineCallbacks
     def test_get_device(self):
         yield self._record_users()
 
         res = yield self.handler.get_device(user1, "abc")
-        self.assertDictContainsSubset({
-            "user_id": user1,
-            "device_id": "abc",
-            "display_name": "display 2",
-            "last_seen_ip": "ip3",
-            "last_seen_ts": 3000000,
-        }, res)
+        self.assertDictContainsSubset(
+            {
+                "user_id": user1,
+                "device_id": "abc",
+                "display_name": "display 2",
+                "last_seen_ip": "ip3",
+                "last_seen_ts": 3000000,
+            },
+            res,
+        )
 
     @defer.inlineCallbacks
     def test_delete_device(self):
@@ -153,8 +163,7 @@ def test_update_device(self):
     def test_update_unknown_device(self):
         update = {"display_name": "new_display"}
         with self.assertRaises(synapse.api.errors.NotFoundError):
-            yield self.handler.update_device("user_id", "unknown_device_id",
-                                             update)
+            yield self.handler.update_device("user_id", "unknown_device_id", update)
 
     @defer.inlineCallbacks
     def _record_users(self):
@@ -168,16 +177,17 @@ def _record_users(self):
         yield self._record_user(user2, "def", "dispkay", "token4", "ip4")
 
     @defer.inlineCallbacks
-    def _record_user(self, user_id, device_id, display_name,
-                     access_token=None, ip=None):
+    def _record_user(
+        self, user_id, device_id, display_name, access_token=None, ip=None
+    ):
         device_id = yield self.handler.check_device_registered(
             user_id=user_id,
             device_id=device_id,
-            initial_device_display_name=display_name
+            initial_device_display_name=display_name,
         )
 
         if ip is not None:
             yield self.store.insert_client_ip(
-                user_id,
-                access_token, ip, "user_agent", device_id)
+                user_id, access_token, ip, "user_agent", device_id
+            )
             self.clock.advance_time(1000)
diff --git a/tests/handlers/test_directory.py b/tests/handlers/test_directory.py
index a353070316a0..06de9f5eca25 100644
--- a/tests/handlers/test_directory.py
+++ b/tests/handlers/test_directory.py
@@ -42,6 +42,7 @@ def setUp(self):
 
         def register_query_handler(query_type, handler):
             self.query_handlers[query_type] = handler
+
         self.mock_registry.register_query_handler = register_query_handler
 
         hs = yield setup_test_homeserver(
@@ -68,10 +69,7 @@ def test_get_local_association(self):
 
         result = yield self.handler.get_association(self.my_room)
 
-        self.assertEquals({
-            "room_id": "!8765qwer:test",
-            "servers": ["test"],
-        }, result)
+        self.assertEquals({"room_id": "!8765qwer:test", "servers": ["test"]}, result)
 
     @defer.inlineCallbacks
     def test_get_remote_association(self):
@@ -81,16 +79,13 @@ def test_get_remote_association(self):
 
         result = yield self.handler.get_association(self.remote_room)
 
-        self.assertEquals({
-            "room_id": "!8765qwer:test",
-            "servers": ["test", "remote"],
-        }, result)
+        self.assertEquals(
+            {"room_id": "!8765qwer:test", "servers": ["test", "remote"]}, result
+        )
         self.mock_federation.make_query.assert_called_with(
             destination="remote",
             query_type="directory",
-            args={
-                "room_alias": "#another:remote",
-            },
+            args={"room_alias": "#another:remote"},
             retry_on_dns_fail=False,
             ignore_backoff=True,
         )
@@ -105,7 +100,4 @@ def test_incoming_fed_query(self):
             {"room_alias": "#your-room:test"}
         )
 
-        self.assertEquals({
-            "room_id": "!8765asdf:test",
-            "servers": ["test"],
-        }, response)
+        self.assertEquals({"room_id": "!8765asdf:test", "servers": ["test"]}, response)
diff --git a/tests/handlers/test_e2e_keys.py b/tests/handlers/test_e2e_keys.py
index ca1542236d5b..57ab228455e4 100644
--- a/tests/handlers/test_e2e_keys.py
+++ b/tests/handlers/test_e2e_keys.py
@@ -28,14 +28,13 @@
 class E2eKeysHandlerTestCase(unittest.TestCase):
     def __init__(self, *args, **kwargs):
         super(E2eKeysHandlerTestCase, self).__init__(*args, **kwargs)
-        self.hs = None       # type: synapse.server.HomeServer
+        self.hs = None  # type: synapse.server.HomeServer
         self.handler = None  # type: synapse.handlers.e2e_keys.E2eKeysHandler
 
     @defer.inlineCallbacks
     def setUp(self):
         self.hs = yield utils.setup_test_homeserver(
-            handlers=None,
-            federation_client=mock.Mock(),
+            handlers=None, federation_client=mock.Mock()
         )
         self.handler = synapse.handlers.e2e_keys.E2eKeysHandler(self.hs)
 
@@ -54,30 +53,21 @@ def test_reupload_one_time_keys(self):
         device_id = "xyz"
         keys = {
             "alg1:k1": "key1",
-            "alg2:k2": {
-                "key": "key2",
-                "signatures": {"k1": "sig1"}
-            },
-            "alg2:k3": {
-                "key": "key3",
-            },
+            "alg2:k2": {"key": "key2", "signatures": {"k1": "sig1"}},
+            "alg2:k3": {"key": "key3"},
         }
 
         res = yield self.handler.upload_keys_for_user(
-            local_user, device_id, {"one_time_keys": keys},
+            local_user, device_id, {"one_time_keys": keys}
         )
-        self.assertDictEqual(res, {
-            "one_time_key_counts": {"alg1": 1, "alg2": 2}
-        })
+        self.assertDictEqual(res, {"one_time_key_counts": {"alg1": 1, "alg2": 2}})
 
         # we should be able to change the signature without a problem
         keys["alg2:k2"]["signatures"]["k1"] = "sig2"
         res = yield self.handler.upload_keys_for_user(
-            local_user, device_id, {"one_time_keys": keys},
+            local_user, device_id, {"one_time_keys": keys}
         )
-        self.assertDictEqual(res, {
-            "one_time_key_counts": {"alg1": 1, "alg2": 2}
-        })
+        self.assertDictEqual(res, {"one_time_key_counts": {"alg1": 1, "alg2": 2}})
 
     @defer.inlineCallbacks
     def test_change_one_time_keys(self):
@@ -87,25 +77,18 @@ def test_change_one_time_keys(self):
         device_id = "xyz"
         keys = {
             "alg1:k1": "key1",
-            "alg2:k2": {
-                "key": "key2",
-                "signatures": {"k1": "sig1"}
-            },
-            "alg2:k3": {
-                "key": "key3",
-            },
+            "alg2:k2": {"key": "key2", "signatures": {"k1": "sig1"}},
+            "alg2:k3": {"key": "key3"},
         }
 
         res = yield self.handler.upload_keys_for_user(
-            local_user, device_id, {"one_time_keys": keys},
+            local_user, device_id, {"one_time_keys": keys}
         )
-        self.assertDictEqual(res, {
-            "one_time_key_counts": {"alg1": 1, "alg2": 2}
-        })
+        self.assertDictEqual(res, {"one_time_key_counts": {"alg1": 1, "alg2": 2}})
 
         try:
             yield self.handler.upload_keys_for_user(
-                local_user, device_id, {"one_time_keys": {"alg1:k1": "key2"}},
+                local_user, device_id, {"one_time_keys": {"alg1:k1": "key2"}}
             )
             self.fail("No error when changing string key")
         except errors.SynapseError:
@@ -113,7 +96,7 @@ def test_change_one_time_keys(self):
 
         try:
             yield self.handler.upload_keys_for_user(
-                local_user, device_id, {"one_time_keys": {"alg2:k3": "key2"}},
+                local_user, device_id, {"one_time_keys": {"alg2:k3": "key2"}}
             )
             self.fail("No error when replacing dict key with string")
         except errors.SynapseError:
@@ -121,9 +104,7 @@ def test_change_one_time_keys(self):
 
         try:
             yield self.handler.upload_keys_for_user(
-                local_user, device_id, {
-                    "one_time_keys": {"alg1:k1": {"key": "key"}}
-                },
+                local_user, device_id, {"one_time_keys": {"alg1:k1": {"key": "key"}}}
             )
             self.fail("No error when replacing string key with dict")
         except errors.SynapseError:
@@ -131,13 +112,12 @@ def test_change_one_time_keys(self):
 
         try:
             yield self.handler.upload_keys_for_user(
-                local_user, device_id, {
+                local_user,
+                device_id,
+                {
                     "one_time_keys": {
-                        "alg2:k2": {
-                            "key": "key3",
-                            "signatures": {"k1": "sig1"},
-                        }
-                    },
+                        "alg2:k2": {"key": "key3", "signatures": {"k1": "sig1"}}
+                    }
                 },
             )
             self.fail("No error when replacing dict key")
@@ -148,31 +128,20 @@ def test_change_one_time_keys(self):
     def test_claim_one_time_key(self):
         local_user = "@boris:" + self.hs.hostname
         device_id = "xyz"
-        keys = {
-            "alg1:k1": "key1",
-        }
+        keys = {"alg1:k1": "key1"}
 
         res = yield self.handler.upload_keys_for_user(
-            local_user, device_id, {"one_time_keys": keys},
+            local_user, device_id, {"one_time_keys": keys}
+        )
+        self.assertDictEqual(res, {"one_time_key_counts": {"alg1": 1}})
+
+        res2 = yield self.handler.claim_one_time_keys(
+            {"one_time_keys": {local_user: {device_id: "alg1"}}}, timeout=None
+        )
+        self.assertEqual(
+            res2,
+            {
+                "failures": {},
+                "one_time_keys": {local_user: {device_id: {"alg1:k1": "key1"}}},
+            },
         )
-        self.assertDictEqual(res, {
-            "one_time_key_counts": {"alg1": 1}
-        })
-
-        res2 = yield self.handler.claim_one_time_keys({
-            "one_time_keys": {
-                local_user: {
-                    device_id: "alg1"
-                }
-            }
-        }, timeout=None)
-        self.assertEqual(res2, {
-            "failures": {},
-            "one_time_keys": {
-                local_user: {
-                    device_id: {
-                        "alg1:k1": "key1"
-                    }
-                }
-            }
-        })
diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py
index 121ce78634ec..fc2b646ba25a 100644
--- a/tests/handlers/test_presence.py
+++ b/tests/handlers/test_presence.py
@@ -39,8 +39,7 @@ def test_offline_to_online(self):
 
         prev_state = UserPresenceState.default(user_id)
         new_state = prev_state.copy_and_replace(
-            state=PresenceState.ONLINE,
-            last_active_ts=now,
+            state=PresenceState.ONLINE, last_active_ts=now
         )
 
         state, persist_and_notify, federation_ping = handle_update(
@@ -54,23 +53,22 @@ def test_offline_to_online(self):
         self.assertEquals(state.last_federation_update_ts, now)
 
         self.assertEquals(wheel_timer.insert.call_count, 3)
-        wheel_timer.insert.assert_has_calls([
-            call(
-                now=now,
-                obj=user_id,
-                then=new_state.last_active_ts + IDLE_TIMER
-            ),
-            call(
-                now=now,
-                obj=user_id,
-                then=new_state.last_user_sync_ts + SYNC_ONLINE_TIMEOUT
-            ),
-            call(
-                now=now,
-                obj=user_id,
-                then=new_state.last_active_ts + LAST_ACTIVE_GRANULARITY
-            ),
-        ], any_order=True)
+        wheel_timer.insert.assert_has_calls(
+            [
+                call(now=now, obj=user_id, then=new_state.last_active_ts + IDLE_TIMER),
+                call(
+                    now=now,
+                    obj=user_id,
+                    then=new_state.last_user_sync_ts + SYNC_ONLINE_TIMEOUT,
+                ),
+                call(
+                    now=now,
+                    obj=user_id,
+                    then=new_state.last_active_ts + LAST_ACTIVE_GRANULARITY,
+                ),
+            ],
+            any_order=True,
+        )
 
     def test_online_to_online(self):
         wheel_timer = Mock()
@@ -79,14 +77,11 @@ def test_online_to_online(self):
 
         prev_state = UserPresenceState.default(user_id)
         prev_state = prev_state.copy_and_replace(
-            state=PresenceState.ONLINE,
-            last_active_ts=now,
-            currently_active=True,
+            state=PresenceState.ONLINE, last_active_ts=now, currently_active=True
         )
 
         new_state = prev_state.copy_and_replace(
-            state=PresenceState.ONLINE,
-            last_active_ts=now,
+            state=PresenceState.ONLINE, last_active_ts=now
         )
 
         state, persist_and_notify, federation_ping = handle_update(
@@ -101,23 +96,22 @@ def test_online_to_online(self):
         self.assertEquals(state.last_federation_update_ts, now)
 
         self.assertEquals(wheel_timer.insert.call_count, 3)
-        wheel_timer.insert.assert_has_calls([
-            call(
-                now=now,
-                obj=user_id,
-                then=new_state.last_active_ts + IDLE_TIMER
-            ),
-            call(
-                now=now,
-                obj=user_id,
-                then=new_state.last_user_sync_ts + SYNC_ONLINE_TIMEOUT
-            ),
-            call(
-                now=now,
-                obj=user_id,
-                then=new_state.last_active_ts + LAST_ACTIVE_GRANULARITY
-            ),
-        ], any_order=True)
+        wheel_timer.insert.assert_has_calls(
+            [
+                call(now=now, obj=user_id, then=new_state.last_active_ts + IDLE_TIMER),
+                call(
+                    now=now,
+                    obj=user_id,
+                    then=new_state.last_user_sync_ts + SYNC_ONLINE_TIMEOUT,
+                ),
+                call(
+                    now=now,
+                    obj=user_id,
+                    then=new_state.last_active_ts + LAST_ACTIVE_GRANULARITY,
+                ),
+            ],
+            any_order=True,
+        )
 
     def test_online_to_online_last_active_noop(self):
         wheel_timer = Mock()
@@ -132,8 +126,7 @@ def test_online_to_online_last_active_noop(self):
         )
 
         new_state = prev_state.copy_and_replace(
-            state=PresenceState.ONLINE,
-            last_active_ts=now,
+            state=PresenceState.ONLINE, last_active_ts=now
         )
 
         state, persist_and_notify, federation_ping = handle_update(
@@ -148,23 +141,22 @@ def test_online_to_online_last_active_noop(self):
         self.assertEquals(state.last_federation_update_ts, now)
 
         self.assertEquals(wheel_timer.insert.call_count, 3)
-        wheel_timer.insert.assert_has_calls([
-            call(
-                now=now,
-                obj=user_id,
-                then=new_state.last_active_ts + IDLE_TIMER
-            ),
-            call(
-                now=now,
-                obj=user_id,
-                then=new_state.last_user_sync_ts + SYNC_ONLINE_TIMEOUT
-            ),
-            call(
-                now=now,
-                obj=user_id,
-                then=new_state.last_active_ts + LAST_ACTIVE_GRANULARITY
-            ),
-        ], any_order=True)
+        wheel_timer.insert.assert_has_calls(
+            [
+                call(now=now, obj=user_id, then=new_state.last_active_ts + IDLE_TIMER),
+                call(
+                    now=now,
+                    obj=user_id,
+                    then=new_state.last_user_sync_ts + SYNC_ONLINE_TIMEOUT,
+                ),
+                call(
+                    now=now,
+                    obj=user_id,
+                    then=new_state.last_active_ts + LAST_ACTIVE_GRANULARITY,
+                ),
+            ],
+            any_order=True,
+        )
 
     def test_online_to_online_last_active(self):
         wheel_timer = Mock()
@@ -178,9 +170,7 @@ def test_online_to_online_last_active(self):
             currently_active=True,
         )
 
-        new_state = prev_state.copy_and_replace(
-            state=PresenceState.ONLINE,
-        )
+        new_state = prev_state.copy_and_replace(state=PresenceState.ONLINE)
 
         state, persist_and_notify, federation_ping = handle_update(
             prev_state, new_state, is_mine=True, wheel_timer=wheel_timer, now=now
@@ -193,18 +183,17 @@ def test_online_to_online_last_active(self):
         self.assertEquals(state.last_federation_update_ts, now)
 
         self.assertEquals(wheel_timer.insert.call_count, 2)
-        wheel_timer.insert.assert_has_calls([
-            call(
-                now=now,
-                obj=user_id,
-                then=new_state.last_active_ts + IDLE_TIMER
-            ),
-            call(
-                now=now,
-                obj=user_id,
-                then=new_state.last_user_sync_ts + SYNC_ONLINE_TIMEOUT
-            )
-        ], any_order=True)
+        wheel_timer.insert.assert_has_calls(
+            [
+                call(now=now, obj=user_id, then=new_state.last_active_ts + IDLE_TIMER),
+                call(
+                    now=now,
+                    obj=user_id,
+                    then=new_state.last_user_sync_ts + SYNC_ONLINE_TIMEOUT,
+                ),
+            ],
+            any_order=True,
+        )
 
     def test_remote_ping_timer(self):
         wheel_timer = Mock()
@@ -213,13 +202,10 @@ def test_remote_ping_timer(self):
 
         prev_state = UserPresenceState.default(user_id)
         prev_state = prev_state.copy_and_replace(
-            state=PresenceState.ONLINE,
-            last_active_ts=now,
+            state=PresenceState.ONLINE, last_active_ts=now
         )
 
-        new_state = prev_state.copy_and_replace(
-            state=PresenceState.ONLINE,
-        )
+        new_state = prev_state.copy_and_replace(state=PresenceState.ONLINE)
 
         state, persist_and_notify, federation_ping = handle_update(
             prev_state, new_state, is_mine=False, wheel_timer=wheel_timer, now=now
@@ -232,13 +218,16 @@ def test_remote_ping_timer(self):
         self.assertEquals(new_state.status_msg, state.status_msg)
 
         self.assertEquals(wheel_timer.insert.call_count, 1)
-        wheel_timer.insert.assert_has_calls([
-            call(
-                now=now,
-                obj=user_id,
-                then=new_state.last_federation_update_ts + FEDERATION_TIMEOUT
-            ),
-        ], any_order=True)
+        wheel_timer.insert.assert_has_calls(
+            [
+                call(
+                    now=now,
+                    obj=user_id,
+                    then=new_state.last_federation_update_ts + FEDERATION_TIMEOUT,
+                )
+            ],
+            any_order=True,
+        )
 
     def test_online_to_offline(self):
         wheel_timer = Mock()
@@ -247,14 +236,10 @@ def test_online_to_offline(self):
 
         prev_state = UserPresenceState.default(user_id)
         prev_state = prev_state.copy_and_replace(
-            state=PresenceState.ONLINE,
-            last_active_ts=now,
-            currently_active=True,
+            state=PresenceState.ONLINE, last_active_ts=now, currently_active=True
         )
 
-        new_state = prev_state.copy_and_replace(
-            state=PresenceState.OFFLINE,
-        )
+        new_state = prev_state.copy_and_replace(state=PresenceState.OFFLINE)
 
         state, persist_and_notify, federation_ping = handle_update(
             prev_state, new_state, is_mine=True, wheel_timer=wheel_timer, now=now
@@ -273,14 +258,10 @@ def test_online_to_idle(self):
 
         prev_state = UserPresenceState.default(user_id)
         prev_state = prev_state.copy_and_replace(
-            state=PresenceState.ONLINE,
-            last_active_ts=now,
-            currently_active=True,
+            state=PresenceState.ONLINE, last_active_ts=now, currently_active=True
         )
 
-        new_state = prev_state.copy_and_replace(
-            state=PresenceState.UNAVAILABLE,
-        )
+        new_state = prev_state.copy_and_replace(state=PresenceState.UNAVAILABLE)
 
         state, persist_and_notify, federation_ping = handle_update(
             prev_state, new_state, is_mine=True, wheel_timer=wheel_timer, now=now
@@ -293,13 +274,16 @@ def test_online_to_idle(self):
         self.assertEquals(new_state.status_msg, state.status_msg)
 
         self.assertEquals(wheel_timer.insert.call_count, 1)
-        wheel_timer.insert.assert_has_calls([
-            call(
-                now=now,
-                obj=user_id,
-                then=new_state.last_user_sync_ts + SYNC_ONLINE_TIMEOUT
-            )
-        ], any_order=True)
+        wheel_timer.insert.assert_has_calls(
+            [
+                call(
+                    now=now,
+                    obj=user_id,
+                    then=new_state.last_user_sync_ts + SYNC_ONLINE_TIMEOUT,
+                )
+            ],
+            any_order=True,
+        )
 
 
 class PresenceTimeoutTestCase(unittest.TestCase):
@@ -314,9 +298,7 @@ def test_idle_timer(self):
             last_user_sync_ts=now,
         )
 
-        new_state = handle_timeout(
-            state, is_mine=True, syncing_user_ids=set(), now=now
-        )
+        new_state = handle_timeout(state, is_mine=True, syncing_user_ids=set(), now=now)
 
         self.assertIsNotNone(new_state)
         self.assertEquals(new_state.state, PresenceState.UNAVAILABLE)
@@ -332,9 +314,7 @@ def test_sync_timeout(self):
             last_user_sync_ts=now - SYNC_ONLINE_TIMEOUT - 1,
         )
 
-        new_state = handle_timeout(
-            state, is_mine=True, syncing_user_ids=set(), now=now
-        )
+        new_state = handle_timeout(state, is_mine=True, syncing_user_ids=set(), now=now)
 
         self.assertIsNotNone(new_state)
         self.assertEquals(new_state.state, PresenceState.OFFLINE)
@@ -369,9 +349,7 @@ def test_federation_ping(self):
             last_federation_update_ts=now - FEDERATION_PING_INTERVAL - 1,
         )
 
-        new_state = handle_timeout(
-            state, is_mine=True, syncing_user_ids=set(), now=now
-        )
+        new_state = handle_timeout(state, is_mine=True, syncing_user_ids=set(), now=now)
 
         self.assertIsNotNone(new_state)
         self.assertEquals(new_state, new_state)
@@ -388,9 +366,7 @@ def test_no_timeout(self):
             last_federation_update_ts=now,
         )
 
-        new_state = handle_timeout(
-            state, is_mine=True, syncing_user_ids=set(), now=now
-        )
+        new_state = handle_timeout(state, is_mine=True, syncing_user_ids=set(), now=now)
 
         self.assertIsNone(new_state)
 
@@ -425,9 +401,7 @@ def test_last_active(self):
             last_federation_update_ts=now,
         )
 
-        new_state = handle_timeout(
-            state, is_mine=True, syncing_user_ids=set(), now=now
-        )
+        new_state = handle_timeout(state, is_mine=True, syncing_user_ids=set(), now=now)
 
         self.assertIsNotNone(new_state)
         self.assertEquals(state, new_state)
diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py
index dc17918a3d6b..9268a6fe2b89 100644
--- a/tests/handlers/test_profile.py
+++ b/tests/handlers/test_profile.py
@@ -54,9 +54,7 @@ def register_query_handler(query_type, handler):
             federation_client=self.mock_federation,
             federation_server=Mock(),
             federation_registry=self.mock_registry,
-            ratelimiter=NonCallableMock(spec_set=[
-                "send_message",
-            ])
+            ratelimiter=NonCallableMock(spec_set=["send_message"]),
         )
 
         self.ratelimiter = hs.get_ratelimiter()
@@ -74,9 +72,7 @@ def register_query_handler(query_type, handler):
 
     @defer.inlineCallbacks
     def test_get_my_name(self):
-        yield self.store.set_profile_displayname(
-            self.frank.localpart, "Frank"
-        )
+        yield self.store.set_profile_displayname(self.frank.localpart, "Frank")
 
         displayname = yield self.handler.get_displayname(self.frank)
 
@@ -85,22 +81,18 @@ def test_get_my_name(self):
     @defer.inlineCallbacks
     def test_set_my_name(self):
         yield self.handler.set_displayname(
-            self.frank,
-            synapse.types.create_requester(self.frank),
-            "Frank Jr."
+            self.frank, synapse.types.create_requester(self.frank), "Frank Jr."
         )
 
         self.assertEquals(
             (yield self.store.get_profile_displayname(self.frank.localpart)),
-            "Frank Jr."
+            "Frank Jr.",
         )
 
     @defer.inlineCallbacks
     def test_set_my_name_noauth(self):
         d = self.handler.set_displayname(
-            self.frank,
-            synapse.types.create_requester(self.bob),
-            "Frank Jr."
+            self.frank, synapse.types.create_requester(self.bob), "Frank Jr."
         )
 
         yield self.assertFailure(d, AuthError)
@@ -145,11 +137,12 @@ def test_get_my_avatar(self):
     @defer.inlineCallbacks
     def test_set_my_avatar(self):
         yield self.handler.set_avatar_url(
-            self.frank, synapse.types.create_requester(self.frank),
-            "http://my.server/pic.gif"
+            self.frank,
+            synapse.types.create_requester(self.frank),
+            "http://my.server/pic.gif",
         )
 
         self.assertEquals(
             (yield self.store.get_profile_avatar_url(self.frank.localpart)),
-            "http://my.server/pic.gif"
+            "http://my.server/pic.gif",
         )
diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py
index 4ea59a58de26..dbec81076f13 100644
--- a/tests/handlers/test_register.py
+++ b/tests/handlers/test_register.py
@@ -46,7 +46,8 @@ def setUp(self):
             profile_handler=Mock(),
         )
         self.macaroon_generator = Mock(
-            generate_access_token=Mock(return_value='secret'))
+            generate_access_token=Mock(return_value='secret')
+        )
         self.hs.get_macaroon_generator = Mock(return_value=self.macaroon_generator)
         self.hs.handlers = RegistrationHandlers(self.hs)
         self.handler = self.hs.get_handlers().registration_handler
@@ -62,7 +63,8 @@ def test_user_is_created_and_logged_in_if_doesnt_exist(self):
         user_id = "@someone:test"
         requester = create_requester("@as:test")
         result_user_id, result_token = yield self.handler.get_or_create_user(
-            requester, local_part, display_name)
+            requester, local_part, display_name
+        )
         self.assertEquals(result_user_id, user_id)
         self.assertEquals(result_token, 'secret')
 
@@ -73,13 +75,15 @@ def test_if_user_exists(self):
         yield store.register(
             user_id=frank.to_string(),
             token="jkv;g498752-43gj['eamb!-5",
-            password_hash=None)
+            password_hash=None,
+        )
         local_part = "frank"
         display_name = "Frank"
         user_id = "@frank:test"
         requester = create_requester("@as:test")
         result_user_id, result_token = yield self.handler.get_or_create_user(
-            requester, local_part, display_name)
+            requester, local_part, display_name
+        )
         self.assertEquals(result_user_id, user_id)
         self.assertEquals(result_token, 'secret')
 
diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py
index f422cf3c5a5f..becfa77bfa9f 100644
--- a/tests/handlers/test_typing.py
+++ b/tests/handlers/test_typing.py
@@ -38,23 +38,19 @@ def _expect_edu(destination, edu_type, content, origin="test"):
         "origin": origin,
         "origin_server_ts": 1000000,
         "pdus": [],
-        "edus": [
-            {
-                "edu_type": edu_type,
-                "content": content,
-            }
-        ],
+        "edus": [{"edu_type": edu_type, "content": content}],
     }
 
 
 def _make_edu_json(origin, edu_type, content):
-    return json.dumps(
-        _expect_edu("test", edu_type, content, origin=origin)
-    ).encode('utf8')
+    return json.dumps(_expect_edu("test", edu_type, content, origin=origin)).encode(
+        'utf8'
+    )
 
 
 class TypingNotificationsTestCase(unittest.TestCase):
     """Tests typing notifications to rooms."""
+
     @defer.inlineCallbacks
     def setUp(self):
         self.clock = MockClock()
@@ -74,18 +70,20 @@ def setUp(self):
             "test",
             auth=self.auth,
             clock=self.clock,
-            datastore=Mock(spec=[
-                # Bits that Federation needs
-                "prep_send_transaction",
-                "delivered_txn",
-                "get_received_txn_response",
-                "set_received_txn_response",
-                "get_destination_retry_timings",
-                "get_devices_by_remote",
-                # Bits that user_directory needs
-                "get_user_directory_stream_pos",
-                "get_current_state_deltas",
-            ]),
+            datastore=Mock(
+                spec=[
+                    # Bits that Federation needs
+                    "prep_send_transaction",
+                    "delivered_txn",
+                    "get_received_txn_response",
+                    "set_received_txn_response",
+                    "get_destination_retry_timings",
+                    "get_devices_by_remote",
+                    # Bits that user_directory needs
+                    "get_user_directory_stream_pos",
+                    "get_current_state_deltas",
+                ]
+            ),
             state_handler=self.state_handler,
             handlers=Mock(),
             notifier=mock_notifier,
@@ -100,19 +98,16 @@ def setUp(self):
         self.event_source = hs.get_event_sources().sources["typing"]
 
         self.datastore = hs.get_datastore()
-        retry_timings_res = {
-            "destination": "",
-            "retry_last_ts": 0,
-            "retry_interval": 0,
-        }
-        self.datastore.get_destination_retry_timings.return_value = (
-            defer.succeed(retry_timings_res)
+        retry_timings_res = {"destination": "", "retry_last_ts": 0, "retry_interval": 0}
+        self.datastore.get_destination_retry_timings.return_value = defer.succeed(
+            retry_timings_res
         )
 
         self.datastore.get_devices_by_remote.return_value = (0, [])
 
         def get_received_txn_response(*args):
             return defer.succeed(None)
+
         self.datastore.get_received_txn_response = get_received_txn_response
 
         self.room_id = "a-room"
@@ -125,10 +120,12 @@ def check_joined_room(room_id, user_id):
 
         def get_joined_hosts_for_room(room_id):
             return set(member.domain for member in self.room_members)
+
         self.datastore.get_joined_hosts_for_room = get_joined_hosts_for_room
 
         def get_current_user_in_room(room_id):
             return set(str(u) for u in self.room_members)
+
         self.state_handler.get_current_user_in_room = get_current_user_in_room
 
         self.datastore.get_user_directory_stream_pos.return_value = (
@@ -136,19 +133,13 @@ def get_current_user_in_room(room_id):
             defer.succeed(1)
         )
 
-        self.datastore.get_current_state_deltas.return_value = (
-            None
-        )
+        self.datastore.get_current_state_deltas.return_value = None
 
         self.auth.check_joined_room = check_joined_room
 
         self.datastore.get_to_device_stream_token = lambda: 0
-        self.datastore.get_new_device_msgs_for_remote = (
-            lambda *args, **kargs: ([], 0)
-        )
-        self.datastore.delete_device_msgs_for_remote = (
-            lambda *args, **kargs: None
-        )
+        self.datastore.get_new_device_msgs_for_remote = lambda *args, **kargs: ([], 0)
+        self.datastore.delete_device_msgs_for_remote = lambda *args, **kargs: None
 
         # Some local users to test with
         self.u_apple = UserID.from_string("@apple:test")
@@ -170,24 +161,23 @@ def test_started_typing_local(self):
             timeout=20000,
         )
 
-        self.on_new_event.assert_has_calls([
-            call('typing_key', 1, rooms=[self.room_id]),
-        ])
+        self.on_new_event.assert_has_calls(
+            [call('typing_key', 1, rooms=[self.room_id])]
+        )
 
         self.assertEquals(self.event_source.get_current_key(), 1)
         events = yield self.event_source.get_new_events(
-            room_ids=[self.room_id],
-            from_key=0,
+            room_ids=[self.room_id], from_key=0
         )
         self.assertEquals(
             events[0],
             [
-                {"type": "m.typing",
-                 "room_id": self.room_id,
-                 "content": {
-                     "user_ids": [self.u_apple.to_string()],
-                 }},
-            ]
+                {
+                    "type": "m.typing",
+                    "room_id": self.room_id,
+                    "content": {"user_ids": [self.u_apple.to_string()]},
+                }
+            ],
         )
 
     @defer.inlineCallbacks
@@ -206,13 +196,13 @@ def test_started_typing_remote_send(self):
                         "room_id": self.room_id,
                         "user_id": self.u_apple.to_string(),
                         "typing": True,
-                    }
+                    },
                 ),
                 json_data_callback=ANY,
                 long_retries=True,
                 backoff_on_404=True,
             ),
-            defer.succeed((200, "OK"))
+            defer.succeed((200, "OK")),
         )
 
         yield self.handler.started_typing(
@@ -240,27 +230,29 @@ def test_started_typing_remote_recv(self):
                     "room_id": self.room_id,
                     "user_id": self.u_onion.to_string(),
                     "typing": True,
-                }
+                },
             ),
             federation_auth=True,
         )
 
-        self.on_new_event.assert_has_calls([
-            call('typing_key', 1, rooms=[self.room_id]),
-        ])
+        self.on_new_event.assert_has_calls(
+            [call('typing_key', 1, rooms=[self.room_id])]
+        )
 
         self.assertEquals(self.event_source.get_current_key(), 1)
         events = yield self.event_source.get_new_events(
-            room_ids=[self.room_id],
-            from_key=0
+            room_ids=[self.room_id], from_key=0
+        )
+        self.assertEquals(
+            events[0],
+            [
+                {
+                    "type": "m.typing",
+                    "room_id": self.room_id,
+                    "content": {"user_ids": [self.u_onion.to_string()]},
+                }
+            ],
         )
-        self.assertEquals(events[0], [{
-            "type": "m.typing",
-            "room_id": self.room_id,
-            "content": {
-                "user_ids": [self.u_onion.to_string()],
-            },
-        }])
 
     @defer.inlineCallbacks
     def test_stopped_typing(self):
@@ -278,17 +270,18 @@ def test_stopped_typing(self):
                         "room_id": self.room_id,
                         "user_id": self.u_apple.to_string(),
                         "typing": False,
-                    }
+                    },
                 ),
                 json_data_callback=ANY,
                 long_retries=True,
                 backoff_on_404=True,
             ),
-            defer.succeed((200, "OK"))
+            defer.succeed((200, "OK")),
         )
 
         # Gut-wrenching
         from synapse.handlers.typing import RoomMember
+
         member = RoomMember(self.room_id, self.u_apple.to_string())
         self.handler._member_typing_until[member] = 1002000
         self.handler._room_typing[self.room_id] = set([self.u_apple.to_string()])
@@ -296,29 +289,29 @@ def test_stopped_typing(self):
         self.assertEquals(self.event_source.get_current_key(), 0)
 
         yield self.handler.stopped_typing(
-            target_user=self.u_apple,
-            auth_user=self.u_apple,
-            room_id=self.room_id,
+            target_user=self.u_apple, auth_user=self.u_apple, room_id=self.room_id
         )
 
-        self.on_new_event.assert_has_calls([
-            call('typing_key', 1, rooms=[self.room_id]),
-        ])
+        self.on_new_event.assert_has_calls(
+            [call('typing_key', 1, rooms=[self.room_id])]
+        )
 
         yield put_json.await_calls()
 
         self.assertEquals(self.event_source.get_current_key(), 1)
         events = yield self.event_source.get_new_events(
-            room_ids=[self.room_id],
-            from_key=0,
+            room_ids=[self.room_id], from_key=0
+        )
+        self.assertEquals(
+            events[0],
+            [
+                {
+                    "type": "m.typing",
+                    "room_id": self.room_id,
+                    "content": {"user_ids": []},
+                }
+            ],
         )
-        self.assertEquals(events[0], [{
-            "type": "m.typing",
-            "room_id": self.room_id,
-            "content": {
-                "user_ids": [],
-            },
-        }])
 
     @defer.inlineCallbacks
     def test_typing_timeout(self):
@@ -333,42 +326,46 @@ def test_typing_timeout(self):
             timeout=10000,
         )
 
-        self.on_new_event.assert_has_calls([
-            call('typing_key', 1, rooms=[self.room_id]),
-        ])
+        self.on_new_event.assert_has_calls(
+            [call('typing_key', 1, rooms=[self.room_id])]
+        )
         self.on_new_event.reset_mock()
 
         self.assertEquals(self.event_source.get_current_key(), 1)
         events = yield self.event_source.get_new_events(
-            room_ids=[self.room_id],
-            from_key=0,
+            room_ids=[self.room_id], from_key=0
+        )
+        self.assertEquals(
+            events[0],
+            [
+                {
+                    "type": "m.typing",
+                    "room_id": self.room_id,
+                    "content": {"user_ids": [self.u_apple.to_string()]},
+                }
+            ],
         )
-        self.assertEquals(events[0], [{
-            "type": "m.typing",
-            "room_id": self.room_id,
-            "content": {
-                "user_ids": [self.u_apple.to_string()],
-            },
-        }])
 
         self.clock.advance_time(16)
 
-        self.on_new_event.assert_has_calls([
-            call('typing_key', 2, rooms=[self.room_id]),
-        ])
+        self.on_new_event.assert_has_calls(
+            [call('typing_key', 2, rooms=[self.room_id])]
+        )
 
         self.assertEquals(self.event_source.get_current_key(), 2)
         events = yield self.event_source.get_new_events(
-            room_ids=[self.room_id],
-            from_key=1,
+            room_ids=[self.room_id], from_key=1
+        )
+        self.assertEquals(
+            events[0],
+            [
+                {
+                    "type": "m.typing",
+                    "room_id": self.room_id,
+                    "content": {"user_ids": []},
+                }
+            ],
         )
-        self.assertEquals(events[0], [{
-            "type": "m.typing",
-            "room_id": self.room_id,
-            "content": {
-                "user_ids": [],
-            },
-        }])
 
         # SYN-230 - see if we can still set after timeout
 
@@ -379,20 +376,22 @@ def test_typing_timeout(self):
             timeout=10000,
         )
 
-        self.on_new_event.assert_has_calls([
-            call('typing_key', 3, rooms=[self.room_id]),
-        ])
+        self.on_new_event.assert_has_calls(
+            [call('typing_key', 3, rooms=[self.room_id])]
+        )
         self.on_new_event.reset_mock()
 
         self.assertEquals(self.event_source.get_current_key(), 3)
         events = yield self.event_source.get_new_events(
-            room_ids=[self.room_id],
-            from_key=0,
+            room_ids=[self.room_id], from_key=0
+        )
+        self.assertEquals(
+            events[0],
+            [
+                {
+                    "type": "m.typing",
+                    "room_id": self.room_id,
+                    "content": {"user_ids": [self.u_apple.to_string()]},
+                }
+            ],
         )
-        self.assertEquals(events[0], [{
-            "type": "m.typing",
-            "room_id": self.room_id,
-            "content": {
-                "user_ids": [self.u_apple.to_string()],
-            },
-        }])
diff --git a/tests/http/test_endpoint.py b/tests/http/test_endpoint.py
index 60e6a7595352..3b0155ed03cc 100644
--- a/tests/http/test_endpoint.py
+++ b/tests/http/test_endpoint.py
@@ -39,15 +39,13 @@ def test_validate_bad_server_names(self):
             "[1234",
             "underscore_.com",
             "percent%65.com",
-            "1234:5678:80",   # too many colons
+            "1234:5678:80",  # too many colons
         ]
         for i in test_data:
             try:
                 parse_and_validate_server_name(i)
                 self.fail(
-                    "Expected parse_and_validate_server_name('%s') to throw" % (
-                        i,
-                    ),
+                    "Expected parse_and_validate_server_name('%s') to throw" % (i,)
                 )
             except ValueError:
                 pass
diff --git a/tests/replication/slave/storage/_base.py b/tests/replication/slave/storage/_base.py
index a103e7be802e..c23b6e2cfd7a 100644
--- a/tests/replication/slave/storage/_base.py
+++ b/tests/replication/slave/storage/_base.py
@@ -31,6 +31,7 @@
 
 class TestReplicationClientHandler(ReplicationClientHandler):
     """Overrides on_rdata so that we can wait for it to happen"""
+
     def __init__(self, store):
         super(TestReplicationClientHandler, self).__init__(store)
         self._rdata_awaiters = []
@@ -56,9 +57,7 @@ def setUp(self):
             "blue",
             http_client=None,
             federation_client=Mock(),
-            ratelimiter=NonCallableMock(spec_set=[
-                "send_message",
-            ]),
+            ratelimiter=NonCallableMock(spec_set=["send_message"]),
         )
         self.hs.get_ratelimiter().send_message.return_value = (True, 0)
 
diff --git a/tests/replication/slave/storage/test_account_data.py b/tests/replication/slave/storage/test_account_data.py
index adf226404e39..87cc2b2fba5f 100644
--- a/tests/replication/slave/storage/test_account_data.py
+++ b/tests/replication/slave/storage/test_account_data.py
@@ -29,20 +29,14 @@ class SlavedAccountDataStoreTestCase(BaseSlavedStoreTestCase):
 
     @defer.inlineCallbacks
     def test_user_account_data(self):
-        yield self.master_store.add_account_data_for_user(
-            USER_ID, TYPE, {"a": 1}
-        )
+        yield self.master_store.add_account_data_for_user(USER_ID, TYPE, {"a": 1})
         yield self.replicate()
         yield self.check(
-            "get_global_account_data_by_type_for_user",
-            [TYPE, USER_ID], {"a": 1}
+            "get_global_account_data_by_type_for_user", [TYPE, USER_ID], {"a": 1}
         )
 
-        yield self.master_store.add_account_data_for_user(
-            USER_ID, TYPE, {"a": 2}
-        )
+        yield self.master_store.add_account_data_for_user(USER_ID, TYPE, {"a": 2})
         yield self.replicate()
         yield self.check(
-            "get_global_account_data_by_type_for_user",
-            [TYPE, USER_ID], {"a": 2}
+            "get_global_account_data_by_type_for_user", [TYPE, USER_ID], {"a": 2}
         )
diff --git a/tests/replication/slave/storage/test_events.py b/tests/replication/slave/storage/test_events.py
index f5b47f5ec0c2..622be2eef8fd 100644
--- a/tests/replication/slave/storage/test_events.py
+++ b/tests/replication/slave/storage/test_events.py
@@ -38,6 +38,7 @@ def patch__eq__(cls):
     def unpatch():
         if eq is not None:
             cls.__eq__ = eq
+
     return unpatch
 
 
@@ -48,10 +49,7 @@ class SlavedEventStoreTestCase(BaseSlavedStoreTestCase):
     def setUp(self):
         # Patch up the equality operator for events so that we can check
         # whether lists of events match using assertEquals
-        self.unpatches = [
-            patch__eq__(_EventInternalMetadata),
-            patch__eq__(FrozenEvent),
-        ]
+        self.unpatches = [patch__eq__(_EventInternalMetadata), patch__eq__(FrozenEvent)]
         return super(SlavedEventStoreTestCase, self).setUp()
 
     def tearDown(self):
@@ -61,33 +59,27 @@ def tearDown(self):
     def test_get_latest_event_ids_in_room(self):
         create = yield self.persist(type="m.room.create", key="", creator=USER_ID)
         yield self.replicate()
-        yield self.check(
-            "get_latest_event_ids_in_room", (ROOM_ID,), [create.event_id]
-        )
+        yield self.check("get_latest_event_ids_in_room", (ROOM_ID,), [create.event_id])
 
         join = yield self.persist(
-            type="m.room.member", key=USER_ID, membership="join",
+            type="m.room.member",
+            key=USER_ID,
+            membership="join",
             prev_events=[(create.event_id, {})],
         )
         yield self.replicate()
-        yield self.check(
-            "get_latest_event_ids_in_room", (ROOM_ID,), [join.event_id]
-        )
+        yield self.check("get_latest_event_ids_in_room", (ROOM_ID,), [join.event_id])
 
     @defer.inlineCallbacks
     def test_redactions(self):
         yield self.persist(type="m.room.create", key="", creator=USER_ID)
         yield self.persist(type="m.room.member", key=USER_ID, membership="join")
 
-        msg = yield self.persist(
-            type="m.room.message", msgtype="m.text", body="Hello"
-        )
+        msg = yield self.persist(type="m.room.message", msgtype="m.text", body="Hello")
         yield self.replicate()
         yield self.check("get_event", [msg.event_id], msg)
 
-        redaction = yield self.persist(
-            type="m.room.redaction", redacts=msg.event_id
-        )
+        redaction = yield self.persist(type="m.room.redaction", redacts=msg.event_id)
         yield self.replicate()
 
         msg_dict = msg.get_dict()
@@ -102,9 +94,7 @@ def test_backfilled_redactions(self):
         yield self.persist(type="m.room.create", key="", creator=USER_ID)
         yield self.persist(type="m.room.member", key=USER_ID, membership="join")
 
-        msg = yield self.persist(
-            type="m.room.message", msgtype="m.text", body="Hello"
-        )
+        msg = yield self.persist(type="m.room.message", msgtype="m.text", body="Hello")
         yield self.replicate()
         yield self.check("get_event", [msg.event_id], msg)
 
@@ -127,10 +117,19 @@ def test_invites(self):
             type="m.room.member", key=USER_ID_2, membership="invite"
         )
         yield self.replicate()
-        yield self.check("get_invited_rooms_for_user", [USER_ID_2], [RoomsForUser(
-            ROOM_ID, USER_ID, "invite", event.event_id,
-            event.internal_metadata.stream_ordering
-        )])
+        yield self.check(
+            "get_invited_rooms_for_user",
+            [USER_ID_2],
+            [
+                RoomsForUser(
+                    ROOM_ID,
+                    USER_ID,
+                    "invite",
+                    event.event_id,
+                    event.internal_metadata.stream_ordering,
+                )
+            ],
+        )
 
     @defer.inlineCallbacks
     def test_push_actions_for_user(self):
@@ -146,40 +145,55 @@ def test_push_actions_for_user(self):
         yield self.check(
             "get_unread_event_push_actions_by_room_for_user",
             [ROOM_ID, USER_ID_2, event1.event_id],
-            {"highlight_count": 0, "notify_count": 0}
+            {"highlight_count": 0, "notify_count": 0},
         )
 
         yield self.persist(
-            type="m.room.message", msgtype="m.text", body="world",
+            type="m.room.message",
+            msgtype="m.text",
+            body="world",
             push_actions=[(USER_ID_2, ["notify"])],
         )
         yield self.replicate()
         yield self.check(
             "get_unread_event_push_actions_by_room_for_user",
             [ROOM_ID, USER_ID_2, event1.event_id],
-            {"highlight_count": 0, "notify_count": 1}
+            {"highlight_count": 0, "notify_count": 1},
         )
 
         yield self.persist(
-            type="m.room.message", msgtype="m.text", body="world",
-            push_actions=[(USER_ID_2, [
-                "notify", {"set_tweak": "highlight", "value": True}
-            ])],
+            type="m.room.message",
+            msgtype="m.text",
+            body="world",
+            push_actions=[
+                (USER_ID_2, ["notify", {"set_tweak": "highlight", "value": True}])
+            ],
         )
         yield self.replicate()
         yield self.check(
             "get_unread_event_push_actions_by_room_for_user",
             [ROOM_ID, USER_ID_2, event1.event_id],
-            {"highlight_count": 1, "notify_count": 2}
+            {"highlight_count": 1, "notify_count": 2},
         )
 
     event_id = 0
 
     @defer.inlineCallbacks
     def persist(
-        self, sender=USER_ID, room_id=ROOM_ID, type={}, key=None, internal={},
-        state=None, reset_state=False, backfill=False,
-        depth=None, prev_events=[], auth_events=[], prev_state=[], redacts=None,
+        self,
+        sender=USER_ID,
+        room_id=ROOM_ID,
+        type={},
+        key=None,
+        internal={},
+        state=None,
+        reset_state=False,
+        backfill=False,
+        depth=None,
+        prev_events=[],
+        auth_events=[],
+        prev_state=[],
+        redacts=None,
         push_actions=[],
         **content
     ):
@@ -219,34 +233,23 @@ def persist(
         self.event_id += 1
 
         if state is not None:
-            state_ids = {
-                key: e.event_id for key, e in state.items()
-            }
+            state_ids = {key: e.event_id for key, e in state.items()}
             context = EventContext.with_state(
-                state_group=None,
-                current_state_ids=state_ids,
-                prev_state_ids=state_ids
+                state_group=None, current_state_ids=state_ids, prev_state_ids=state_ids
             )
         else:
             state_handler = self.hs.get_state_handler()
             context = yield state_handler.compute_event_context(event)
 
         yield self.master_store.add_push_actions_to_staging(
-            event.event_id, {
-                user_id: actions
-                for user_id, actions in push_actions
-            },
+            event.event_id, {user_id: actions for user_id, actions in push_actions}
         )
 
         ordering = None
         if backfill:
-            yield self.master_store.persist_events(
-                [(event, context)], backfilled=True
-            )
+            yield self.master_store.persist_events([(event, context)], backfilled=True)
         else:
-            ordering, _ = yield self.master_store.persist_event(
-                event, context,
-            )
+            ordering, _ = yield self.master_store.persist_event(event, context)
 
         if ordering:
             event.internal_metadata.stream_ordering = ordering
diff --git a/tests/replication/slave/storage/test_receipts.py b/tests/replication/slave/storage/test_receipts.py
index e6d670cc1f27..ae1adeded103 100644
--- a/tests/replication/slave/storage/test_receipts.py
+++ b/tests/replication/slave/storage/test_receipts.py
@@ -34,6 +34,6 @@ def test_receipt(self):
             ROOM_ID, "m.read", USER_ID, [EVENT_ID], {}
         )
         yield self.replicate()
-        yield self.check("get_receipts_for_user", [USER_ID, "m.read"], {
-            ROOM_ID: EVENT_ID
-        })
+        yield self.check(
+            "get_receipts_for_user", [USER_ID, "m.read"], {ROOM_ID: EVENT_ID}
+        )
diff --git a/tests/rest/client/test_transactions.py b/tests/rest/client/test_transactions.py
index d46c27e7e9f7..708dc26e6147 100644
--- a/tests/rest/client/test_transactions.py
+++ b/tests/rest/client/test_transactions.py
@@ -11,7 +11,6 @@
 
 
 class HttpTransactionCacheTestCase(unittest.TestCase):
-
     def setUp(self):
         self.clock = MockClock()
         self.hs = Mock()
@@ -24,9 +23,7 @@ def setUp(self):
 
     @defer.inlineCallbacks
     def test_executes_given_function(self):
-        cb = Mock(
-            return_value=defer.succeed(self.mock_http_response)
-        )
+        cb = Mock(return_value=defer.succeed(self.mock_http_response))
         res = yield self.cache.fetch_or_execute(
             self.mock_key, cb, "some_arg", keyword="arg"
         )
@@ -35,9 +32,7 @@ def test_executes_given_function(self):
 
     @defer.inlineCallbacks
     def test_deduplicates_based_on_key(self):
-        cb = Mock(
-            return_value=defer.succeed(self.mock_http_response)
-        )
+        cb = Mock(return_value=defer.succeed(self.mock_http_response))
         for i in range(3):  # invoke multiple times
             res = yield self.cache.fetch_or_execute(
                 self.mock_key, cb, "some_arg", keyword="arg", changing_args=i
@@ -120,29 +115,18 @@ def cb():
 
     @defer.inlineCallbacks
     def test_cleans_up(self):
-        cb = Mock(
-            return_value=defer.succeed(self.mock_http_response)
-        )
-        yield self.cache.fetch_or_execute(
-            self.mock_key, cb, "an arg"
-        )
+        cb = Mock(return_value=defer.succeed(self.mock_http_response))
+        yield self.cache.fetch_or_execute(self.mock_key, cb, "an arg")
         # should NOT have cleaned up yet
         self.clock.advance_time_msec(CLEANUP_PERIOD_MS / 2)
 
-        yield self.cache.fetch_or_execute(
-            self.mock_key, cb, "an arg"
-        )
+        yield self.cache.fetch_or_execute(self.mock_key, cb, "an arg")
         # still using cache
         cb.assert_called_once_with("an arg")
 
         self.clock.advance_time_msec(CLEANUP_PERIOD_MS)
 
-        yield self.cache.fetch_or_execute(
-            self.mock_key, cb, "an arg"
-        )
+        yield self.cache.fetch_or_execute(self.mock_key, cb, "an arg")
         # no longer using cache
         self.assertEqual(cb.call_count, 2)
-        self.assertEqual(
-            cb.call_args_list,
-            [call("an arg",), call("an arg",)]
-        )
+        self.assertEqual(cb.call_args_list, [call("an arg"), call("an arg")])
diff --git a/tests/rest/client/v1/test_admin.py b/tests/rest/client/v1/test_admin.py
index fb28883d30c4..67d9ab94e2c8 100644
--- a/tests/rest/client/v1/test_admin.py
+++ b/tests/rest/client/v1/test_admin.py
@@ -215,6 +215,7 @@ def test_missing_parts(self):
         mac.  Admin is optional.  Additional checks are done for length and
         type.
         """
+
         def nonce():
             request, channel = make_request("GET", self.url)
             render(request, self.resource, self.clock)
@@ -289,7 +290,9 @@ def nonce():
         self.assertEqual('Invalid password', channel.json_body["error"])
 
         # Must not have null bytes
-        body = json.dumps({"nonce": nonce(), "username": "a", "password": u"abcd\u0000"})
+        body = json.dumps(
+            {"nonce": nonce(), "username": "a", "password": u"abcd\u0000"}
+        )
         request, channel = make_request("POST", self.url, body.encode('utf8'))
         render(request, self.resource, self.clock)
 
diff --git a/tests/rest/client/v1/test_events.py b/tests/rest/client/v1/test_events.py
index 50418153fa91..0316b74fa111 100644
--- a/tests/rest/client/v1/test_events.py
+++ b/tests/rest/client/v1/test_events.py
@@ -43,9 +43,7 @@ def setUp(self):
         hs = yield setup_test_homeserver(
             http_client=None,
             federation_client=Mock(),
-            ratelimiter=NonCallableMock(spec_set=[
-                "send_message",
-            ]),
+            ratelimiter=NonCallableMock(spec_set=["send_message"]),
         )
         self.ratelimiter = hs.get_ratelimiter()
         self.ratelimiter.send_message.return_value = (True, 0)
@@ -83,7 +81,7 @@ def test_stream_basic_permissions(self):
         # behaviour is used instead to be consistent with the r0 spec.
         # see issue #2602
         (code, response) = yield self.mock_resource.trigger_get(
-            "/events?access_token=%s" % ("invalid" + self.token, )
+            "/events?access_token=%s" % ("invalid" + self.token,)
         )
         self.assertEquals(401, code, msg=str(response))
 
@@ -98,18 +96,12 @@ def test_stream_basic_permissions(self):
 
     @defer.inlineCallbacks
     def test_stream_room_permissions(self):
-        room_id = yield self.create_room_as(
-            self.other_user,
-            tok=self.other_token
-        )
+        room_id = yield self.create_room_as(self.other_user, tok=self.other_token)
         yield self.send(room_id, tok=self.other_token)
 
         # invited to room (expect no content for room)
         yield self.invite(
-            room_id,
-            src=self.other_user,
-            targ=self.user_id,
-            tok=self.other_token
+            room_id, src=self.other_user, targ=self.user_id, tok=self.other_token
         )
 
         (code, response) = yield self.mock_resource.trigger_get(
@@ -120,13 +112,16 @@ def test_stream_room_permissions(self):
         # We may get a presence event for ourselves down
         self.assertEquals(
             0,
-            len([
-                c for c in response["chunk"]
-                if not (
-                    c.get("type") == "m.presence"
-                    and c["content"].get("user_id") == self.user_id
-                )
-            ])
+            len(
+                [
+                    c
+                    for c in response["chunk"]
+                    if not (
+                        c.get("type") == "m.presence"
+                        and c["content"].get("user_id") == self.user_id
+                    )
+                ]
+            ),
         )
 
         # joined room (expect all content for room)
diff --git a/tests/rest/client/v1/test_profile.py b/tests/rest/client/v1/test_profile.py
index 0516ce3cfbd2..9ba0ffc19f8a 100644
--- a/tests/rest/client/v1/test_profile.py
+++ b/tests/rest/client/v1/test_profile.py
@@ -36,12 +36,14 @@ class ProfileTestCase(unittest.TestCase):
     @defer.inlineCallbacks
     def setUp(self):
         self.mock_resource = MockHttpResource(prefix=PATH_PREFIX)
-        self.mock_handler = Mock(spec=[
-            "get_displayname",
-            "set_displayname",
-            "get_avatar_url",
-            "set_avatar_url",
-        ])
+        self.mock_handler = Mock(
+            spec=[
+                "get_displayname",
+                "set_displayname",
+                "get_avatar_url",
+                "set_avatar_url",
+            ]
+        )
 
         hs = yield setup_test_homeserver(
             "test",
@@ -49,7 +51,7 @@ def setUp(self):
             resource_for_client=self.mock_resource,
             federation=Mock(),
             federation_client=Mock(),
-            profile_handler=self.mock_handler
+            profile_handler=self.mock_handler,
         )
 
         def _get_user_by_req(request=None, allow_guest=False):
@@ -78,9 +80,7 @@ def test_set_my_name(self):
         mocked_set.return_value = defer.succeed(())
 
         (code, response) = yield self.mock_resource.trigger(
-            "PUT",
-            "/profile/%s/displayname" % (myid),
-            b'{"displayname": "Frank Jr."}'
+            "PUT", "/profile/%s/displayname" % (myid), b'{"displayname": "Frank Jr."}'
         )
 
         self.assertEquals(200, code)
@@ -94,14 +94,12 @@ def test_set_my_name_noauth(self):
         mocked_set.side_effect = AuthError(400, "message")
 
         (code, response) = yield self.mock_resource.trigger(
-            "PUT", "/profile/%s/displayname" % ("@4567:test"),
-            b'{"displayname": "Frank Jr."}'
+            "PUT",
+            "/profile/%s/displayname" % ("@4567:test"),
+            b'{"displayname": "Frank Jr."}',
         )
 
-        self.assertTrue(
-            400 <= code < 499,
-            msg="code %d is in the 4xx range" % (code)
-        )
+        self.assertTrue(400 <= code < 499, msg="code %d is in the 4xx range" % (code))
 
     @defer.inlineCallbacks
     def test_get_other_name(self):
@@ -121,14 +119,12 @@ def test_set_other_name(self):
         mocked_set.side_effect = SynapseError(400, "message")
 
         (code, response) = yield self.mock_resource.trigger(
-            "PUT", "/profile/%s/displayname" % ("@opaque:elsewhere"),
-            b'{"displayname":"bob"}'
+            "PUT",
+            "/profile/%s/displayname" % ("@opaque:elsewhere"),
+            b'{"displayname":"bob"}',
         )
 
-        self.assertTrue(
-            400 <= code <= 499,
-            msg="code %d is in the 4xx range" % (code)
-        )
+        self.assertTrue(400 <= code <= 499, msg="code %d is in the 4xx range" % (code))
 
     @defer.inlineCallbacks
     def test_get_my_avatar(self):
@@ -151,7 +147,7 @@ def test_set_my_avatar(self):
         (code, response) = yield self.mock_resource.trigger(
             "PUT",
             "/profile/%s/avatar_url" % (myid),
-            b'{"avatar_url": "http://my.server/pic.gif"}'
+            b'{"avatar_url": "http://my.server/pic.gif"}',
         )
 
         self.assertEquals(200, code)
diff --git a/tests/rest/client/v1/test_register.py b/tests/rest/client/v1/test_register.py
index 83a23cd8fe3c..6f15d69ecd54 100644
--- a/tests/rest/client/v1/test_register.py
+++ b/tests/rest/client/v1/test_register.py
@@ -32,6 +32,7 @@ class CreateUserServletTestCase(unittest.TestCase):
     """
     Tests for CreateUserRestServlet.
     """
+
     if PY3:
         skip = "Not ported to Python 3."
 
diff --git a/tests/rest/client/v1/test_typing.py b/tests/rest/client/v1/test_typing.py
index bddb3302e481..7f1a435e7ba3 100644
--- a/tests/rest/client/v1/test_typing.py
+++ b/tests/rest/client/v1/test_typing.py
@@ -31,6 +31,7 @@
 
 class RoomTypingTestCase(RestTestCase):
     """ Tests /rooms/$room_id/typing/$user_id REST API. """
+
     user_id = "@sid:red"
 
     user = UserID.from_string(user_id)
@@ -47,9 +48,7 @@ def setUp(self):
             clock=self.clock,
             http_client=None,
             federation_client=Mock(),
-            ratelimiter=NonCallableMock(spec_set=[
-                "send_message",
-            ]),
+            ratelimiter=NonCallableMock(spec_set=["send_message"]),
         )
         self.hs = hs
 
@@ -71,6 +70,7 @@ def get_user_by_access_token(token=None, allow_guest=False):
 
         def _insert_client_ip(*args, **kwargs):
             return defer.succeed(None)
+
         hs.get_datastore().insert_client_ip = _insert_client_ip
 
         def get_room_members(room_id):
@@ -94,6 +94,7 @@ def fetch_room_distributions_into(
                 else:
                     if remotedomains is not None:
                         remotedomains.add(member.domain)
+
         hs.get_room_member_handler().fetch_room_distributions_into = (
             fetch_room_distributions_into
         )
@@ -107,37 +108,42 @@ def fetch_room_distributions_into(
     @defer.inlineCallbacks
     def test_set_typing(self):
         (code, _) = yield self.mock_resource.trigger(
-            "PUT", "/rooms/%s/typing/%s" % (self.room_id, self.user_id),
-            '{"typing": true, "timeout": 30000}'
+            "PUT",
+            "/rooms/%s/typing/%s" % (self.room_id, self.user_id),
+            '{"typing": true, "timeout": 30000}',
         )
         self.assertEquals(200, code)
 
         self.assertEquals(self.event_source.get_current_key(), 1)
         events = yield self.event_source.get_new_events(
-            from_key=0,
-            room_ids=[self.room_id],
+            from_key=0, room_ids=[self.room_id]
+        )
+        self.assertEquals(
+            events[0],
+            [
+                {
+                    "type": "m.typing",
+                    "room_id": self.room_id,
+                    "content": {"user_ids": [self.user_id]},
+                }
+            ],
         )
-        self.assertEquals(events[0], [{
-            "type": "m.typing",
-            "room_id": self.room_id,
-            "content": {
-                "user_ids": [self.user_id],
-            }
-        }])
 
     @defer.inlineCallbacks
     def test_set_not_typing(self):
         (code, _) = yield self.mock_resource.trigger(
-            "PUT", "/rooms/%s/typing/%s" % (self.room_id, self.user_id),
-            '{"typing": false}'
+            "PUT",
+            "/rooms/%s/typing/%s" % (self.room_id, self.user_id),
+            '{"typing": false}',
         )
         self.assertEquals(200, code)
 
     @defer.inlineCallbacks
     def test_typing_timeout(self):
         (code, _) = yield self.mock_resource.trigger(
-            "PUT", "/rooms/%s/typing/%s" % (self.room_id, self.user_id),
-            '{"typing": true, "timeout": 30000}'
+            "PUT",
+            "/rooms/%s/typing/%s" % (self.room_id, self.user_id),
+            '{"typing": true, "timeout": 30000}',
         )
         self.assertEquals(200, code)
 
@@ -148,8 +154,9 @@ def test_typing_timeout(self):
         self.assertEquals(self.event_source.get_current_key(), 2)
 
         (code, _) = yield self.mock_resource.trigger(
-            "PUT", "/rooms/%s/typing/%s" % (self.room_id, self.user_id),
-            '{"typing": true, "timeout": 30000}'
+            "PUT",
+            "/rooms/%s/typing/%s" % (self.room_id, self.user_id),
+            '{"typing": true, "timeout": 30000}',
         )
         self.assertEquals(200, code)
 
diff --git a/tests/rest/client/v1/utils.py b/tests/rest/client/v1/utils.py
index e3bc5f378d22..9f862f9dfa02 100644
--- a/tests/rest/client/v1/utils.py
+++ b/tests/rest/client/v1/utils.py
@@ -55,25 +55,39 @@ def create_room_as(self, room_creator, is_public=True, tok=None):
 
     @defer.inlineCallbacks
     def invite(self, room=None, src=None, targ=None, expect_code=200, tok=None):
-        yield self.change_membership(room=room, src=src, targ=targ, tok=tok,
-                                     membership=Membership.INVITE,
-                                     expect_code=expect_code)
+        yield self.change_membership(
+            room=room,
+            src=src,
+            targ=targ,
+            tok=tok,
+            membership=Membership.INVITE,
+            expect_code=expect_code,
+        )
 
     @defer.inlineCallbacks
     def join(self, room=None, user=None, expect_code=200, tok=None):
-        yield self.change_membership(room=room, src=user, targ=user, tok=tok,
-                                     membership=Membership.JOIN,
-                                     expect_code=expect_code)
+        yield self.change_membership(
+            room=room,
+            src=user,
+            targ=user,
+            tok=tok,
+            membership=Membership.JOIN,
+            expect_code=expect_code,
+        )
 
     @defer.inlineCallbacks
     def leave(self, room=None, user=None, expect_code=200, tok=None):
-        yield self.change_membership(room=room, src=user, targ=user, tok=tok,
-                                     membership=Membership.LEAVE,
-                                     expect_code=expect_code)
+        yield self.change_membership(
+            room=room,
+            src=user,
+            targ=user,
+            tok=tok,
+            membership=Membership.LEAVE,
+            expect_code=expect_code,
+        )
 
     @defer.inlineCallbacks
-    def change_membership(self, room, src, targ, membership, tok=None,
-                          expect_code=200):
+    def change_membership(self, room, src, targ, membership, tok=None, expect_code=200):
         temp_id = self.auth_user_id
         self.auth_user_id = src
 
@@ -81,16 +95,15 @@ def change_membership(self, room, src, targ, membership, tok=None,
         if tok:
             path = path + "?access_token=%s" % tok
 
-        data = {
-            "membership": membership
-        }
+        data = {"membership": membership}
 
         (code, response) = yield self.mock_resource.trigger(
             "PUT", path, json.dumps(data)
         )
         self.assertEquals(
-            expect_code, code,
-            msg="Expected: %d, got: %d, resp: %r" % (expect_code, code, response)
+            expect_code,
+            code,
+            msg="Expected: %d, got: %d, resp: %r" % (expect_code, code, response),
         )
 
         self.auth_user_id = temp_id
@@ -100,17 +113,15 @@ def register(self, user_id):
         (code, response) = yield self.mock_resource.trigger(
             "POST",
             "/register",
-            json.dumps({
-                "user": user_id,
-                "password": "test",
-                "type": "m.login.password"
-            }))
+            json.dumps(
+                {"user": user_id, "password": "test", "type": "m.login.password"}
+            ),
+        )
         self.assertEquals(200, code, msg=response)
         defer.returnValue(response)
 
     @defer.inlineCallbacks
-    def send(self, room_id, body=None, txn_id=None, tok=None,
-             expect_code=200):
+    def send(self, room_id, body=None, txn_id=None, tok=None, expect_code=200):
         if txn_id is None:
             txn_id = "m%s" % (str(time.time()))
         if body is None:
@@ -132,8 +143,9 @@ def assert_dict(self, required, actual):
             actual (dict): The test result. Extra keys will not be checked.
         """
         for key in required:
-            self.assertEquals(required[key], actual[key],
-                              msg="%s mismatch. %s" % (key, actual))
+            self.assertEquals(
+                required[key], actual[key], msg="%s mismatch. %s" % (key, actual)
+            )
 
 
 @attr.s
@@ -156,7 +168,9 @@ def create_room_as(self, room_creator, is_public=True, tok=None):
         if tok:
             path = path + "?access_token=%s" % tok
 
-        request, channel = make_request("POST", path, json.dumps(content).encode('utf8'))
+        request, channel = make_request(
+            "POST", path, json.dumps(content).encode('utf8')
+        )
         request.render(self.resource)
         wait_until_result(self.hs.get_reactor(), channel)
 
@@ -204,9 +218,7 @@ def change_membership(self, room, src, targ, membership, tok=None, expect_code=2
 
         data = {"membership": membership}
 
-        request, channel = make_request(
-            "PUT", path, json.dumps(data).encode('utf8')
-        )
+        request, channel = make_request("PUT", path, json.dumps(data).encode('utf8'))
 
         request.render(self.resource)
         wait_until_result(self.hs.get_reactor(), channel)
diff --git a/tests/rest/client/v2_alpha/test_register.py b/tests/rest/client/v2_alpha/test_register.py
index f6293f11a871..9487babac343 100644
--- a/tests/rest/client/v2_alpha/test_register.py
+++ b/tests/rest/client/v2_alpha/test_register.py
@@ -101,9 +101,7 @@ def test_POST_bad_password(self):
         wait_until_result(self.clock, channel)
 
         self.assertEquals(channel.result["code"], b"400", channel.result)
-        self.assertEquals(
-            channel.json_body["error"], "Invalid password"
-        )
+        self.assertEquals(channel.json_body["error"], "Invalid password")
 
     def test_POST_bad_username(self):
         request_data = json.dumps({"username": 777, "password": "monkey"})
@@ -112,9 +110,7 @@ def test_POST_bad_username(self):
         wait_until_result(self.clock, channel)
 
         self.assertEquals(channel.result["code"], b"400", channel.result)
-        self.assertEquals(
-            channel.json_body["error"], "Invalid username"
-        )
+        self.assertEquals(channel.json_body["error"], "Invalid username")
 
     def test_POST_user_valid(self):
         user_id = "@kermit:muppet"
@@ -157,10 +153,7 @@ def test_POST_disabled_registration(self):
         wait_until_result(self.clock, channel)
 
         self.assertEquals(channel.result["code"], b"403", channel.result)
-        self.assertEquals(
-            channel.json_body["error"],
-            "Registration has been disabled",
-        )
+        self.assertEquals(channel.json_body["error"], "Registration has been disabled")
 
     def test_POST_guest_registration(self):
         user_id = "a@b"
@@ -188,6 +181,4 @@ def test_POST_disabled_guest_registration(self):
         wait_until_result(self.clock, channel)
 
         self.assertEquals(channel.result["code"], b"403", channel.result)
-        self.assertEquals(
-            channel.json_body["error"], "Guest access is disabled"
-        )
+        self.assertEquals(channel.json_body["error"], "Guest access is disabled")
diff --git a/tests/rest/media/v1/test_media_storage.py b/tests/rest/media/v1/test_media_storage.py
index bf254a260d45..a86901c2d833 100644
--- a/tests/rest/media/v1/test_media_storage.py
+++ b/tests/rest/media/v1/test_media_storage.py
@@ -41,13 +41,11 @@ def setUp(self):
         hs.get_reactor = Mock(return_value=reactor)
         hs.config.media_store_path = self.primary_base_path
 
-        storage_providers = [FileStorageProviderBackend(
-            hs, self.secondary_base_path
-        )]
+        storage_providers = [FileStorageProviderBackend(hs, self.secondary_base_path)]
 
         self.filepaths = MediaFilePaths(self.primary_base_path)
         self.media_storage = MediaStorage(
-            hs, self.primary_base_path, self.filepaths, storage_providers,
+            hs, self.primary_base_path, self.filepaths, storage_providers
         )
 
     def tearDown(self):
diff --git a/tests/server.py b/tests/server.py
index e249668d2104..05708be8b917 100644
--- a/tests/server.py
+++ b/tests/server.py
@@ -136,6 +136,7 @@ class ThreadedMemoryReactorClock(MemoryReactorClock):
     """
     A MemoryReactorClock that supports callFromThread.
     """
+
     def callFromThread(self, callback, *args, **kwargs):
         """
         Make the callback fire in the next reactor iteration.
@@ -184,6 +185,7 @@ class ThreadPool:
         """
         Threadless thread pool.
         """
+
         def start(self):
             pass
 
diff --git a/tests/storage/test__base.py b/tests/storage/test__base.py
index 52eff2a10494..52eb05bfbf90 100644
--- a/tests/storage/test__base.py
+++ b/tests/storage/test__base.py
@@ -25,7 +25,6 @@
 
 
 class CacheTestCase(unittest.TestCase):
-
     def setUp(self):
         self.cache = Cache("test")
 
@@ -97,7 +96,6 @@ def test_eviction_lru(self):
 
 
 class CacheDecoratorTestCase(unittest.TestCase):
-
     @defer.inlineCallbacks
     def test_passthrough(self):
         class A(object):
@@ -180,8 +178,7 @@ def func(self, key):
             yield a.func(k)
 
         self.assertTrue(
-            callcount[0] >= 14,
-            msg="Expected callcount >= 14, got %d" % (callcount[0])
+            callcount[0] >= 14, msg="Expected callcount >= 14, got %d" % (callcount[0])
         )
 
     def test_prefill(self):
diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py
index 099861b27cf1..fbb25a88441a 100644
--- a/tests/storage/test_appservice.py
+++ b/tests/storage/test_appservice.py
@@ -34,7 +34,6 @@
 
 
 class ApplicationServiceStoreTestCase(unittest.TestCase):
-
     @defer.inlineCallbacks
     def setUp(self):
         self.as_yaml_files = []
@@ -44,20 +43,14 @@ def setUp(self):
             password_providers=[],
         )
         hs = yield setup_test_homeserver(
-            config=config,
-            federation_sender=Mock(),
-            federation_client=Mock(),
+            config=config, federation_sender=Mock(), federation_client=Mock()
         )
 
         self.as_token = "token1"
         self.as_url = "some_url"
         self.as_id = "as1"
         self._add_appservice(
-            self.as_token,
-            self.as_id,
-            self.as_url,
-            "some_hs_token",
-            "bob"
+            self.as_token, self.as_id, self.as_url, "some_hs_token", "bob"
         )
         self._add_appservice("token2", "as2", "some_url", "some_hs_token", "bob")
         self._add_appservice("token3", "as3", "some_url", "some_hs_token", "bob")
@@ -73,8 +66,14 @@ def tearDown(self):
                 pass
 
     def _add_appservice(self, as_token, id, url, hs_token, sender):
-        as_yaml = dict(url=url, as_token=as_token, hs_token=hs_token,
-                       id=id, sender_localpart=sender, namespaces={})
+        as_yaml = dict(
+            url=url,
+            as_token=as_token,
+            hs_token=hs_token,
+            id=id,
+            sender_localpart=sender,
+            namespaces={},
+        )
         # use the token as the filename
         with open(as_token, 'w') as outfile:
             outfile.write(yaml.dump(as_yaml))
@@ -85,24 +84,13 @@ def test_retrieve_unknown_service_token(self):
         self.assertEquals(service, None)
 
     def test_retrieval_of_service(self):
-        stored_service = self.store.get_app_service_by_token(
-            self.as_token
-        )
+        stored_service = self.store.get_app_service_by_token(self.as_token)
         self.assertEquals(stored_service.token, self.as_token)
         self.assertEquals(stored_service.id, self.as_id)
         self.assertEquals(stored_service.url, self.as_url)
-        self.assertEquals(
-            stored_service.namespaces[ApplicationService.NS_ALIASES],
-            []
-        )
-        self.assertEquals(
-            stored_service.namespaces[ApplicationService.NS_ROOMS],
-            []
-        )
-        self.assertEquals(
-            stored_service.namespaces[ApplicationService.NS_USERS],
-            []
-        )
+        self.assertEquals(stored_service.namespaces[ApplicationService.NS_ALIASES], [])
+        self.assertEquals(stored_service.namespaces[ApplicationService.NS_ROOMS], [])
+        self.assertEquals(stored_service.namespaces[ApplicationService.NS_USERS], [])
 
     def test_retrieval_of_all_services(self):
         services = self.store.get_app_services()
@@ -110,7 +98,6 @@ def test_retrieval_of_all_services(self):
 
 
 class ApplicationServiceTransactionStoreTestCase(unittest.TestCase):
-
     @defer.inlineCallbacks
     def setUp(self):
         self.as_yaml_files = []
@@ -121,33 +108,15 @@ def setUp(self):
             password_providers=[],
         )
         hs = yield setup_test_homeserver(
-            config=config,
-            federation_sender=Mock(),
-            federation_client=Mock(),
+            config=config, federation_sender=Mock(), federation_client=Mock()
         )
         self.db_pool = hs.get_db_pool()
 
         self.as_list = [
-            {
-                "token": "token1",
-                "url": "https://matrix-as.org",
-                "id": "id_1"
-            },
-            {
-                "token": "alpha_tok",
-                "url": "https://alpha.com",
-                "id": "id_alpha"
-            },
-            {
-                "token": "beta_tok",
-                "url": "https://beta.com",
-                "id": "id_beta"
-            },
-            {
-                "token": "gamma_tok",
-                "url": "https://gamma.com",
-                "id": "id_gamma"
-            },
+            {"token": "token1", "url": "https://matrix-as.org", "id": "id_1"},
+            {"token": "alpha_tok", "url": "https://alpha.com", "id": "id_alpha"},
+            {"token": "beta_tok", "url": "https://beta.com", "id": "id_beta"},
+            {"token": "gamma_tok", "url": "https://gamma.com", "id": "id_gamma"},
         ]
         for s in self.as_list:
             yield self._add_service(s["url"], s["token"], s["id"])
@@ -157,8 +126,14 @@ def setUp(self):
         self.store = TestTransactionStore(None, hs)
 
     def _add_service(self, url, as_token, id):
-        as_yaml = dict(url=url, as_token=as_token, hs_token="something",
-                       id=id, sender_localpart="a_sender", namespaces={})
+        as_yaml = dict(
+            url=url,
+            as_token=as_token,
+            hs_token="something",
+            id=id,
+            sender_localpart="a_sender",
+            namespaces={},
+        )
         # use the token as the filename
         with open(as_token, 'w') as outfile:
             outfile.write(yaml.dump(as_yaml))
@@ -168,21 +143,21 @@ def _set_state(self, id, state, txn=None):
         return self.db_pool.runQuery(
             "INSERT INTO application_services_state(as_id, state, last_txn) "
             "VALUES(?,?,?)",
-            (id, state, txn)
+            (id, state, txn),
         )
 
     def _insert_txn(self, as_id, txn_id, events):
         return self.db_pool.runQuery(
             "INSERT INTO application_services_txns(as_id, txn_id, event_ids) "
             "VALUES(?,?,?)",
-            (as_id, txn_id, json.dumps([e.event_id for e in events]))
+            (as_id, txn_id, json.dumps([e.event_id for e in events])),
         )
 
     def _set_last_txn(self, as_id, txn_id):
         return self.db_pool.runQuery(
             "INSERT INTO application_services_state(as_id, last_txn, state) "
             "VALUES(?,?,?)",
-            (as_id, txn_id, ApplicationServiceState.UP)
+            (as_id, txn_id, ApplicationServiceState.UP),
         )
 
     @defer.inlineCallbacks
@@ -193,24 +168,16 @@ def test_get_appservice_state_none(self):
 
     @defer.inlineCallbacks
     def test_get_appservice_state_up(self):
-        yield self._set_state(
-            self.as_list[0]["id"], ApplicationServiceState.UP
-        )
+        yield self._set_state(self.as_list[0]["id"], ApplicationServiceState.UP)
         service = Mock(id=self.as_list[0]["id"])
         state = yield self.store.get_appservice_state(service)
         self.assertEquals(ApplicationServiceState.UP, state)
 
     @defer.inlineCallbacks
     def test_get_appservice_state_down(self):
-        yield self._set_state(
-            self.as_list[0]["id"], ApplicationServiceState.UP
-        )
-        yield self._set_state(
-            self.as_list[1]["id"], ApplicationServiceState.DOWN
-        )
-        yield self._set_state(
-            self.as_list[2]["id"], ApplicationServiceState.DOWN
-        )
+        yield self._set_state(self.as_list[0]["id"], ApplicationServiceState.UP)
+        yield self._set_state(self.as_list[1]["id"], ApplicationServiceState.DOWN)
+        yield self._set_state(self.as_list[2]["id"], ApplicationServiceState.DOWN)
         service = Mock(id=self.as_list[1]["id"])
         state = yield self.store.get_appservice_state(service)
         self.assertEquals(ApplicationServiceState.DOWN, state)
@@ -225,34 +192,22 @@ def test_get_appservices_by_state_none(self):
     @defer.inlineCallbacks
     def test_set_appservices_state_down(self):
         service = Mock(id=self.as_list[1]["id"])
-        yield self.store.set_appservice_state(
-            service,
-            ApplicationServiceState.DOWN
-        )
+        yield self.store.set_appservice_state(service, ApplicationServiceState.DOWN)
         rows = yield self.db_pool.runQuery(
             "SELECT as_id FROM application_services_state WHERE state=?",
-            (ApplicationServiceState.DOWN,)
+            (ApplicationServiceState.DOWN,),
         )
         self.assertEquals(service.id, rows[0][0])
 
     @defer.inlineCallbacks
     def test_set_appservices_state_multiple_up(self):
         service = Mock(id=self.as_list[1]["id"])
-        yield self.store.set_appservice_state(
-            service,
-            ApplicationServiceState.UP
-        )
-        yield self.store.set_appservice_state(
-            service,
-            ApplicationServiceState.DOWN
-        )
-        yield self.store.set_appservice_state(
-            service,
-            ApplicationServiceState.UP
-        )
+        yield self.store.set_appservice_state(service, ApplicationServiceState.UP)
+        yield self.store.set_appservice_state(service, ApplicationServiceState.DOWN)
+        yield self.store.set_appservice_state(service, ApplicationServiceState.UP)
         rows = yield self.db_pool.runQuery(
             "SELECT as_id FROM application_services_state WHERE state=?",
-            (ApplicationServiceState.UP,)
+            (ApplicationServiceState.UP,),
         )
         self.assertEquals(service.id, rows[0][0])
 
@@ -319,14 +274,13 @@ def test_complete_appservice_txn_first_txn(self):
 
         res = yield self.db_pool.runQuery(
             "SELECT last_txn FROM application_services_state WHERE as_id=?",
-            (service.id,)
+            (service.id,),
         )
         self.assertEquals(1, len(res))
         self.assertEquals(txn_id, res[0][0])
 
         res = yield self.db_pool.runQuery(
-            "SELECT * FROM application_services_txns WHERE txn_id=?",
-            (txn_id,)
+            "SELECT * FROM application_services_txns WHERE txn_id=?", (txn_id,)
         )
         self.assertEquals(0, len(res))
 
@@ -340,17 +294,15 @@ def test_complete_appservice_txn_existing_in_state_table(self):
         yield self.store.complete_appservice_txn(txn_id=txn_id, service=service)
 
         res = yield self.db_pool.runQuery(
-            "SELECT last_txn, state FROM application_services_state WHERE "
-            "as_id=?",
-            (service.id,)
+            "SELECT last_txn, state FROM application_services_state WHERE " "as_id=?",
+            (service.id,),
         )
         self.assertEquals(1, len(res))
         self.assertEquals(txn_id, res[0][0])
         self.assertEquals(ApplicationServiceState.UP, res[0][1])
 
         res = yield self.db_pool.runQuery(
-            "SELECT * FROM application_services_txns WHERE txn_id=?",
-            (txn_id,)
+            "SELECT * FROM application_services_txns WHERE txn_id=?", (txn_id,)
         )
         self.assertEquals(0, len(res))
 
@@ -382,12 +334,8 @@ def test_get_oldest_unsent_txn(self):
 
     @defer.inlineCallbacks
     def test_get_appservices_by_state_single(self):
-        yield self._set_state(
-            self.as_list[0]["id"], ApplicationServiceState.DOWN
-        )
-        yield self._set_state(
-            self.as_list[1]["id"], ApplicationServiceState.UP
-        )
+        yield self._set_state(self.as_list[0]["id"], ApplicationServiceState.DOWN)
+        yield self._set_state(self.as_list[1]["id"], ApplicationServiceState.UP)
 
         services = yield self.store.get_appservices_by_state(
             ApplicationServiceState.DOWN
@@ -397,18 +345,10 @@ def test_get_appservices_by_state_single(self):
 
     @defer.inlineCallbacks
     def test_get_appservices_by_state_multiple(self):
-        yield self._set_state(
-            self.as_list[0]["id"], ApplicationServiceState.DOWN
-        )
-        yield self._set_state(
-            self.as_list[1]["id"], ApplicationServiceState.UP
-        )
-        yield self._set_state(
-            self.as_list[2]["id"], ApplicationServiceState.DOWN
-        )
-        yield self._set_state(
-            self.as_list[3]["id"], ApplicationServiceState.UP
-        )
+        yield self._set_state(self.as_list[0]["id"], ApplicationServiceState.DOWN)
+        yield self._set_state(self.as_list[1]["id"], ApplicationServiceState.UP)
+        yield self._set_state(self.as_list[2]["id"], ApplicationServiceState.DOWN)
+        yield self._set_state(self.as_list[3]["id"], ApplicationServiceState.UP)
 
         services = yield self.store.get_appservices_by_state(
             ApplicationServiceState.DOWN
@@ -416,20 +356,17 @@ def test_get_appservices_by_state_multiple(self):
         self.assertEquals(2, len(services))
         self.assertEquals(
             set([self.as_list[2]["id"], self.as_list[0]["id"]]),
-            set([services[0].id, services[1].id])
+            set([services[0].id, services[1].id]),
         )
 
 
 # required for ApplicationServiceTransactionStoreTestCase tests
-class TestTransactionStore(ApplicationServiceTransactionStore,
-                           ApplicationServiceStore):
-
+class TestTransactionStore(ApplicationServiceTransactionStore, ApplicationServiceStore):
     def __init__(self, db_conn, hs):
         super(TestTransactionStore, self).__init__(db_conn, hs)
 
 
 class ApplicationServiceStoreConfigTestCase(unittest.TestCase):
-
     def _write_config(self, suffix, **kwargs):
         vals = {
             "id": "id" + suffix,
@@ -452,8 +389,7 @@ def test_unique_works(self):
         f2 = self._write_config(suffix="2")
 
         config = Mock(
-            app_service_config_files=[f1, f2], event_cache_size=1,
-            password_providers=[]
+            app_service_config_files=[f1, f2], event_cache_size=1, password_providers=[]
         )
         hs = yield setup_test_homeserver(
             config=config,
@@ -470,8 +406,7 @@ def test_duplicate_ids(self):
         f2 = self._write_config(id="id", suffix="2")
 
         config = Mock(
-            app_service_config_files=[f1, f2], event_cache_size=1,
-            password_providers=[]
+            app_service_config_files=[f1, f2], event_cache_size=1, password_providers=[]
         )
         hs = yield setup_test_homeserver(
             config=config,
@@ -494,8 +429,7 @@ def test_duplicate_as_tokens(self):
         f2 = self._write_config(as_token="as_token", suffix="2")
 
         config = Mock(
-            app_service_config_files=[f1, f2], event_cache_size=1,
-            password_providers=[]
+            app_service_config_files=[f1, f2], event_cache_size=1, password_providers=[]
         )
         hs = yield setup_test_homeserver(
             config=config,
diff --git a/tests/storage/test_background_update.py b/tests/storage/test_background_update.py
index ab1f31057224..b4f6baf4418a 100644
--- a/tests/storage/test_background_update.py
+++ b/tests/storage/test_background_update.py
@@ -7,7 +7,6 @@
 
 
 class BackgroundUpdateTestCase(unittest.TestCase):
-
     @defer.inlineCallbacks
     def setUp(self):
         hs = yield setup_test_homeserver()  # type: synapse.server.HomeServer
@@ -51,9 +50,7 @@ def update(progress, count):
         yield self.store.start_background_update("test_update", {"my_key": 1})
 
         self.update_handler.reset_mock()
-        result = yield self.store.do_next_background_update(
-            duration_ms * desired_count
-        )
+        result = yield self.store.do_next_background_update(duration_ms * desired_count)
         self.assertIsNotNone(result)
         self.update_handler.assert_called_once_with(
             {"my_key": 1}, self.store.DEFAULT_BACKGROUND_BATCH_SIZE
@@ -67,18 +64,12 @@ def update(progress, count):
 
         self.update_handler.side_effect = update
         self.update_handler.reset_mock()
-        result = yield self.store.do_next_background_update(
-            duration_ms * desired_count
-        )
+        result = yield self.store.do_next_background_update(duration_ms * desired_count)
         self.assertIsNotNone(result)
-        self.update_handler.assert_called_once_with(
-            {"my_key": 2}, desired_count
-        )
+        self.update_handler.assert_called_once_with({"my_key": 2}, desired_count)
 
         # third step: we don't expect to be called any more
         self.update_handler.reset_mock()
-        result = yield self.store.do_next_background_update(
-            duration_ms * desired_count
-        )
+        result = yield self.store.do_next_background_update(duration_ms * desired_count)
         self.assertIsNone(result)
         self.assertFalse(self.update_handler.called)
diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py
index 1d1234ee391e..7cb5f0e4cfcb 100644
--- a/tests/storage/test_base.py
+++ b/tests/storage/test_base.py
@@ -40,10 +40,12 @@ def setUp(self):
 
         def runInteraction(func, *args, **kwargs):
             return defer.succeed(func(self.mock_txn, *args, **kwargs))
+
         self.db_pool.runInteraction = runInteraction
 
         def runWithConnection(func, *args, **kwargs):
             return defer.succeed(func(self.mock_conn, *args, **kwargs))
+
         self.db_pool.runWithConnection = runWithConnection
 
         config = Mock()
@@ -63,8 +65,7 @@ def test_insert_1col(self):
         self.mock_txn.rowcount = 1
 
         yield self.datastore._simple_insert(
-            table="tablename",
-            values={"columname": "Value"}
+            table="tablename", values={"columname": "Value"}
         )
 
         self.mock_txn.execute.assert_called_with(
@@ -78,12 +79,11 @@ def test_insert_3cols(self):
         yield self.datastore._simple_insert(
             table="tablename",
             # Use OrderedDict() so we can assert on the SQL generated
-            values=OrderedDict([("colA", 1), ("colB", 2), ("colC", 3)])
+            values=OrderedDict([("colA", 1), ("colB", 2), ("colC", 3)]),
         )
 
         self.mock_txn.execute.assert_called_with(
-            "INSERT INTO tablename (colA, colB, colC) VALUES(?, ?, ?)",
-            (1, 2, 3,)
+            "INSERT INTO tablename (colA, colB, colC) VALUES(?, ?, ?)", (1, 2, 3)
         )
 
     @defer.inlineCallbacks
@@ -92,9 +92,7 @@ def test_select_one_1col(self):
         self.mock_txn.__iter__ = Mock(return_value=iter([("Value",)]))
 
         value = yield self.datastore._simple_select_one_onecol(
-            table="tablename",
-            keyvalues={"keycol": "TheKey"},
-            retcol="retcol"
+            table="tablename", keyvalues={"keycol": "TheKey"}, retcol="retcol"
         )
 
         self.assertEquals("Value", value)
@@ -110,13 +108,12 @@ def test_select_one_3col(self):
         ret = yield self.datastore._simple_select_one(
             table="tablename",
             keyvalues={"keycol": "TheKey"},
-            retcols=["colA", "colB", "colC"]
+            retcols=["colA", "colB", "colC"],
         )
 
         self.assertEquals({"colA": 1, "colB": 2, "colC": 3}, ret)
         self.mock_txn.execute.assert_called_with(
-            "SELECT colA, colB, colC FROM tablename WHERE keycol = ?",
-            ["TheKey"]
+            "SELECT colA, colB, colC FROM tablename WHERE keycol = ?", ["TheKey"]
         )
 
     @defer.inlineCallbacks
@@ -128,7 +125,7 @@ def test_select_one_missing(self):
             table="tablename",
             keyvalues={"keycol": "Not here"},
             retcols=["colA"],
-            allow_none=True
+            allow_none=True,
         )
 
         self.assertFalse(ret)
@@ -137,20 +134,15 @@ def test_select_one_missing(self):
     def test_select_list(self):
         self.mock_txn.rowcount = 3
         self.mock_txn.__iter__ = Mock(return_value=iter([(1,), (2,), (3,)]))
-        self.mock_txn.description = (
-            ("colA", None, None, None, None, None, None),
-        )
+        self.mock_txn.description = (("colA", None, None, None, None, None, None),)
 
         ret = yield self.datastore._simple_select_list(
-            table="tablename",
-            keyvalues={"keycol": "A set"},
-            retcols=["colA"],
+            table="tablename", keyvalues={"keycol": "A set"}, retcols=["colA"]
         )
 
         self.assertEquals([{"colA": 1}, {"colA": 2}, {"colA": 3}], ret)
         self.mock_txn.execute.assert_called_with(
-            "SELECT colA FROM tablename WHERE keycol = ?",
-            ["A set"]
+            "SELECT colA FROM tablename WHERE keycol = ?", ["A set"]
         )
 
     @defer.inlineCallbacks
@@ -160,12 +152,12 @@ def test_update_one_1col(self):
         yield self.datastore._simple_update_one(
             table="tablename",
             keyvalues={"keycol": "TheKey"},
-            updatevalues={"columnname": "New Value"}
+            updatevalues={"columnname": "New Value"},
         )
 
         self.mock_txn.execute.assert_called_with(
             "UPDATE tablename SET columnname = ? WHERE keycol = ?",
-            ["New Value", "TheKey"]
+            ["New Value", "TheKey"],
         )
 
     @defer.inlineCallbacks
@@ -175,13 +167,12 @@ def test_update_one_4cols(self):
         yield self.datastore._simple_update_one(
             table="tablename",
             keyvalues=OrderedDict([("colA", 1), ("colB", 2)]),
-            updatevalues=OrderedDict([("colC", 3), ("colD", 4)])
+            updatevalues=OrderedDict([("colC", 3), ("colD", 4)]),
         )
 
         self.mock_txn.execute.assert_called_with(
-            "UPDATE tablename SET colC = ?, colD = ? WHERE"
-            " colA = ? AND colB = ?",
-            [3, 4, 1, 2]
+            "UPDATE tablename SET colC = ?, colD = ? WHERE" " colA = ? AND colB = ?",
+            [3, 4, 1, 2],
         )
 
     @defer.inlineCallbacks
@@ -189,8 +180,7 @@ def test_delete_one(self):
         self.mock_txn.rowcount = 1
 
         yield self.datastore._simple_delete_one(
-            table="tablename",
-            keyvalues={"keycol": "Go away"},
+            table="tablename", keyvalues={"keycol": "Go away"}
         )
 
         self.mock_txn.execute.assert_called_with(
diff --git a/tests/storage/test_client_ips.py b/tests/storage/test_client_ips.py
index 7a58c6eb24e5..ea00bbe84cd4 100644
--- a/tests/storage/test_client_ips.py
+++ b/tests/storage/test_client_ips.py
@@ -37,8 +37,7 @@ def test_insert_new_client_ip(self):
         self.clock.now = 12345678
         user_id = "@user:id"
         yield self.store.insert_client_ip(
-            user_id,
-            "access_token", "ip", "user_agent", "device_id",
+            user_id, "access_token", "ip", "user_agent", "device_id"
         )
 
         result = yield self.store.get_last_client_ip_by_device(user_id, "device_id")
@@ -53,7 +52,7 @@ def test_insert_new_client_ip(self):
                 "user_agent": "user_agent",
                 "last_seen": 12345678000,
             },
-            r
+            r,
         )
 
     @defer.inlineCallbacks
@@ -62,7 +61,7 @@ def test_disabled_monthly_active_user(self):
         self.hs.config.max_mau_value = 50
         user_id = "@user:server"
         yield self.store.insert_client_ip(
-            user_id, "access_token", "ip", "user_agent", "device_id",
+            user_id, "access_token", "ip", "user_agent", "device_id"
         )
         active = yield self.store._user_last_seen_monthly_active(user_id)
         self.assertFalse(active)
@@ -78,7 +77,7 @@ def test_adding_monthly_active_user_when_full(self):
             return_value=defer.succeed(lots_of_users)
         )
         yield self.store.insert_client_ip(
-            user_id, "access_token", "ip", "user_agent", "device_id",
+            user_id, "access_token", "ip", "user_agent", "device_id"
         )
         active = yield self.store._user_last_seen_monthly_active(user_id)
         self.assertFalse(active)
@@ -92,7 +91,7 @@ def test_adding_monthly_active_user_when_space(self):
         self.assertFalse(active)
 
         yield self.store.insert_client_ip(
-            user_id, "access_token", "ip", "user_agent", "device_id",
+            user_id, "access_token", "ip", "user_agent", "device_id"
         )
         active = yield self.store._user_last_seen_monthly_active(user_id)
         self.assertTrue(active)
@@ -107,10 +106,10 @@ def test_updating_monthly_active_user_when_space(self):
         self.assertFalse(active)
 
         yield self.store.insert_client_ip(
-            user_id, "access_token", "ip", "user_agent", "device_id",
+            user_id, "access_token", "ip", "user_agent", "device_id"
         )
         yield self.store.insert_client_ip(
-            user_id, "access_token", "ip", "user_agent", "device_id",
+            user_id, "access_token", "ip", "user_agent", "device_id"
         )
         active = yield self.store._user_last_seen_monthly_active(user_id)
         self.assertTrue(active)
diff --git a/tests/storage/test_devices.py b/tests/storage/test_devices.py
index a54cc6bc32b2..63bc42d9e096 100644
--- a/tests/storage/test_devices.py
+++ b/tests/storage/test_devices.py
@@ -34,62 +34,58 @@ def setUp(self):
 
     @defer.inlineCallbacks
     def test_store_new_device(self):
-        yield self.store.store_device(
-            "user_id", "device_id", "display_name"
-        )
+        yield self.store.store_device("user_id", "device_id", "display_name")
 
         res = yield self.store.get_device("user_id", "device_id")
-        self.assertDictContainsSubset({
-            "user_id": "user_id",
-            "device_id": "device_id",
-            "display_name": "display_name",
-        }, res)
+        self.assertDictContainsSubset(
+            {
+                "user_id": "user_id",
+                "device_id": "device_id",
+                "display_name": "display_name",
+            },
+            res,
+        )
 
     @defer.inlineCallbacks
     def test_get_devices_by_user(self):
-        yield self.store.store_device(
-            "user_id", "device1", "display_name 1"
-        )
-        yield self.store.store_device(
-            "user_id", "device2", "display_name 2"
-        )
-        yield self.store.store_device(
-            "user_id2", "device3", "display_name 3"
-        )
+        yield self.store.store_device("user_id", "device1", "display_name 1")
+        yield self.store.store_device("user_id", "device2", "display_name 2")
+        yield self.store.store_device("user_id2", "device3", "display_name 3")
 
         res = yield self.store.get_devices_by_user("user_id")
         self.assertEqual(2, len(res.keys()))
-        self.assertDictContainsSubset({
-            "user_id": "user_id",
-            "device_id": "device1",
-            "display_name": "display_name 1",
-        }, res["device1"])
-        self.assertDictContainsSubset({
-            "user_id": "user_id",
-            "device_id": "device2",
-            "display_name": "display_name 2",
-        }, res["device2"])
+        self.assertDictContainsSubset(
+            {
+                "user_id": "user_id",
+                "device_id": "device1",
+                "display_name": "display_name 1",
+            },
+            res["device1"],
+        )
+        self.assertDictContainsSubset(
+            {
+                "user_id": "user_id",
+                "device_id": "device2",
+                "display_name": "display_name 2",
+            },
+            res["device2"],
+        )
 
     @defer.inlineCallbacks
     def test_update_device(self):
-        yield self.store.store_device(
-            "user_id", "device_id", "display_name 1"
-        )
+        yield self.store.store_device("user_id", "device_id", "display_name 1")
 
         res = yield self.store.get_device("user_id", "device_id")
         self.assertEqual("display_name 1", res["display_name"])
 
         # do a no-op first
-        yield self.store.update_device(
-            "user_id", "device_id",
-        )
+        yield self.store.update_device("user_id", "device_id")
         res = yield self.store.get_device("user_id", "device_id")
         self.assertEqual("display_name 1", res["display_name"])
 
         # do the update
         yield self.store.update_device(
-            "user_id", "device_id",
-            new_display_name="display_name 2",
+            "user_id", "device_id", new_display_name="display_name 2"
         )
 
         # check it worked
@@ -100,7 +96,6 @@ def test_update_device(self):
     def test_update_unknown_device(self):
         with self.assertRaises(synapse.api.errors.StoreError) as cm:
             yield self.store.update_device(
-                "user_id", "unknown_device_id",
-                new_display_name="display_name 2",
+                "user_id", "unknown_device_id", new_display_name="display_name 2"
             )
         self.assertEqual(404, cm.exception.code)
diff --git a/tests/storage/test_directory.py b/tests/storage/test_directory.py
index 129ebaf34389..9a8ba2fcfe29 100644
--- a/tests/storage/test_directory.py
+++ b/tests/storage/test_directory.py
@@ -24,7 +24,6 @@
 
 
 class DirectoryStoreTestCase(unittest.TestCase):
-
     @defer.inlineCallbacks
     def setUp(self):
         hs = yield setup_test_homeserver()
@@ -37,38 +36,29 @@ def setUp(self):
     @defer.inlineCallbacks
     def test_room_to_alias(self):
         yield self.store.create_room_alias_association(
-            room_alias=self.alias,
-            room_id=self.room.to_string(),
-            servers=["test"],
+            room_alias=self.alias, room_id=self.room.to_string(), servers=["test"]
         )
 
         self.assertEquals(
             ["#my-room:test"],
-            (yield self.store.get_aliases_for_room(self.room.to_string()))
+            (yield self.store.get_aliases_for_room(self.room.to_string())),
         )
 
     @defer.inlineCallbacks
     def test_alias_to_room(self):
         yield self.store.create_room_alias_association(
-            room_alias=self.alias,
-            room_id=self.room.to_string(),
-            servers=["test"],
+            room_alias=self.alias, room_id=self.room.to_string(), servers=["test"]
         )
 
         self.assertObjectHasAttributes(
-            {
-                "room_id": self.room.to_string(),
-                "servers": ["test"],
-            },
-            (yield self.store.get_association_from_room_alias(self.alias))
+            {"room_id": self.room.to_string(), "servers": ["test"]},
+            (yield self.store.get_association_from_room_alias(self.alias)),
         )
 
     @defer.inlineCallbacks
     def test_delete_alias(self):
         yield self.store.create_room_alias_association(
-            room_alias=self.alias,
-            room_id=self.room.to_string(),
-            servers=["test"],
+            room_alias=self.alias, room_id=self.room.to_string(), servers=["test"]
         )
 
         room_id = yield self.store.delete_room_alias(self.alias)
diff --git a/tests/storage/test_end_to_end_keys.py b/tests/storage/test_end_to_end_keys.py
index 84ce492a2cf5..d45c775c2d66 100644
--- a/tests/storage/test_end_to_end_keys.py
+++ b/tests/storage/test_end_to_end_keys.py
@@ -35,70 +35,49 @@ def test_key_without_device_name(self):
         now = 1470174257070
         json = {"key": "value"}
 
-        yield self.store.store_device(
-            "user", "device", None
-        )
+        yield self.store.store_device("user", "device", None)
 
-        yield self.store.set_e2e_device_keys(
-            "user", "device", now, json)
+        yield self.store.set_e2e_device_keys("user", "device", now, json)
 
         res = yield self.store.get_e2e_device_keys((("user", "device"),))
         self.assertIn("user", res)
         self.assertIn("device", res["user"])
         dev = res["user"]["device"]
-        self.assertDictContainsSubset({
-            "keys": json,
-            "device_display_name": None,
-        }, dev)
+        self.assertDictContainsSubset({"keys": json, "device_display_name": None}, dev)
 
     @defer.inlineCallbacks
     def test_get_key_with_device_name(self):
         now = 1470174257070
         json = {"key": "value"}
 
-        yield self.store.set_e2e_device_keys(
-            "user", "device", now, json)
-        yield self.store.store_device(
-            "user", "device", "display_name"
-        )
+        yield self.store.set_e2e_device_keys("user", "device", now, json)
+        yield self.store.store_device("user", "device", "display_name")
 
         res = yield self.store.get_e2e_device_keys((("user", "device"),))
         self.assertIn("user", res)
         self.assertIn("device", res["user"])
         dev = res["user"]["device"]
-        self.assertDictContainsSubset({
-            "keys": json,
-            "device_display_name": "display_name",
-        }, dev)
+        self.assertDictContainsSubset(
+            {"keys": json, "device_display_name": "display_name"}, dev
+        )
 
     @defer.inlineCallbacks
     def test_multiple_devices(self):
         now = 1470174257070
 
-        yield self.store.store_device(
-            "user1", "device1", None
-        )
-        yield self.store.store_device(
-            "user1", "device2", None
-        )
-        yield self.store.store_device(
-            "user2", "device1", None
-        )
-        yield self.store.store_device(
-            "user2", "device2", None
-        )
+        yield self.store.store_device("user1", "device1", None)
+        yield self.store.store_device("user1", "device2", None)
+        yield self.store.store_device("user2", "device1", None)
+        yield self.store.store_device("user2", "device2", None)
 
-        yield self.store.set_e2e_device_keys(
-            "user1", "device1", now, 'json11')
-        yield self.store.set_e2e_device_keys(
-            "user1", "device2", now, 'json12')
-        yield self.store.set_e2e_device_keys(
-            "user2", "device1", now, 'json21')
-        yield self.store.set_e2e_device_keys(
-            "user2", "device2", now, 'json22')
-
-        res = yield self.store.get_e2e_device_keys((("user1", "device1"),
-                                                    ("user2", "device2")))
+        yield self.store.set_e2e_device_keys("user1", "device1", now, 'json11')
+        yield self.store.set_e2e_device_keys("user1", "device2", now, 'json12')
+        yield self.store.set_e2e_device_keys("user2", "device1", now, 'json21')
+        yield self.store.set_e2e_device_keys("user2", "device2", now, 'json22')
+
+        res = yield self.store.get_e2e_device_keys(
+            (("user1", "device1"), ("user2", "device2"))
+        )
         self.assertIn("user1", res)
         self.assertIn("device1", res["user1"])
         self.assertNotIn("device2", res["user1"])
diff --git a/tests/storage/test_event_federation.py b/tests/storage/test_event_federation.py
index 69412c5aadd8..66eb11958107 100644
--- a/tests/storage/test_event_federation.py
+++ b/tests/storage/test_event_federation.py
@@ -33,23 +33,32 @@ def test_get_prev_events_for_room(self):
         def insert_event(txn, i):
             event_id = '$event_%i:local' % i
 
-            txn.execute((
-                "INSERT INTO events ("
-                "   room_id, event_id, type, depth, topological_ordering,"
-                "   content, processed, outlier) "
-                "VALUES (?, ?, 'm.test', ?, ?, 'test', ?, ?)"
-            ), (room_id, event_id, i, i, True, False))
+            txn.execute(
+                (
+                    "INSERT INTO events ("
+                    "   room_id, event_id, type, depth, topological_ordering,"
+                    "   content, processed, outlier) "
+                    "VALUES (?, ?, 'm.test', ?, ?, 'test', ?, ?)"
+                ),
+                (room_id, event_id, i, i, True, False),
+            )
 
-            txn.execute((
-                'INSERT INTO event_forward_extremities (room_id, event_id) '
-                'VALUES (?, ?)'
-            ), (room_id, event_id))
+            txn.execute(
+                (
+                    'INSERT INTO event_forward_extremities (room_id, event_id) '
+                    'VALUES (?, ?)'
+                ),
+                (room_id, event_id),
+            )
 
-            txn.execute((
-                'INSERT INTO event_reference_hashes '
-                '(event_id, algorithm, hash) '
-                "VALUES (?, 'sha256', ?)"
-            ), (event_id, b'ffff'))
+            txn.execute(
+                (
+                    'INSERT INTO event_reference_hashes '
+                    '(event_id, algorithm, hash) '
+                    "VALUES (?, 'sha256', ?)"
+                ),
+                (event_id, b'ffff'),
+            )
 
         for i in range(0, 11):
             yield self.store.runInteraction("insert", insert_event, i)
diff --git a/tests/storage/test_event_push_actions.py b/tests/storage/test_event_push_actions.py
index 8430fc7ba6a9..5e87b4530df8 100644
--- a/tests/storage/test_event_push_actions.py
+++ b/tests/storage/test_event_push_actions.py
@@ -24,12 +24,13 @@
 
 PlAIN_NOTIF = ["notify", {"set_tweak": "highlight", "value": False}]
 HIGHLIGHT = [
-    "notify", {"set_tweak": "sound", "value": "default"}, {"set_tweak": "highlight"}
+    "notify",
+    {"set_tweak": "sound", "value": "default"},
+    {"set_tweak": "highlight"},
 ]
 
 
 class EventPushActionsStoreTestCase(tests.unittest.TestCase):
-
     @defer.inlineCallbacks
     def setUp(self):
         hs = yield tests.utils.setup_test_homeserver()
@@ -55,12 +56,11 @@ def test_count_aggregation(self):
         @defer.inlineCallbacks
         def _assert_counts(noitf_count, highlight_count):
             counts = yield self.store.runInteraction(
-                "", self.store._get_unread_counts_by_pos_txn,
-                room_id, user_id, 0
+                "", self.store._get_unread_counts_by_pos_txn, room_id, user_id, 0
             )
             self.assertEquals(
                 counts,
-                {"notify_count": noitf_count, "highlight_count": highlight_count}
+                {"notify_count": noitf_count, "highlight_count": highlight_count},
             )
 
         @defer.inlineCallbacks
@@ -72,11 +72,13 @@ def _inject_actions(stream, action):
             event.depth = stream
 
             yield self.store.add_push_actions_to_staging(
-                event.event_id, {user_id: action},
+                event.event_id, {user_id: action}
             )
             yield self.store.runInteraction(
-                "", self.store._set_push_actions_for_event_and_users_txn,
-                [(event, None)], [(event, None)],
+                "",
+                self.store._set_push_actions_for_event_and_users_txn,
+                [(event, None)],
+                [(event, None)],
             )
 
         def _rotate(stream):
@@ -86,8 +88,11 @@ def _rotate(stream):
 
         def _mark_read(stream, depth):
             return self.store.runInteraction(
-                "", self.store._remove_old_push_actions_before_txn,
-                room_id, user_id, stream
+                "",
+                self.store._remove_old_push_actions_before_txn,
+                room_id,
+                user_id,
+                stream,
             )
 
         yield _assert_counts(0, 0)
@@ -112,9 +117,7 @@ def _mark_read(stream, depth):
         yield _rotate(7)
 
         yield self.store._simple_delete(
-            table="event_push_actions",
-            keyvalues={"1": 1},
-            desc="",
+            table="event_push_actions", keyvalues={"1": 1}, desc=""
         )
 
         yield _assert_counts(1, 0)
@@ -132,18 +135,21 @@ def _mark_read(stream, depth):
     @defer.inlineCallbacks
     def test_find_first_stream_ordering_after_ts(self):
         def add_event(so, ts):
-            return self.store._simple_insert("events", {
-                "stream_ordering": so,
-                "received_ts": ts,
-                "event_id": "event%i" % so,
-                "type": "",
-                "room_id": "",
-                "content": "",
-                "processed": True,
-                "outlier": False,
-                "topological_ordering": 0,
-                "depth": 0,
-            })
+            return self.store._simple_insert(
+                "events",
+                {
+                    "stream_ordering": so,
+                    "received_ts": ts,
+                    "event_id": "event%i" % so,
+                    "type": "",
+                    "room_id": "",
+                    "content": "",
+                    "processed": True,
+                    "outlier": False,
+                    "topological_ordering": 0,
+                    "depth": 0,
+                },
+            )
 
         # start with the base case where there are no events in the table
         r = yield self.store.find_first_stream_ordering_after_ts(11)
@@ -160,31 +166,27 @@ def add_event(so, ts):
 
         # add a bunch of dummy events to the events table
         for (stream_ordering, ts) in (
-                (3, 110),
-                (4, 120),
-                (5, 120),
-                (10, 130),
-                (20, 140),
+            (3, 110),
+            (4, 120),
+            (5, 120),
+            (10, 130),
+            (20, 140),
         ):
             yield add_event(stream_ordering, ts)
 
         r = yield self.store.find_first_stream_ordering_after_ts(110)
-        self.assertEqual(r, 3,
-                         "First event after 110ms should be 3, was %i" % r)
+        self.assertEqual(r, 3, "First event after 110ms should be 3, was %i" % r)
 
         # 4 and 5 are both after 120: we want 4 rather than 5
         r = yield self.store.find_first_stream_ordering_after_ts(120)
-        self.assertEqual(r, 4,
-                         "First event after 120ms should be 4, was %i" % r)
+        self.assertEqual(r, 4, "First event after 120ms should be 4, was %i" % r)
 
         r = yield self.store.find_first_stream_ordering_after_ts(129)
-        self.assertEqual(r, 10,
-                         "First event after 129ms should be 10, was %i" % r)
+        self.assertEqual(r, 10, "First event after 129ms should be 10, was %i" % r)
 
         # check we can get the last event
         r = yield self.store.find_first_stream_ordering_after_ts(140)
-        self.assertEqual(r, 20,
-                         "First event after 14ms should be 20, was %i" % r)
+        self.assertEqual(r, 20, "First event after 14ms should be 20, was %i" % r)
 
         # off the end
         r = yield self.store.find_first_stream_ordering_after_ts(160)
diff --git a/tests/storage/test_keys.py b/tests/storage/test_keys.py
index 3a3d00278232..ad0a55b324b0 100644
--- a/tests/storage/test_keys.py
+++ b/tests/storage/test_keys.py
@@ -39,15 +39,12 @@ def test_get_server_verify_keys(self):
         key2 = signedjson.key.decode_verify_key_base64(
             "ed25519", "key2", "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw"
         )
-        yield self.store.store_server_verify_key(
-            "server1", "from_server", 0, key1
-        )
-        yield self.store.store_server_verify_key(
-            "server1", "from_server", 0, key2
-        )
+        yield self.store.store_server_verify_key("server1", "from_server", 0, key1)
+        yield self.store.store_server_verify_key("server1", "from_server", 0, key2)
 
         res = yield self.store.get_server_verify_keys(
-            "server1", ["ed25519:key1", "ed25519:key2", "ed25519:key3"])
+            "server1", ["ed25519:key1", "ed25519:key2", "ed25519:key3"]
+        )
 
         self.assertEqual(len(res.keys()), 2)
         self.assertEqual(res["ed25519:key1"].version, "key1")
diff --git a/tests/storage/test_monthly_active_users.py b/tests/storage/test_monthly_active_users.py
index cbd480cd4215..22b1072d9f57 100644
--- a/tests/storage/test_monthly_active_users.py
+++ b/tests/storage/test_monthly_active_users.py
@@ -40,19 +40,13 @@ def test_initialise_reserved_users(self):
         user2_email = "user2@matrix.org"
         threepids = [
             {'medium': 'email', 'address': user1_email},
-            {'medium': 'email', 'address': user2_email}
+            {'medium': 'email', 'address': user2_email},
         ]
         user_num = len(threepids)
 
-        yield self.store.register(
-            user_id=user1,
-            token="123",
-            password_hash=None)
+        yield self.store.register(user_id=user1, token="123", password_hash=None)
 
-        yield self.store.register(
-            user_id=user2,
-            token="456",
-            password_hash=None)
+        yield self.store.register(user_id=user2, token="456", password_hash=None)
 
         now = int(self.hs.get_clock().time_msec())
         yield self.store.user_add_threepid(user1, "email", user1_email, now, now)
diff --git a/tests/storage/test_presence.py b/tests/storage/test_presence.py
index 3276b3950469..12c540dfab40 100644
--- a/tests/storage/test_presence.py
+++ b/tests/storage/test_presence.py
@@ -24,7 +24,6 @@
 
 
 class PresenceStoreTestCase(unittest.TestCase):
-
     @defer.inlineCallbacks
     def setUp(self):
         hs = yield setup_test_homeserver(clock=MockClock())
@@ -38,16 +37,19 @@ def setUp(self):
     def test_presence_list(self):
         self.assertEquals(
             [],
-            (yield self.store.get_presence_list(
-                observer_localpart=self.u_apple.localpart,
-            ))
+            (
+                yield self.store.get_presence_list(
+                    observer_localpart=self.u_apple.localpart
+                )
+            ),
         )
         self.assertEquals(
             [],
-            (yield self.store.get_presence_list(
-                observer_localpart=self.u_apple.localpart,
-                accepted=True,
-            ))
+            (
+                yield self.store.get_presence_list(
+                    observer_localpart=self.u_apple.localpart, accepted=True
+                )
+            ),
         )
 
         yield self.store.add_presence_list_pending(
@@ -57,16 +59,19 @@ def test_presence_list(self):
 
         self.assertEquals(
             [{"observed_user_id": "@banana:test", "accepted": 0}],
-            (yield self.store.get_presence_list(
-                observer_localpart=self.u_apple.localpart,
-            ))
+            (
+                yield self.store.get_presence_list(
+                    observer_localpart=self.u_apple.localpart
+                )
+            ),
         )
         self.assertEquals(
             [],
-            (yield self.store.get_presence_list(
-                observer_localpart=self.u_apple.localpart,
-                accepted=True,
-            ))
+            (
+                yield self.store.get_presence_list(
+                    observer_localpart=self.u_apple.localpart, accepted=True
+                )
+            ),
         )
 
         yield self.store.set_presence_list_accepted(
@@ -76,16 +81,19 @@ def test_presence_list(self):
 
         self.assertEquals(
             [{"observed_user_id": "@banana:test", "accepted": 1}],
-            (yield self.store.get_presence_list(
-                observer_localpart=self.u_apple.localpart,
-            ))
+            (
+                yield self.store.get_presence_list(
+                    observer_localpart=self.u_apple.localpart
+                )
+            ),
         )
         self.assertEquals(
             [{"observed_user_id": "@banana:test", "accepted": 1}],
-            (yield self.store.get_presence_list(
-                observer_localpart=self.u_apple.localpart,
-                accepted=True,
-            ))
+            (
+                yield self.store.get_presence_list(
+                    observer_localpart=self.u_apple.localpart, accepted=True
+                )
+            ),
         )
 
         yield self.store.del_presence_list(
@@ -95,14 +103,17 @@ def test_presence_list(self):
 
         self.assertEquals(
             [],
-            (yield self.store.get_presence_list(
-                observer_localpart=self.u_apple.localpart,
-            ))
+            (
+                yield self.store.get_presence_list(
+                    observer_localpart=self.u_apple.localpart
+                )
+            ),
         )
         self.assertEquals(
             [],
-            (yield self.store.get_presence_list(
-                observer_localpart=self.u_apple.localpart,
-                accepted=True,
-            ))
+            (
+                yield self.store.get_presence_list(
+                    observer_localpart=self.u_apple.localpart, accepted=True
+                )
+            ),
         )
diff --git a/tests/storage/test_profile.py b/tests/storage/test_profile.py
index 2c95e5e95a81..5acbc8be0c57 100644
--- a/tests/storage/test_profile.py
+++ b/tests/storage/test_profile.py
@@ -24,7 +24,6 @@
 
 
 class ProfileStoreTestCase(unittest.TestCase):
-
     @defer.inlineCallbacks
     def setUp(self):
         hs = yield setup_test_homeserver()
@@ -35,24 +34,17 @@ def setUp(self):
 
     @defer.inlineCallbacks
     def test_displayname(self):
-        yield self.store.create_profile(
-            self.u_frank.localpart
-        )
+        yield self.store.create_profile(self.u_frank.localpart)
 
-        yield self.store.set_profile_displayname(
-            self.u_frank.localpart, "Frank"
-        )
+        yield self.store.set_profile_displayname(self.u_frank.localpart, "Frank")
 
         self.assertEquals(
-            "Frank",
-            (yield self.store.get_profile_displayname(self.u_frank.localpart))
+            "Frank", (yield self.store.get_profile_displayname(self.u_frank.localpart))
         )
 
     @defer.inlineCallbacks
     def test_avatar_url(self):
-        yield self.store.create_profile(
-            self.u_frank.localpart
-        )
+        yield self.store.create_profile(self.u_frank.localpart)
 
         yield self.store.set_profile_avatar_url(
             self.u_frank.localpart, "http://my.site/here"
@@ -60,5 +52,5 @@ def test_avatar_url(self):
 
         self.assertEquals(
             "http://my.site/here",
-            (yield self.store.get_profile_avatar_url(self.u_frank.localpart))
+            (yield self.store.get_profile_avatar_url(self.u_frank.localpart)),
         )
diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py
index 475ec900c4a2..85ce61e841cf 100644
--- a/tests/storage/test_redaction.py
+++ b/tests/storage/test_redaction.py
@@ -26,12 +26,10 @@
 
 
 class RedactionTestCase(unittest.TestCase):
-
     @defer.inlineCallbacks
     def setUp(self):
         hs = yield setup_test_homeserver(
-            resource_for_federation=Mock(),
-            http_client=None,
+            resource_for_federation=Mock(), http_client=None
         )
 
         self.store = hs.get_datastore()
@@ -46,17 +44,20 @@ def setUp(self):
         self.depth = 1
 
     @defer.inlineCallbacks
-    def inject_room_member(self, room, user, membership, replaces_state=None,
-                           extra_content={}):
+    def inject_room_member(
+        self, room, user, membership, replaces_state=None, extra_content={}
+    ):
         content = {"membership": membership}
         content.update(extra_content)
-        builder = self.event_builder_factory.new({
-            "type": EventTypes.Member,
-            "sender": user.to_string(),
-            "state_key": user.to_string(),
-            "room_id": room.to_string(),
-            "content": content,
-        })
+        builder = self.event_builder_factory.new(
+            {
+                "type": EventTypes.Member,
+                "sender": user.to_string(),
+                "state_key": user.to_string(),
+                "room_id": room.to_string(),
+                "content": content,
+            }
+        )
 
         event, context = yield self.event_creation_handler.create_new_client_event(
             builder
@@ -70,13 +71,15 @@ def inject_room_member(self, room, user, membership, replaces_state=None,
     def inject_message(self, room, user, body):
         self.depth += 1
 
-        builder = self.event_builder_factory.new({
-            "type": EventTypes.Message,
-            "sender": user.to_string(),
-            "state_key": user.to_string(),
-            "room_id": room.to_string(),
-            "content": {"body": body, "msgtype": u"message"},
-        })
+        builder = self.event_builder_factory.new(
+            {
+                "type": EventTypes.Message,
+                "sender": user.to_string(),
+                "state_key": user.to_string(),
+                "room_id": room.to_string(),
+                "content": {"body": body, "msgtype": u"message"},
+            }
+        )
 
         event, context = yield self.event_creation_handler.create_new_client_event(
             builder
@@ -88,14 +91,16 @@ def inject_message(self, room, user, body):
 
     @defer.inlineCallbacks
     def inject_redaction(self, room, event_id, user, reason):
-        builder = self.event_builder_factory.new({
-            "type": EventTypes.Redaction,
-            "sender": user.to_string(),
-            "state_key": user.to_string(),
-            "room_id": room.to_string(),
-            "content": {"reason": reason},
-            "redacts": event_id,
-        })
+        builder = self.event_builder_factory.new(
+            {
+                "type": EventTypes.Redaction,
+                "sender": user.to_string(),
+                "state_key": user.to_string(),
+                "room_id": room.to_string(),
+                "content": {"reason": reason},
+                "redacts": event_id,
+            }
+        )
 
         event, context = yield self.event_creation_handler.create_new_client_event(
             builder
@@ -105,9 +110,7 @@ def inject_redaction(self, room, event_id, user, reason):
 
     @defer.inlineCallbacks
     def test_redact(self):
-        yield self.inject_room_member(
-            self.room1, self.u_alice, Membership.JOIN
-        )
+        yield self.inject_room_member(self.room1, self.u_alice, Membership.JOIN)
 
         msg_event = yield self.inject_message(self.room1, self.u_alice, u"t")
 
@@ -157,13 +160,10 @@ def test_redact(self):
 
     @defer.inlineCallbacks
     def test_redact_join(self):
-        yield self.inject_room_member(
-            self.room1, self.u_alice, Membership.JOIN
-        )
+        yield self.inject_room_member(self.room1, self.u_alice, Membership.JOIN)
 
         msg_event = yield self.inject_room_member(
-            self.room1, self.u_bob, Membership.JOIN,
-            extra_content={"blue": "red"},
+            self.room1, self.u_bob, Membership.JOIN, extra_content={"blue": "red"}
         )
 
         event = yield self.store.get_event(msg_event.event_id)
diff --git a/tests/storage/test_registration.py b/tests/storage/test_registration.py
index 7821ea3fa306..bd96896bb35e 100644
--- a/tests/storage/test_registration.py
+++ b/tests/storage/test_registration.py
@@ -21,7 +21,6 @@
 
 
 class RegistrationStoreTestCase(unittest.TestCase):
-
     @defer.inlineCallbacks
     def setUp(self):
         hs = yield setup_test_homeserver()
@@ -30,10 +29,7 @@ def setUp(self):
         self.store = hs.get_datastore()
 
         self.user_id = "@my-user:test"
-        self.tokens = [
-            "AbCdEfGhIjKlMnOpQrStUvWxYz",
-            "BcDeFgHiJkLmNoPqRsTuVwXyZa"
-        ]
+        self.tokens = ["AbCdEfGhIjKlMnOpQrStUvWxYz", "BcDeFgHiJkLmNoPqRsTuVwXyZa"]
         self.pwhash = "{xx1}123456789"
         self.device_id = "akgjhdjklgshg"
 
@@ -51,34 +47,26 @@ def test_register(self):
                 "consent_server_notice_sent": None,
                 "appservice_id": None,
             },
-            (yield self.store.get_user_by_id(self.user_id))
+            (yield self.store.get_user_by_id(self.user_id)),
         )
 
         result = yield self.store.get_user_by_access_token(self.tokens[0])
 
-        self.assertDictContainsSubset(
-            {
-                "name": self.user_id,
-            },
-            result
-        )
+        self.assertDictContainsSubset({"name": self.user_id}, result)
 
         self.assertTrue("token_id" in result)
 
     @defer.inlineCallbacks
     def test_add_tokens(self):
         yield self.store.register(self.user_id, self.tokens[0], self.pwhash)
-        yield self.store.add_access_token_to_user(self.user_id, self.tokens[1],
-                                                  self.device_id)
+        yield self.store.add_access_token_to_user(
+            self.user_id, self.tokens[1], self.device_id
+        )
 
         result = yield self.store.get_user_by_access_token(self.tokens[1])
 
         self.assertDictContainsSubset(
-            {
-                "name": self.user_id,
-                "device_id": self.device_id,
-            },
-            result
+            {"name": self.user_id, "device_id": self.device_id}, result
         )
 
         self.assertTrue("token_id" in result)
@@ -87,12 +75,13 @@ def test_add_tokens(self):
     def test_user_delete_access_tokens(self):
         # add some tokens
         yield self.store.register(self.user_id, self.tokens[0], self.pwhash)
-        yield self.store.add_access_token_to_user(self.user_id, self.tokens[1],
-                                                  self.device_id)
+        yield self.store.add_access_token_to_user(
+            self.user_id, self.tokens[1], self.device_id
+        )
 
         # now delete some
         yield self.store.user_delete_access_tokens(
-            self.user_id, device_id=self.device_id,
+            self.user_id, device_id=self.device_id
         )
 
         # check they were deleted
@@ -107,8 +96,7 @@ def test_user_delete_access_tokens(self):
         yield self.store.user_delete_access_tokens(self.user_id)
 
         user = yield self.store.get_user_by_access_token(self.tokens[0])
-        self.assertIsNone(user,
-                          "access token was not deleted without device_id")
+        self.assertIsNone(user, "access token was not deleted without device_id")
 
 
 class TokenGenerator:
@@ -117,4 +105,4 @@ def __init__(self):
 
     def generate(self, user_id):
         self._last_issued_token += 1
-        return u"%s-%d" % (user_id, self._last_issued_token,)
+        return u"%s-%d" % (user_id, self._last_issued_token)
diff --git a/tests/storage/test_room.py b/tests/storage/test_room.py
index ae8ae94b6d63..84d49b55c1cb 100644
--- a/tests/storage/test_room.py
+++ b/tests/storage/test_room.py
@@ -24,7 +24,6 @@
 
 
 class RoomStoreTestCase(unittest.TestCase):
-
     @defer.inlineCallbacks
     def setUp(self):
         hs = yield setup_test_homeserver()
@@ -40,7 +39,7 @@ def setUp(self):
         yield self.store.store_room(
             self.room.to_string(),
             room_creator_user_id=self.u_creator.to_string(),
-            is_public=True
+            is_public=True,
         )
 
     @defer.inlineCallbacks
@@ -49,14 +48,13 @@ def test_get_room(self):
             {
                 "room_id": self.room.to_string(),
                 "creator": self.u_creator.to_string(),
-                "is_public": True
+                "is_public": True,
             },
-            (yield self.store.get_room(self.room.to_string()))
+            (yield self.store.get_room(self.room.to_string())),
         )
 
 
 class RoomEventsStoreTestCase(unittest.TestCase):
-
     @defer.inlineCallbacks
     def setUp(self):
         hs = setup_test_homeserver()
@@ -69,18 +67,13 @@ def setUp(self):
         self.room = RoomID.from_string("!abcde:test")
 
         yield self.store.store_room(
-            self.room.to_string(),
-            room_creator_user_id="@creator:text",
-            is_public=True
+            self.room.to_string(), room_creator_user_id="@creator:text", is_public=True
         )
 
     @defer.inlineCallbacks
     def inject_room_event(self, **kwargs):
         yield self.store.persist_event(
-            self.event_factory.create_event(
-                room_id=self.room.to_string(),
-                **kwargs
-            )
+            self.event_factory.create_event(room_id=self.room.to_string(), **kwargs)
         )
 
     @defer.inlineCallbacks
@@ -88,22 +81,15 @@ def STALE_test_room_name(self):
         name = u"A-Room-Name"
 
         yield self.inject_room_event(
-            etype=EventTypes.Name,
-            name=name,
-            content={"name": name},
-            depth=1,
+            etype=EventTypes.Name, name=name, content={"name": name}, depth=1
         )
 
-        state = yield self.store.get_current_state(
-            room_id=self.room.to_string()
-        )
+        state = yield self.store.get_current_state(room_id=self.room.to_string())
 
         self.assertEquals(1, len(state))
         self.assertObjectHasAttributes(
-            {"type": "m.room.name",
-             "room_id": self.room.to_string(),
-             "name": name},
-            state[0]
+            {"type": "m.room.name", "room_id": self.room.to_string(), "name": name},
+            state[0],
         )
 
     @defer.inlineCallbacks
@@ -111,22 +97,15 @@ def STALE_test_room_topic(self):
         topic = u"A place for things"
 
         yield self.inject_room_event(
-            etype=EventTypes.Topic,
-            topic=topic,
-            content={"topic": topic},
-            depth=1,
+            etype=EventTypes.Topic, topic=topic, content={"topic": topic}, depth=1
         )
 
-        state = yield self.store.get_current_state(
-            room_id=self.room.to_string()
-        )
+        state = yield self.store.get_current_state(room_id=self.room.to_string())
 
         self.assertEquals(1, len(state))
         self.assertObjectHasAttributes(
-            {"type": "m.room.topic",
-             "room_id": self.room.to_string(),
-             "topic": topic},
-            state[0]
+            {"type": "m.room.topic", "room_id": self.room.to_string(), "topic": topic},
+            state[0],
         )
 
     # Not testing the various 'level' methods for now because there's lots
diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py
index c5fd54f67e88..0d9908926a94 100644
--- a/tests/storage/test_roommember.py
+++ b/tests/storage/test_roommember.py
@@ -26,12 +26,10 @@
 
 
 class RoomMemberStoreTestCase(unittest.TestCase):
-
     @defer.inlineCallbacks
     def setUp(self):
         hs = yield setup_test_homeserver(
-            resource_for_federation=Mock(),
-            http_client=None,
+            resource_for_federation=Mock(), http_client=None
         )
         # We can't test the RoomMemberStore on its own without the other event
         # storage logic
@@ -49,13 +47,15 @@ def setUp(self):
 
     @defer.inlineCallbacks
     def inject_room_member(self, room, user, membership, replaces_state=None):
-        builder = self.event_builder_factory.new({
-            "type": EventTypes.Member,
-            "sender": user.to_string(),
-            "state_key": user.to_string(),
-            "room_id": room.to_string(),
-            "content": {"membership": membership},
-        })
+        builder = self.event_builder_factory.new(
+            {
+                "type": EventTypes.Member,
+                "sender": user.to_string(),
+                "state_key": user.to_string(),
+                "room_id": room.to_string(),
+                "content": {"membership": membership},
+            }
+        )
 
         event, context = yield self.event_creation_handler.create_new_client_event(
             builder
@@ -71,9 +71,12 @@ def test_one_member(self):
 
         self.assertEquals(
             [self.room.to_string()],
-            [m.room_id for m in (
-                yield self.store.get_rooms_for_user_where_membership_is(
-                    self.u_alice.to_string(), [Membership.JOIN]
+            [
+                m.room_id
+                for m in (
+                    yield self.store.get_rooms_for_user_where_membership_is(
+                        self.u_alice.to_string(), [Membership.JOIN]
+                    )
                 )
-            )]
+            ],
         )
diff --git a/tests/storage/test_state.py b/tests/storage/test_state.py
index f7871cd42611..ed5b41644a5c 100644
--- a/tests/storage/test_state.py
+++ b/tests/storage/test_state.py
@@ -45,20 +45,20 @@ def setUp(self):
         self.room = RoomID.from_string("!abc123:test")
 
         yield self.store.store_room(
-            self.room.to_string(),
-            room_creator_user_id="@creator:text",
-            is_public=True
+            self.room.to_string(), room_creator_user_id="@creator:text", is_public=True
         )
 
     @defer.inlineCallbacks
     def inject_state_event(self, room, sender, typ, state_key, content):
-        builder = self.event_builder_factory.new({
-            "type": typ,
-            "sender": sender.to_string(),
-            "state_key": state_key,
-            "room_id": room.to_string(),
-            "content": content,
-        })
+        builder = self.event_builder_factory.new(
+            {
+                "type": typ,
+                "sender": sender.to_string(),
+                "state_key": state_key,
+                "room_id": room.to_string(),
+                "content": content,
+            }
+        )
 
         event, context = yield self.event_creation_handler.create_new_client_event(
             builder
@@ -80,27 +80,31 @@ def test_get_state_for_event(self):
         # this defaults to a linear DAG as each new injection defaults to whatever
         # forward extremities are currently in the DB for this room.
         e1 = yield self.inject_state_event(
-            self.room, self.u_alice, EventTypes.Create, '', {},
+            self.room, self.u_alice, EventTypes.Create, '', {}
         )
         e2 = yield self.inject_state_event(
-            self.room, self.u_alice, EventTypes.Name, '', {
-                "name": "test room"
-            },
+            self.room, self.u_alice, EventTypes.Name, '', {"name": "test room"}
         )
         e3 = yield self.inject_state_event(
-            self.room, self.u_alice, EventTypes.Member, self.u_alice.to_string(), {
-                "membership": Membership.JOIN
-            },
+            self.room,
+            self.u_alice,
+            EventTypes.Member,
+            self.u_alice.to_string(),
+            {"membership": Membership.JOIN},
         )
         e4 = yield self.inject_state_event(
-            self.room, self.u_bob, EventTypes.Member, self.u_bob.to_string(), {
-                "membership": Membership.JOIN
-            },
+            self.room,
+            self.u_bob,
+            EventTypes.Member,
+            self.u_bob.to_string(),
+            {"membership": Membership.JOIN},
         )
         e5 = yield self.inject_state_event(
-            self.room, self.u_bob, EventTypes.Member, self.u_bob.to_string(), {
-                "membership": Membership.LEAVE
-            },
+            self.room,
+            self.u_bob,
+            EventTypes.Member,
+            self.u_bob.to_string(),
+            {"membership": Membership.LEAVE},
         )
 
         # check we get the full state as of the final event
@@ -110,65 +114,66 @@ def test_get_state_for_event(self):
 
         self.assertIsNotNone(e4)
 
-        self.assertStateMapEqual({
-            (e1.type, e1.state_key): e1,
-            (e2.type, e2.state_key): e2,
-            (e3.type, e3.state_key): e3,
-            # e4 is overwritten by e5
-            (e5.type, e5.state_key): e5,
-        }, state)
+        self.assertStateMapEqual(
+            {
+                (e1.type, e1.state_key): e1,
+                (e2.type, e2.state_key): e2,
+                (e3.type, e3.state_key): e3,
+                # e4 is overwritten by e5
+                (e5.type, e5.state_key): e5,
+            },
+            state,
+        )
 
         # check we can filter to the m.room.name event (with a '' state key)
         state = yield self.store.get_state_for_event(
             e5.event_id, [(EventTypes.Name, '')], filtered_types=None
         )
 
-        self.assertStateMapEqual({
-            (e2.type, e2.state_key): e2,
-        }, state)
+        self.assertStateMapEqual({(e2.type, e2.state_key): e2}, state)
 
         # check we can filter to the m.room.name event (with a wildcard None state key)
         state = yield self.store.get_state_for_event(
             e5.event_id, [(EventTypes.Name, None)], filtered_types=None
         )
 
-        self.assertStateMapEqual({
-            (e2.type, e2.state_key): e2,
-        }, state)
+        self.assertStateMapEqual({(e2.type, e2.state_key): e2}, state)
 
         # check we can grab the m.room.member events (with a wildcard None state key)
         state = yield self.store.get_state_for_event(
             e5.event_id, [(EventTypes.Member, None)], filtered_types=None
         )
 
-        self.assertStateMapEqual({
-            (e3.type, e3.state_key): e3,
-            (e5.type, e5.state_key): e5,
-        }, state)
+        self.assertStateMapEqual(
+            {(e3.type, e3.state_key): e3, (e5.type, e5.state_key): e5}, state
+        )
 
         # check we can use filter_types to grab a specific room member
         # without filtering out the other event types
         state = yield self.store.get_state_for_event(
-            e5.event_id, [(EventTypes.Member, self.u_alice.to_string())],
+            e5.event_id,
+            [(EventTypes.Member, self.u_alice.to_string())],
             filtered_types=[EventTypes.Member],
         )
 
-        self.assertStateMapEqual({
-            (e1.type, e1.state_key): e1,
-            (e2.type, e2.state_key): e2,
-            (e3.type, e3.state_key): e3,
-        }, state)
+        self.assertStateMapEqual(
+            {
+                (e1.type, e1.state_key): e1,
+                (e2.type, e2.state_key): e2,
+                (e3.type, e3.state_key): e3,
+            },
+            state,
+        )
 
         # check that types=[], filtered_types=[EventTypes.Member]
         # doesn't return all members
         state = yield self.store.get_state_for_event(
-            e5.event_id, [], filtered_types=[EventTypes.Member],
+            e5.event_id, [], filtered_types=[EventTypes.Member]
         )
 
-        self.assertStateMapEqual({
-            (e1.type, e1.state_key): e1,
-            (e2.type, e2.state_key): e2,
-        }, state)
+        self.assertStateMapEqual(
+            {(e1.type, e1.state_key): e1, (e2.type, e2.state_key): e2}, state
+        )
 
         #######################################################
         # _get_some_state_from_cache tests against a full cache
@@ -184,10 +189,13 @@ def test_get_state_for_event(self):
         )
 
         self.assertEqual(is_all, True)
-        self.assertDictEqual({
-            (e1.type, e1.state_key): e1.event_id,
-            (e2.type, e2.state_key): e2.event_id,
-        }, state_dict)
+        self.assertDictEqual(
+            {
+                (e1.type, e1.state_key): e1.event_id,
+                (e2.type, e2.state_key): e2.event_id,
+            },
+            state_dict,
+        )
 
         # test _get_some_state_from_cache correctly filters in members with wildcard types
         (state_dict, is_all) = yield self.store._get_some_state_from_cache(
@@ -195,25 +203,33 @@ def test_get_state_for_event(self):
         )
 
         self.assertEqual(is_all, True)
-        self.assertDictEqual({
-            (e1.type, e1.state_key): e1.event_id,
-            (e2.type, e2.state_key): e2.event_id,
-            (e3.type, e3.state_key): e3.event_id,
-            # e4 is overwritten by e5
-            (e5.type, e5.state_key): e5.event_id,
-        }, state_dict)
+        self.assertDictEqual(
+            {
+                (e1.type, e1.state_key): e1.event_id,
+                (e2.type, e2.state_key): e2.event_id,
+                (e3.type, e3.state_key): e3.event_id,
+                # e4 is overwritten by e5
+                (e5.type, e5.state_key): e5.event_id,
+            },
+            state_dict,
+        )
 
         # test _get_some_state_from_cache correctly filters in members with specific types
         (state_dict, is_all) = yield self.store._get_some_state_from_cache(
-            group, [(EventTypes.Member, e5.state_key)], filtered_types=[EventTypes.Member]
+            group,
+            [(EventTypes.Member, e5.state_key)],
+            filtered_types=[EventTypes.Member],
         )
 
         self.assertEqual(is_all, True)
-        self.assertDictEqual({
-            (e1.type, e1.state_key): e1.event_id,
-            (e2.type, e2.state_key): e2.event_id,
-            (e5.type, e5.state_key): e5.event_id,
-        }, state_dict)
+        self.assertDictEqual(
+            {
+                (e1.type, e1.state_key): e1.event_id,
+                (e2.type, e2.state_key): e2.event_id,
+                (e5.type, e5.state_key): e5.event_id,
+            },
+            state_dict,
+        )
 
         # test _get_some_state_from_cache correctly filters in members with specific types
         # and no filtered_types
@@ -222,24 +238,27 @@ def test_get_state_for_event(self):
         )
 
         self.assertEqual(is_all, True)
-        self.assertDictEqual({
-            (e5.type, e5.state_key): e5.event_id,
-        }, state_dict)
+        self.assertDictEqual({(e5.type, e5.state_key): e5.event_id}, state_dict)
 
         #######################################################
         # deliberately remove e2 (room name) from the _state_group_cache
 
-        (is_all, known_absent, state_dict_ids) = self.store._state_group_cache.get(group)
+        (is_all, known_absent, state_dict_ids) = self.store._state_group_cache.get(
+            group
+        )
 
         self.assertEqual(is_all, True)
         self.assertEqual(known_absent, set())
-        self.assertDictEqual(state_dict_ids, {
-            (e1.type, e1.state_key): e1.event_id,
-            (e2.type, e2.state_key): e2.event_id,
-            (e3.type, e3.state_key): e3.event_id,
-            # e4 is overwritten by e5
-            (e5.type, e5.state_key): e5.event_id,
-        })
+        self.assertDictEqual(
+            state_dict_ids,
+            {
+                (e1.type, e1.state_key): e1.event_id,
+                (e2.type, e2.state_key): e2.event_id,
+                (e3.type, e3.state_key): e3.event_id,
+                # e4 is overwritten by e5
+                (e5.type, e5.state_key): e5.event_id,
+            },
+        )
 
         state_dict_ids.pop((e2.type, e2.state_key))
         self.store._state_group_cache.invalidate(group)
@@ -252,22 +271,32 @@ def test_get_state_for_event(self):
                 (e1.type, e1.state_key),
                 (e3.type, e3.state_key),
                 (e5.type, e5.state_key),
-            )
+            ),
         )
 
-        (is_all, known_absent, state_dict_ids) = self.store._state_group_cache.get(group)
+        (is_all, known_absent, state_dict_ids) = self.store._state_group_cache.get(
+            group
+        )
 
         self.assertEqual(is_all, False)
-        self.assertEqual(known_absent, set([
-            (e1.type, e1.state_key),
-            (e3.type, e3.state_key),
-            (e5.type, e5.state_key),
-        ]))
-        self.assertDictEqual(state_dict_ids, {
-            (e1.type, e1.state_key): e1.event_id,
-            (e3.type, e3.state_key): e3.event_id,
-            (e5.type, e5.state_key): e5.event_id,
-        })
+        self.assertEqual(
+            known_absent,
+            set(
+                [
+                    (e1.type, e1.state_key),
+                    (e3.type, e3.state_key),
+                    (e5.type, e5.state_key),
+                ]
+            ),
+        )
+        self.assertDictEqual(
+            state_dict_ids,
+            {
+                (e1.type, e1.state_key): e1.event_id,
+                (e3.type, e3.state_key): e3.event_id,
+                (e5.type, e5.state_key): e5.event_id,
+            },
+        )
 
         ############################################
         # test that things work with a partial cache
@@ -279,9 +308,7 @@ def test_get_state_for_event(self):
         )
 
         self.assertEqual(is_all, False)
-        self.assertDictEqual({
-            (e1.type, e1.state_key): e1.event_id,
-        }, state_dict)
+        self.assertDictEqual({(e1.type, e1.state_key): e1.event_id}, state_dict)
 
         # test _get_some_state_from_cache correctly filters in members wildcard types
         (state_dict, is_all) = yield self.store._get_some_state_from_cache(
@@ -289,23 +316,31 @@ def test_get_state_for_event(self):
         )
 
         self.assertEqual(is_all, False)
-        self.assertDictEqual({
-            (e1.type, e1.state_key): e1.event_id,
-            (e3.type, e3.state_key): e3.event_id,
-            # e4 is overwritten by e5
-            (e5.type, e5.state_key): e5.event_id,
-        }, state_dict)
+        self.assertDictEqual(
+            {
+                (e1.type, e1.state_key): e1.event_id,
+                (e3.type, e3.state_key): e3.event_id,
+                # e4 is overwritten by e5
+                (e5.type, e5.state_key): e5.event_id,
+            },
+            state_dict,
+        )
 
         # test _get_some_state_from_cache correctly filters in members with specific types
         (state_dict, is_all) = yield self.store._get_some_state_from_cache(
-            group, [(EventTypes.Member, e5.state_key)], filtered_types=[EventTypes.Member]
+            group,
+            [(EventTypes.Member, e5.state_key)],
+            filtered_types=[EventTypes.Member],
         )
 
         self.assertEqual(is_all, False)
-        self.assertDictEqual({
-            (e1.type, e1.state_key): e1.event_id,
-            (e5.type, e5.state_key): e5.event_id,
-        }, state_dict)
+        self.assertDictEqual(
+            {
+                (e1.type, e1.state_key): e1.event_id,
+                (e5.type, e5.state_key): e5.event_id,
+            },
+            state_dict,
+        )
 
         # test _get_some_state_from_cache correctly filters in members with specific types
         # and no filtered_types
@@ -314,6 +349,4 @@ def test_get_state_for_event(self):
         )
 
         self.assertEqual(is_all, True)
-        self.assertDictEqual({
-            (e5.type, e5.state_key): e5.event_id,
-        }, state_dict)
+        self.assertDictEqual({(e5.type, e5.state_key): e5.event_id}, state_dict)
diff --git a/tests/storage/test_user_directory.py b/tests/storage/test_user_directory.py
index 23fad12bca62..7a273eab4880 100644
--- a/tests/storage/test_user_directory.py
+++ b/tests/storage/test_user_directory.py
@@ -39,20 +39,12 @@ def setUp(self):
             {
                 ALICE: ProfileInfo(None, "alice"),
                 BOB: ProfileInfo(None, "bob"),
-                BOBBY: ProfileInfo(None, "bobby")
+                BOBBY: ProfileInfo(None, "bobby"),
             },
         )
-        yield self.store.add_users_to_public_room(
-            "!room:id",
-            [ALICE, BOB],
-        )
+        yield self.store.add_users_to_public_room("!room:id", [ALICE, BOB])
         yield self.store.add_users_who_share_room(
-            "!room:id",
-            False,
-            (
-                (ALICE, BOB),
-                (BOB, ALICE),
-            ),
+            "!room:id", False, ((ALICE, BOB), (BOB, ALICE))
         )
 
     @defer.inlineCallbacks
@@ -62,11 +54,9 @@ def test_search_user_dir(self):
         r = yield self.store.search_user_dir(ALICE, "bob", 10)
         self.assertFalse(r["limited"])
         self.assertEqual(1, len(r["results"]))
-        self.assertDictEqual(r["results"][0], {
-            "user_id": BOB,
-            "display_name": "bob",
-            "avatar_url": None,
-        })
+        self.assertDictEqual(
+            r["results"][0], {"user_id": BOB, "display_name": "bob", "avatar_url": None}
+        )
 
     @defer.inlineCallbacks
     def test_search_user_dir_all_users(self):
@@ -75,15 +65,13 @@ def test_search_user_dir_all_users(self):
             r = yield self.store.search_user_dir(ALICE, "bob", 10)
             self.assertFalse(r["limited"])
             self.assertEqual(2, len(r["results"]))
-            self.assertDictEqual(r["results"][0], {
-                "user_id": BOB,
-                "display_name": "bob",
-                "avatar_url": None,
-            })
-            self.assertDictEqual(r["results"][1], {
-                "user_id": BOBBY,
-                "display_name": "bobby",
-                "avatar_url": None,
-            })
+            self.assertDictEqual(
+                r["results"][0],
+                {"user_id": BOB, "display_name": "bob", "avatar_url": None},
+            )
+            self.assertDictEqual(
+                r["results"][1],
+                {"user_id": BOBBY, "display_name": "bobby", "avatar_url": None},
+            )
         finally:
             self.hs.config.user_directory_search_all_users = False
diff --git a/tests/test_distributor.py b/tests/test_distributor.py
index 71d11cda77c9..b57f36e6ac26 100644
--- a/tests/test_distributor.py
+++ b/tests/test_distributor.py
@@ -22,7 +22,6 @@
 
 
 class DistributorTestCase(unittest.TestCase):
-
     def setUp(self):
         self.dist = Distributor()
 
@@ -44,18 +43,14 @@ def test_signal_catch(self):
 
         observers[0].side_effect = Exception("Awoogah!")
 
-        with patch(
-            "synapse.util.distributor.logger", spec=["warning"]
-        ) as mock_logger:
+        with patch("synapse.util.distributor.logger", spec=["warning"]) as mock_logger:
             self.dist.fire("alarm", "Go")
 
             observers[0].assert_called_once_with("Go")
             observers[1].assert_called_once_with("Go")
 
             self.assertEquals(mock_logger.warning.call_count, 1)
-            self.assertIsInstance(
-                mock_logger.warning.call_args[0][0], str
-            )
+            self.assertIsInstance(mock_logger.warning.call_args[0][0], str)
 
     def test_signal_prereg(self):
         observer = Mock()
@@ -69,4 +64,5 @@ def test_signal_prereg(self):
     def test_signal_undeclared(self):
         def code():
             self.dist.fire("notification")
+
         self.assertRaises(KeyError, code)
diff --git a/tests/test_dns.py b/tests/test_dns.py
index b647d92697f4..90bd34be349d 100644
--- a/tests/test_dns.py
+++ b/tests/test_dns.py
@@ -27,7 +27,6 @@
 
 @unittest.DEBUG
 class DnsTestCase(unittest.TestCase):
-
     @defer.inlineCallbacks
     def test_resolve(self):
         dns_client_mock = Mock()
@@ -36,14 +35,11 @@ def test_resolve(self):
         host_name = "example.com"
 
         answer_srv = dns.RRHeader(
-            type=dns.SRV,
-            payload=dns.Record_SRV(
-                target=host_name,
-            )
+            type=dns.SRV, payload=dns.Record_SRV(target=host_name)
         )
 
         dns_client_mock.lookupService.return_value = defer.succeed(
-            ([answer_srv], None, None),
+            ([answer_srv], None, None)
         )
 
         cache = {}
@@ -68,9 +64,7 @@ def test_from_cache_expired_and_dns_fail(self):
         entry = Mock(spec_set=["expires"])
         entry.expires = 0
 
-        cache = {
-            service_name: [entry]
-        }
+        cache = {service_name: [entry]}
 
         servers = yield resolve_service(
             service_name, dns_client=dns_client_mock, cache=cache
@@ -93,12 +87,10 @@ def test_from_cache(self):
         entry = Mock(spec_set=["expires"])
         entry.expires = 999999999
 
-        cache = {
-            service_name: [entry]
-        }
+        cache = {service_name: [entry]}
 
         servers = yield resolve_service(
-            service_name, dns_client=dns_client_mock, cache=cache, clock=clock,
+            service_name, dns_client=dns_client_mock, cache=cache, clock=clock
         )
 
         self.assertFalse(dns_client_mock.lookupService.called)
@@ -117,9 +109,7 @@ def test_empty_cache(self):
         cache = {}
 
         with self.assertRaises(error.DNSServerError):
-            yield resolve_service(
-                service_name, dns_client=dns_client_mock, cache=cache
-            )
+            yield resolve_service(service_name, dns_client=dns_client_mock, cache=cache)
 
     @defer.inlineCallbacks
     def test_name_error(self):
diff --git a/tests/test_event_auth.py b/tests/test_event_auth.py
index 06112430e555..411b4a9f86b9 100644
--- a/tests/test_event_auth.py
+++ b/tests/test_event_auth.py
@@ -35,10 +35,7 @@ def test_random_users_cannot_send_state_before_first_pl(self):
         }
 
         # creator should be able to send state
-        event_auth.check(
-            _random_state_event(creator), auth_events,
-            do_sig_check=False,
-        )
+        event_auth.check(_random_state_event(creator), auth_events, do_sig_check=False)
 
         # joiner should not be able to send state
         self.assertRaises(
@@ -61,13 +58,9 @@ def test_state_default_level(self):
         auth_events = {
             ("m.room.create", ""): _create_event(creator),
             ("m.room.member", creator): _join_event(creator),
-            ("m.room.power_levels", ""): _power_levels_event(creator, {
-                "state_default": "30",
-                "users": {
-                    pleb: "29",
-                    king: "30",
-                },
-            }),
+            ("m.room.power_levels", ""): _power_levels_event(
+                creator, {"state_default": "30", "users": {pleb: "29", king: "30"}}
+            ),
             ("m.room.member", pleb): _join_event(pleb),
             ("m.room.member", king): _join_event(king),
         }
@@ -82,10 +75,7 @@ def test_state_default_level(self):
         ),
 
         # king should be able to send state
-        event_auth.check(
-            _random_state_event(king), auth_events,
-            do_sig_check=False,
-        )
+        event_auth.check(_random_state_event(king), auth_events, do_sig_check=False)
 
 
 # helpers for making events
@@ -94,52 +84,54 @@ def test_state_default_level(self):
 
 
 def _create_event(user_id):
-    return FrozenEvent({
-        "room_id": TEST_ROOM_ID,
-        "event_id": _get_event_id(),
-        "type": "m.room.create",
-        "sender": user_id,
-        "content": {
-            "creator": user_id,
-        },
-    })
+    return FrozenEvent(
+        {
+            "room_id": TEST_ROOM_ID,
+            "event_id": _get_event_id(),
+            "type": "m.room.create",
+            "sender": user_id,
+            "content": {"creator": user_id},
+        }
+    )
 
 
 def _join_event(user_id):
-    return FrozenEvent({
-        "room_id": TEST_ROOM_ID,
-        "event_id": _get_event_id(),
-        "type": "m.room.member",
-        "sender": user_id,
-        "state_key": user_id,
-        "content": {
-            "membership": "join",
-        },
-    })
+    return FrozenEvent(
+        {
+            "room_id": TEST_ROOM_ID,
+            "event_id": _get_event_id(),
+            "type": "m.room.member",
+            "sender": user_id,
+            "state_key": user_id,
+            "content": {"membership": "join"},
+        }
+    )
 
 
 def _power_levels_event(sender, content):
-    return FrozenEvent({
-        "room_id": TEST_ROOM_ID,
-        "event_id": _get_event_id(),
-        "type": "m.room.power_levels",
-        "sender": sender,
-        "state_key": "",
-        "content": content,
-    })
+    return FrozenEvent(
+        {
+            "room_id": TEST_ROOM_ID,
+            "event_id": _get_event_id(),
+            "type": "m.room.power_levels",
+            "sender": sender,
+            "state_key": "",
+            "content": content,
+        }
+    )
 
 
 def _random_state_event(sender):
-    return FrozenEvent({
-        "room_id": TEST_ROOM_ID,
-        "event_id": _get_event_id(),
-        "type": "test.state",
-        "sender": sender,
-        "state_key": "",
-        "content": {
-            "membership": "join",
-        },
-    })
+    return FrozenEvent(
+        {
+            "room_id": TEST_ROOM_ID,
+            "event_id": _get_event_id(),
+            "type": "test.state",
+            "sender": sender,
+            "state_key": "",
+            "content": {"membership": "join"},
+        }
+    )
 
 
 event_count = 0
@@ -149,4 +141,4 @@ def _get_event_id():
     global event_count
     c = event_count
     event_count += 1
-    return "!%i:example.com" % (c, )
+    return "!%i:example.com" % (c,)
diff --git a/tests/test_preview.py b/tests/test_preview.py
index 446843367ef5..84ef5e5ba4b6 100644
--- a/tests/test_preview.py
+++ b/tests/test_preview.py
@@ -22,7 +22,6 @@
 
 
 class PreviewTestCase(unittest.TestCase):
-
     def test_long_summarize(self):
         example_paras = [
             u"""Tromsø (Norwegian pronunciation: [ˈtrʊmsœ] ( listen); Northern Sami:
@@ -32,7 +31,6 @@ def test_long_summarize(self):
             alternative spellings of the city.Tromsø is considered the northernmost
             city in the world with a population above 50,000. The most populous town
             north of it is Alta, Norway, with a population of 14,272 (2013).""",
-
             u"""Tromsø lies in Northern Norway. The municipality has a population of
             (2015) 72,066, but with an annual influx of students it has over 75,000
             most of the year. It is the largest urban area in Northern Norway and the
@@ -46,7 +44,6 @@ def test_long_summarize(self):
             Sandnessund Bridge. Tromsø Airport connects the city to many destinations
             in Europe. The city is warmer than most other places located on the same
             latitude, due to the warming effect of the Gulf Stream.""",
-
             u"""The city centre of Tromsø contains the highest number of old wooden
             houses in Northern Norway, the oldest house dating from 1789. The Arctic
             Cathedral, a modern church from 1965, is probably the most famous landmark
@@ -67,7 +64,7 @@ def test_long_summarize(self):
             u" the city of Tromsø. Outside of Norway, Tromso and Tromsö are"
             u" alternative spellings of the city.Tromsø is considered the northernmost"
             u" city in the world with a population above 50,000. The most populous town"
-            u" north of it is Alta, Norway, with a population of 14,272 (2013)."
+            u" north of it is Alta, Norway, with a population of 14,272 (2013).",
         )
 
         desc = summarize_paragraphs(example_paras[1:], min_size=200, max_size=500)
@@ -80,7 +77,7 @@ def test_long_summarize(self):
             u" third largest north of the Arctic Circle (following Murmansk and Norilsk)."
             u" Most of Tromsø, including the city centre, is located on the island of"
             u" Tromsøya, 350 kilometres (217 mi) north of the Arctic Circle. In 2012,"
-            u" Tromsøya had a population of 36,088. Substantial parts of the urban…"
+            u" Tromsøya had a population of 36,088. Substantial parts of the urban…",
         )
 
     def test_short_summarize(self):
@@ -88,11 +85,9 @@ def test_short_summarize(self):
             u"Tromsø (Norwegian pronunciation: [ˈtrʊmsœ] ( listen); Northern Sami:"
             u" Romsa; Finnish: Tromssa[2] Kven: Tromssa) is a city and municipality in"
             u" Troms county, Norway.",
-
             u"Tromsø lies in Northern Norway. The municipality has a population of"
             u" (2015) 72,066, but with an annual influx of students it has over 75,000"
             u" most of the year.",
-
             u"The city centre of Tromsø contains the highest number of old wooden"
             u" houses in Northern Norway, the oldest house dating from 1789. The Arctic"
             u" Cathedral, a modern church from 1965, is probably the most famous landmark"
@@ -109,7 +104,7 @@ def test_short_summarize(self):
             u"\n"
             u"Tromsø lies in Northern Norway. The municipality has a population of"
             u" (2015) 72,066, but with an annual influx of students it has over 75,000"
-            u" most of the year."
+            u" most of the year.",
         )
 
     def test_small_then_large_summarize(self):
@@ -117,7 +112,6 @@ def test_small_then_large_summarize(self):
             u"Tromsø (Norwegian pronunciation: [ˈtrʊmsœ] ( listen); Northern Sami:"
             u" Romsa; Finnish: Tromssa[2] Kven: Tromssa) is a city and municipality in"
             u" Troms county, Norway.",
-
             u"Tromsø lies in Northern Norway. The municipality has a population of"
             u" (2015) 72,066, but with an annual influx of students it has over 75,000"
             u" most of the year."
@@ -138,7 +132,7 @@ def test_small_then_large_summarize(self):
             u" (2015) 72,066, but with an annual influx of students it has over 75,000"
             u" most of the year. The city centre of Tromsø contains the highest number"
             u" of old wooden houses in Northern Norway, the oldest house dating from"
-            u" 1789. The Arctic Cathedral, a modern church from…"
+            u" 1789. The Arctic Cathedral, a modern church from…",
         )
 
 
@@ -155,10 +149,7 @@ def test_simple(self):
 
         og = decode_and_calc_og(html, "http://example.com/test.html")
 
-        self.assertEquals(og, {
-            u"og:title": u"Foo",
-            u"og:description": u"Some text."
-        })
+        self.assertEquals(og, {u"og:title": u"Foo", u"og:description": u"Some text."})
 
     def test_comment(self):
         html = u"""
@@ -173,10 +164,7 @@ def test_comment(self):
 
         og = decode_and_calc_og(html, "http://example.com/test.html")
 
-        self.assertEquals(og, {
-            u"og:title": u"Foo",
-            u"og:description": u"Some text."
-        })
+        self.assertEquals(og, {u"og:title": u"Foo", u"og:description": u"Some text."})
 
     def test_comment2(self):
         html = u"""
@@ -194,10 +182,13 @@ def test_comment2(self):
 
         og = decode_and_calc_og(html, "http://example.com/test.html")
 
-        self.assertEquals(og, {
-            u"og:title": u"Foo",
-            u"og:description": u"Some text.\n\nSome more text.\n\nText\n\nMore text"
-        })
+        self.assertEquals(
+            og,
+            {
+                u"og:title": u"Foo",
+                u"og:description": u"Some text.\n\nSome more text.\n\nText\n\nMore text",
+            },
+        )
 
     def test_script(self):
         html = u"""
@@ -212,10 +203,7 @@ def test_script(self):
 
         og = decode_and_calc_og(html, "http://example.com/test.html")
 
-        self.assertEquals(og, {
-            u"og:title": u"Foo",
-            u"og:description": u"Some text."
-        })
+        self.assertEquals(og, {u"og:title": u"Foo", u"og:description": u"Some text."})
 
     def test_missing_title(self):
         html = u"""
@@ -228,10 +216,7 @@ def test_missing_title(self):
 
         og = decode_and_calc_og(html, "http://example.com/test.html")
 
-        self.assertEquals(og, {
-            u"og:title": None,
-            u"og:description": u"Some text."
-        })
+        self.assertEquals(og, {u"og:title": None, u"og:description": u"Some text."})
 
     def test_h1_as_title(self):
         html = u"""
@@ -245,10 +230,7 @@ def test_h1_as_title(self):
 
         og = decode_and_calc_og(html, "http://example.com/test.html")
 
-        self.assertEquals(og, {
-            u"og:title": u"Title",
-            u"og:description": u"Some text."
-        })
+        self.assertEquals(og, {u"og:title": u"Title", u"og:description": u"Some text."})
 
     def test_missing_title_and_broken_h1(self):
         html = u"""
@@ -262,7 +244,4 @@ def test_missing_title_and_broken_h1(self):
 
         og = decode_and_calc_og(html, "http://example.com/test.html")
 
-        self.assertEquals(og, {
-            u"og:title": None,
-            u"og:description": u"Some text."
-        })
+        self.assertEquals(og, {u"og:title": None, u"og:description": u"Some text."})
diff --git a/tests/test_state.py b/tests/test_state.py
index 429a18cbf75e..96fdb8636caa 100644
--- a/tests/test_state.py
+++ b/tests/test_state.py
@@ -29,8 +29,15 @@
 _next_event_id = 1000
 
 
-def create_event(name=None, type=None, state_key=None, depth=2, event_id=None,
-                 prev_events=[], **kwargs):
+def create_event(
+    name=None,
+    type=None,
+    state_key=None,
+    depth=2,
+    event_id=None,
+    prev_events=[],
+    **kwargs
+):
     global _next_event_id
 
     if not event_id:
@@ -39,9 +46,9 @@ def create_event(name=None, type=None, state_key=None, depth=2, event_id=None,
 
     if not name:
         if state_key is not None:
-            name = "<%s-%s, %s>" % (type, state_key, event_id,)
+            name = "<%s-%s, %s>" % (type, state_key, event_id)
         else:
-            name = "<%s, %s>" % (type, event_id,)
+            name = "<%s, %s>" % (type, event_id)
 
     d = {
         "event_id": event_id,
@@ -80,8 +87,9 @@ def get_state_groups_ids(self, room_id, event_ids):
 
         return defer.succeed(groups)
 
-    def store_state_group(self, event_id, room_id, prev_group, delta_ids,
-                          current_state_ids):
+    def store_state_group(
+        self, event_id, room_id, prev_group, delta_ids, current_state_ids
+    ):
         state_group = self._next_group
         self._next_group += 1
 
@@ -91,7 +99,8 @@ def store_state_group(self, event_id, room_id, prev_group, delta_ids,
 
     def get_events(self, event_ids, **kwargs):
         return {
-            e_id: self._event_id_to_event[e_id] for e_id in event_ids
+            e_id: self._event_id_to_event[e_id]
+            for e_id in event_ids
             if e_id in self._event_id_to_event
         }
 
@@ -129,9 +138,7 @@ def __init__(self, nodes, edges):
                 prev_events = []
 
             events[event_id] = create_event(
-                event_id=event_id,
-                prev_events=prev_events,
-                **fields
+                event_id=event_id, prev_events=prev_events, **fields
             )
 
         self._leaves = clobbered
@@ -147,10 +154,15 @@ def get_leaves(self):
 class StateTestCase(unittest.TestCase):
     def setUp(self):
         self.store = StateGroupStore()
-        hs = Mock(spec_set=[
-            "get_datastore", "get_auth", "get_state_handler", "get_clock",
-            "get_state_resolution_handler",
-        ])
+        hs = Mock(
+            spec_set=[
+                "get_datastore",
+                "get_auth",
+                "get_state_handler",
+                "get_clock",
+                "get_state_resolution_handler",
+            ]
+        )
         hs.get_datastore.return_value = self.store
         hs.get_state_handler.return_value = None
         hs.get_clock.return_value = MockClock()
@@ -164,35 +176,13 @@ def setUp(self):
     def test_branch_no_conflict(self):
         graph = Graph(
             nodes={
-                "START": DictObj(
-                    type=EventTypes.Create,
-                    state_key="",
-                    depth=1,
-                ),
-                "A": DictObj(
-                    type=EventTypes.Message,
-                    depth=2,
-                ),
-                "B": DictObj(
-                    type=EventTypes.Message,
-                    depth=3,
-                ),
-                "C": DictObj(
-                    type=EventTypes.Name,
-                    state_key="",
-                    depth=3,
-                ),
-                "D": DictObj(
-                    type=EventTypes.Message,
-                    depth=4,
-                ),
+                "START": DictObj(type=EventTypes.Create, state_key="", depth=1),
+                "A": DictObj(type=EventTypes.Message, depth=2),
+                "B": DictObj(type=EventTypes.Message, depth=3),
+                "C": DictObj(type=EventTypes.Name, state_key="", depth=3),
+                "D": DictObj(type=EventTypes.Message, depth=4),
             },
-            edges={
-                "A": ["START"],
-                "B": ["A"],
-                "C": ["A"],
-                "D": ["B", "C"]
-            }
+            edges={"A": ["START"], "B": ["A"], "C": ["A"], "D": ["B", "C"]},
         )
 
         self.store.register_events(graph.walk())
@@ -224,27 +214,11 @@ def test_branch_basic_conflict(self):
                     membership=Membership.JOIN,
                     depth=2,
                 ),
-                "B": DictObj(
-                    type=EventTypes.Name,
-                    state_key="",
-                    depth=3,
-                ),
-                "C": DictObj(
-                    type=EventTypes.Name,
-                    state_key="",
-                    depth=4,
-                ),
-                "D": DictObj(
-                    type=EventTypes.Message,
-                    depth=5,
-                ),
+                "B": DictObj(type=EventTypes.Name, state_key="", depth=3),
+                "C": DictObj(type=EventTypes.Name, state_key="", depth=4),
+                "D": DictObj(type=EventTypes.Message, depth=5),
             },
-            edges={
-                "A": ["START"],
-                "B": ["A"],
-                "C": ["A"],
-                "D": ["B", "C"]
-            }
+            edges={"A": ["START"], "B": ["A"], "C": ["A"], "D": ["B", "C"]},
         )
 
         self.store.register_events(graph.walk())
@@ -259,8 +233,7 @@ def test_branch_basic_conflict(self):
         prev_state_ids = yield context_store["D"].get_prev_state_ids(self.store)
 
         self.assertSetEqual(
-            {"START", "A", "C"},
-            {e_id for e_id in prev_state_ids.values()}
+            {"START", "A", "C"}, {e_id for e_id in prev_state_ids.values()}
         )
 
     @defer.inlineCallbacks
@@ -280,11 +253,7 @@ def test_branch_have_banned_conflict(self):
                     membership=Membership.JOIN,
                     depth=2,
                 ),
-                "B": DictObj(
-                    type=EventTypes.Name,
-                    state_key="",
-                    depth=3,
-                ),
+                "B": DictObj(type=EventTypes.Name, state_key="", depth=3),
                 "C": DictObj(
                     type=EventTypes.Member,
                     state_key="@user_id_2:example.com",
@@ -298,18 +267,9 @@ def test_branch_have_banned_conflict(self):
                     depth=4,
                     sender="@user_id_2:example.com",
                 ),
-                "E": DictObj(
-                    type=EventTypes.Message,
-                    depth=5,
-                ),
+                "E": DictObj(type=EventTypes.Message, depth=5),
             },
-            edges={
-                "A": ["START"],
-                "B": ["A"],
-                "C": ["B"],
-                "D": ["B"],
-                "E": ["C", "D"]
-            }
+            edges={"A": ["START"], "B": ["A"], "C": ["B"], "D": ["B"], "E": ["C", "D"]},
         )
 
         self.store.register_events(graph.walk())
@@ -324,8 +284,7 @@ def test_branch_have_banned_conflict(self):
         prev_state_ids = yield context_store["E"].get_prev_state_ids(self.store)
 
         self.assertSetEqual(
-            {"START", "A", "B", "C"},
-            {e for e in prev_state_ids.values()}
+            {"START", "A", "B", "C"}, {e for e in prev_state_ids.values()}
         )
 
     @defer.inlineCallbacks
@@ -357,30 +316,17 @@ def test_branch_have_perms_conflict(self):
                 state_key="",
                 content={
                     "events": {"m.room.name": 50},
-                    "users": {userid1: 100,
-                              userid2: 60},
+                    "users": {userid1: 100, userid2: 60},
                 },
             ),
-            "A5": DictObj(
-                type=EventTypes.Name,
-                state_key="",
-            ),
+            "A5": DictObj(type=EventTypes.Name, state_key=""),
             "B": DictObj(
                 type=EventTypes.PowerLevels,
                 state_key="",
-                content={
-                    "events": {"m.room.name": 50},
-                    "users": {userid2: 30},
-                },
-            ),
-            "C": DictObj(
-                type=EventTypes.Name,
-                state_key="",
-                sender=userid2,
-            ),
-            "D": DictObj(
-                type=EventTypes.Message,
+                content={"events": {"m.room.name": 50}, "users": {userid2: 30}},
             ),
+            "C": DictObj(type=EventTypes.Name, state_key="", sender=userid2),
+            "D": DictObj(type=EventTypes.Message),
         }
         edges = {
             "A2": ["A1"],
@@ -389,7 +335,7 @@ def test_branch_have_perms_conflict(self):
             "A5": ["A4"],
             "B": ["A5"],
             "C": ["A5"],
-            "D": ["B", "C"]
+            "D": ["B", "C"],
         }
         self._add_depths(nodes, edges)
         graph = Graph(nodes, edges)
@@ -406,8 +352,7 @@ def test_branch_have_perms_conflict(self):
         prev_state_ids = yield context_store["D"].get_prev_state_ids(self.store)
 
         self.assertSetEqual(
-            {"A1", "A2", "A3", "A5", "B"},
-            {e for e in prev_state_ids.values()}
+            {"A1", "A2", "A3", "A5", "B"}, {e for e in prev_state_ids.values()}
         )
 
     def _add_depths(self, nodes, edges):
@@ -432,9 +377,7 @@ def test_annotate_with_old_message(self):
             create_event(type="test2", state_key=""),
         ]
 
-        context = yield self.state.compute_event_context(
-            event, old_state=old_state
-        )
+        context = yield self.state.compute_event_context(event, old_state=old_state)
 
         current_state_ids = yield context.get_current_state_ids(self.store)
 
@@ -454,9 +397,7 @@ def test_annotate_with_old_state(self):
             create_event(type="test2", state_key=""),
         ]
 
-        context = yield self.state.compute_event_context(
-            event, old_state=old_state
-        )
+        context = yield self.state.compute_event_context(event, old_state=old_state)
 
         prev_state_ids = yield context.get_prev_state_ids(self.store)
 
@@ -468,8 +409,7 @@ def test_annotate_with_old_state(self):
     def test_trivial_annotate_message(self):
         prev_event_id = "prev_event_id"
         event = create_event(
-            type="test_message", name="event2",
-            prev_events=[(prev_event_id, {})],
+            type="test_message", name="event2", prev_events=[(prev_event_id, {})]
         )
 
         old_state = [
@@ -479,7 +419,10 @@ def test_trivial_annotate_message(self):
         ]
 
         group_name = self.store.store_state_group(
-            prev_event_id, event.room_id, None, None,
+            prev_event_id,
+            event.room_id,
+            None,
+            None,
             {(e.type, e.state_key): e.event_id for e in old_state},
         )
         self.store.register_event_id_state_group(prev_event_id, group_name)
@@ -489,8 +432,7 @@ def test_trivial_annotate_message(self):
         current_state_ids = yield context.get_current_state_ids(self.store)
 
         self.assertEqual(
-            set([e.event_id for e in old_state]),
-            set(current_state_ids.values())
+            set([e.event_id for e in old_state]), set(current_state_ids.values())
         )
 
         self.assertEqual(group_name, context.state_group)
@@ -499,8 +441,7 @@ def test_trivial_annotate_message(self):
     def test_trivial_annotate_state(self):
         prev_event_id = "prev_event_id"
         event = create_event(
-            type="state", state_key="", name="event2",
-            prev_events=[(prev_event_id, {})],
+            type="state", state_key="", name="event2", prev_events=[(prev_event_id, {})]
         )
 
         old_state = [
@@ -510,7 +451,10 @@ def test_trivial_annotate_state(self):
         ]
 
         group_name = self.store.store_state_group(
-            prev_event_id, event.room_id, None, None,
+            prev_event_id,
+            event.room_id,
+            None,
+            None,
             {(e.type, e.state_key): e.event_id for e in old_state},
         )
         self.store.register_event_id_state_group(prev_event_id, group_name)
@@ -520,8 +464,7 @@ def test_trivial_annotate_state(self):
         prev_state_ids = yield context.get_prev_state_ids(self.store)
 
         self.assertEqual(
-            set([e.event_id for e in old_state]),
-            set(prev_state_ids.values())
+            set([e.event_id for e in old_state]), set(prev_state_ids.values())
         )
 
         self.assertIsNotNone(context.state_group)
@@ -531,13 +474,12 @@ def test_resolve_message_conflict(self):
         prev_event_id1 = "event_id1"
         prev_event_id2 = "event_id2"
         event = create_event(
-            type="test_message", name="event3",
+            type="test_message",
+            name="event3",
             prev_events=[(prev_event_id1, {}), (prev_event_id2, {})],
         )
 
-        creation = create_event(
-            type=EventTypes.Create, state_key=""
-        )
+        creation = create_event(type=EventTypes.Create, state_key="")
 
         old_state_1 = [
             creation,
@@ -557,7 +499,7 @@ def test_resolve_message_conflict(self):
         self.store.register_events(old_state_2)
 
         context = yield self._get_context(
-            event, prev_event_id1, old_state_1, prev_event_id2, old_state_2,
+            event, prev_event_id1, old_state_1, prev_event_id2, old_state_2
         )
 
         current_state_ids = yield context.get_current_state_ids(self.store)
@@ -571,13 +513,13 @@ def test_resolve_state_conflict(self):
         prev_event_id1 = "event_id1"
         prev_event_id2 = "event_id2"
         event = create_event(
-            type="test4", state_key="", name="event",
+            type="test4",
+            state_key="",
+            name="event",
             prev_events=[(prev_event_id1, {}), (prev_event_id2, {})],
         )
 
-        creation = create_event(
-            type=EventTypes.Create, state_key=""
-        )
+        creation = create_event(type=EventTypes.Create, state_key="")
 
         old_state_1 = [
             creation,
@@ -599,7 +541,7 @@ def test_resolve_state_conflict(self):
         self.store.get_events = store.get_events
 
         context = yield self._get_context(
-            event, prev_event_id1, old_state_1, prev_event_id2, old_state_2,
+            event, prev_event_id1, old_state_1, prev_event_id2, old_state_2
         )
 
         current_state_ids = yield context.get_current_state_ids(self.store)
@@ -613,29 +555,25 @@ def test_standard_depth_conflict(self):
         prev_event_id1 = "event_id1"
         prev_event_id2 = "event_id2"
         event = create_event(
-            type="test4", name="event",
+            type="test4",
+            name="event",
             prev_events=[(prev_event_id1, {}), (prev_event_id2, {})],
         )
 
         member_event = create_event(
             type=EventTypes.Member,
             state_key="@user_id:example.com",
-            content={
-                "membership": Membership.JOIN,
-            }
+            content={"membership": Membership.JOIN},
         )
 
         power_levels = create_event(
-            type=EventTypes.PowerLevels, state_key="",
-            content={"users": {
-                "@foo:bar": "100",
-                "@user_id:example.com": "100",
-            }}
+            type=EventTypes.PowerLevels,
+            state_key="",
+            content={"users": {"@foo:bar": "100", "@user_id:example.com": "100"}},
         )
 
         creation = create_event(
-            type=EventTypes.Create, state_key="",
-            content={"creator": "@foo:bar"}
+            type=EventTypes.Create, state_key="", content={"creator": "@foo:bar"}
         )
 
         old_state_1 = [
@@ -658,14 +596,12 @@ def test_standard_depth_conflict(self):
         self.store.get_events = store.get_events
 
         context = yield self._get_context(
-            event, prev_event_id1, old_state_1, prev_event_id2, old_state_2,
+            event, prev_event_id1, old_state_1, prev_event_id2, old_state_2
         )
 
         current_state_ids = yield context.get_current_state_ids(self.store)
 
-        self.assertEqual(
-            old_state_2[3].event_id, current_state_ids[("test1", "1")]
-        )
+        self.assertEqual(old_state_2[3].event_id, current_state_ids[("test1", "1")])
 
         # Reverse the depth to make sure we are actually using the depths
         # during state resolution.
@@ -688,25 +624,30 @@ def test_standard_depth_conflict(self):
         store.register_events(old_state_2)
 
         context = yield self._get_context(
-            event, prev_event_id1, old_state_1, prev_event_id2, old_state_2,
+            event, prev_event_id1, old_state_1, prev_event_id2, old_state_2
         )
 
         current_state_ids = yield context.get_current_state_ids(self.store)
 
-        self.assertEqual(
-            old_state_1[3].event_id, current_state_ids[("test1", "1")]
-        )
+        self.assertEqual(old_state_1[3].event_id, current_state_ids[("test1", "1")])
 
-    def _get_context(self, event, prev_event_id_1, old_state_1, prev_event_id_2,
-                     old_state_2):
+    def _get_context(
+        self, event, prev_event_id_1, old_state_1, prev_event_id_2, old_state_2
+    ):
         sg1 = self.store.store_state_group(
-            prev_event_id_1, event.room_id, None, None,
+            prev_event_id_1,
+            event.room_id,
+            None,
+            None,
             {(e.type, e.state_key): e.event_id for e in old_state_1},
         )
         self.store.register_event_id_state_group(prev_event_id_1, sg1)
 
         sg2 = self.store.store_state_group(
-            prev_event_id_2, event.room_id, None, None,
+            prev_event_id_2,
+            event.room_id,
+            None,
+            None,
             {(e.type, e.state_key): e.event_id for e in old_state_2},
         )
         self.store.register_event_id_state_group(prev_event_id_2, sg2)
diff --git a/tests/test_test_utils.py b/tests/test_test_utils.py
index bc97c122458a..b921ac52c07a 100644
--- a/tests/test_test_utils.py
+++ b/tests/test_test_utils.py
@@ -18,7 +18,6 @@
 
 
 class MockClockTestCase(unittest.TestCase):
-
     def setUp(self):
         self.clock = MockClock()
 
@@ -34,10 +33,12 @@ def test_later(self):
 
         def _cb0():
             invoked[0] = 1
+
         self.clock.call_later(10, _cb0)
 
         def _cb1():
             invoked[1] = 1
+
         self.clock.call_later(20, _cb1)
 
         self.assertFalse(invoked[0])
@@ -56,10 +57,12 @@ def test_cancel_later(self):
 
         def _cb0():
             invoked[0] = 1
+
         t0 = self.clock.call_later(10, _cb0)
 
         def _cb1():
             invoked[1] = 1
+
         self.clock.call_later(20, _cb1)
 
         self.clock.cancel_call_later(t0)
diff --git a/tests/test_types.py b/tests/test_types.py
index 729bd676c144..be072d402b85 100644
--- a/tests/test_types.py
+++ b/tests/test_types.py
@@ -69,10 +69,7 @@ def test_parse(self):
         self.assertEqual("my.domain", group_id.domain)
 
     def test_validate(self):
-        bad_ids = [
-            "$badsigil:domain",
-            "+:empty",
-        ] + [
+        bad_ids = ["$badsigil:domain", "+:empty"] + [
             "+group" + c + ":domain" for c in "A%?æ£"
         ]
         for id_string in bad_ids:
diff --git a/tests/test_visibility.py b/tests/test_visibility.py
index 0dc1a924d331..8643d6312570 100644
--- a/tests/test_visibility.py
+++ b/tests/test_visibility.py
@@ -54,14 +54,12 @@ def test_filtering(self):
         events_to_filter = []
 
         for i in range(0, 10):
-            user = "@user%i:%s" % (
-                i, "test_server" if i == 5 else "other_server"
-            )
+            user = "@user%i:%s" % (i, "test_server" if i == 5 else "other_server")
             evt = yield self.inject_room_member(user, extra_content={"a": "b"})
             events_to_filter.append(evt)
 
         filtered = yield filter_events_for_server(
-            self.store, "test_server", events_to_filter,
+            self.store, "test_server", events_to_filter
         )
 
         # the result should be 5 redacted events, and 5 unredacted events.
@@ -100,19 +98,21 @@ def test_erased_user(self):
 
         # ... and the filtering happens.
         filtered = yield filter_events_for_server(
-            self.store, "test_server", events_to_filter,
+            self.store, "test_server", events_to_filter
         )
 
         for i in range(0, len(events_to_filter)):
             self.assertEqual(
-                events_to_filter[i].event_id, filtered[i].event_id,
-                "Unexpected event at result position %i" % (i, )
+                events_to_filter[i].event_id,
+                filtered[i].event_id,
+                "Unexpected event at result position %i" % (i,),
             )
 
         for i in (0, 3):
             self.assertEqual(
-                events_to_filter[i].content["body"], filtered[i].content["body"],
-                "Unexpected event content at result position %i" % (i,)
+                events_to_filter[i].content["body"],
+                filtered[i].content["body"],
+                "Unexpected event content at result position %i" % (i,),
             )
 
         for i in (1, 4):
@@ -121,13 +121,15 @@ def test_erased_user(self):
     @defer.inlineCallbacks
     def inject_visibility(self, user_id, visibility):
         content = {"history_visibility": visibility}
-        builder = self.event_builder_factory.new({
-            "type": "m.room.history_visibility",
-            "sender": user_id,
-            "state_key": "",
-            "room_id": TEST_ROOM_ID,
-            "content": content,
-        })
+        builder = self.event_builder_factory.new(
+            {
+                "type": "m.room.history_visibility",
+                "sender": user_id,
+                "state_key": "",
+                "room_id": TEST_ROOM_ID,
+                "content": content,
+            }
+        )
 
         event, context = yield self.event_creation_handler.create_new_client_event(
             builder
@@ -139,13 +141,15 @@ def inject_visibility(self, user_id, visibility):
     def inject_room_member(self, user_id, membership="join", extra_content={}):
         content = {"membership": membership}
         content.update(extra_content)
-        builder = self.event_builder_factory.new({
-            "type": "m.room.member",
-            "sender": user_id,
-            "state_key": user_id,
-            "room_id": TEST_ROOM_ID,
-            "content": content,
-        })
+        builder = self.event_builder_factory.new(
+            {
+                "type": "m.room.member",
+                "sender": user_id,
+                "state_key": user_id,
+                "room_id": TEST_ROOM_ID,
+                "content": content,
+            }
+        )
 
         event, context = yield self.event_creation_handler.create_new_client_event(
             builder
@@ -158,12 +162,14 @@ def inject_room_member(self, user_id, membership="join", extra_content={}):
     def inject_message(self, user_id, content=None):
         if content is None:
             content = {"body": "testytest"}
-        builder = self.event_builder_factory.new({
-            "type": "m.room.message",
-            "sender": user_id,
-            "room_id": TEST_ROOM_ID,
-            "content": content,
-        })
+        builder = self.event_builder_factory.new(
+            {
+                "type": "m.room.message",
+                "sender": user_id,
+                "room_id": TEST_ROOM_ID,
+                "content": content,
+            }
+        )
 
         event, context = yield self.event_creation_handler.create_new_client_event(
             builder
@@ -192,56 +198,54 @@ def test_large_room(self):
         # history_visibility event.
         room_state = []
 
-        history_visibility_evt = FrozenEvent({
-            "event_id": "$history_vis",
-            "type": "m.room.history_visibility",
-            "sender": "@resident_user_0:test.com",
-            "state_key": "",
-            "room_id": TEST_ROOM_ID,
-            "content": {"history_visibility": "joined"},
-        })
+        history_visibility_evt = FrozenEvent(
+            {
+                "event_id": "$history_vis",
+                "type": "m.room.history_visibility",
+                "sender": "@resident_user_0:test.com",
+                "state_key": "",
+                "room_id": TEST_ROOM_ID,
+                "content": {"history_visibility": "joined"},
+            }
+        )
         room_state.append(history_visibility_evt)
         test_store.add_event(history_visibility_evt)
 
         for i in range(0, 100000):
-            user = "@resident_user_%i:test.com" % (i, )
-            evt = FrozenEvent({
-                "event_id": "$res_event_%i" % (i, ),
-                "type": "m.room.member",
-                "state_key": user,
-                "sender": user,
-                "room_id": TEST_ROOM_ID,
-                "content": {
-                    "membership": "join",
-                    "extra": "zzz,"
-                },
-            })
+            user = "@resident_user_%i:test.com" % (i,)
+            evt = FrozenEvent(
+                {
+                    "event_id": "$res_event_%i" % (i,),
+                    "type": "m.room.member",
+                    "state_key": user,
+                    "sender": user,
+                    "room_id": TEST_ROOM_ID,
+                    "content": {"membership": "join", "extra": "zzz,"},
+                }
+            )
             room_state.append(evt)
             test_store.add_event(evt)
 
         events_to_filter = []
         for i in range(0, 10):
-            user = "@user%i:%s" % (
-                i, "test_server" if i == 5 else "other_server"
+            user = "@user%i:%s" % (i, "test_server" if i == 5 else "other_server")
+            evt = FrozenEvent(
+                {
+                    "event_id": "$evt%i" % (i,),
+                    "type": "m.room.member",
+                    "state_key": user,
+                    "sender": user,
+                    "room_id": TEST_ROOM_ID,
+                    "content": {"membership": "join", "extra": "zzz"},
+                }
             )
-            evt = FrozenEvent({
-                "event_id": "$evt%i" % (i, ),
-                "type": "m.room.member",
-                "state_key": user,
-                "sender": user,
-                "room_id": TEST_ROOM_ID,
-                "content": {
-                    "membership": "join",
-                    "extra": "zzz",
-                },
-            })
             events_to_filter.append(evt)
             room_state.append(evt)
 
             test_store.add_event(evt)
-            test_store.set_state_ids_for_event(evt, {
-                (e.type, e.state_key): e.event_id for e in room_state
-            })
+            test_store.set_state_ids_for_event(
+                evt, {(e.type, e.state_key): e.event_id for e in room_state}
+            )
 
         pr = cProfile.Profile()
         pr.enable()
@@ -249,7 +253,7 @@ def test_large_room(self):
         logger.info("Starting filtering")
         start = time.time()
         filtered = yield filter_events_for_server(
-            test_store, "test_server", events_to_filter,
+            test_store, "test_server", events_to_filter
         )
         logger.info("Filtering took %f seconds", time.time() - start)
 
@@ -275,6 +279,7 @@ class _TestStore(object):
     filter_events_for_server
 
     """
+
     def __init__(self):
         # data for get_events: a map from event_id to event
         self.events = {}
@@ -298,8 +303,8 @@ def get_state_ids_for_events(self, events, types):
                 continue
             if type != "m.room.member" or state_key is not None:
                 raise RuntimeError(
-                    "Unimplemented: get_state_ids with type (%s, %s)" %
-                    (type, state_key),
+                    "Unimplemented: get_state_ids with type (%s, %s)"
+                    % (type, state_key)
                 )
             include_memberships = True
 
@@ -316,9 +321,7 @@ def get_state_ids_for_events(self, events, types):
         return succeed(res)
 
     def get_events(self, events):
-        return succeed({
-            event_id: self.events[event_id] for event_id in events
-        })
+        return succeed({event_id: self.events[event_id] for event_id in events})
 
     def are_users_erased(self, users):
         return succeed({u: False for u in users})
diff --git a/tests/unittest.py b/tests/unittest.py
index b15b06726b30..f448a6dfbd40 100644
--- a/tests/unittest.py
+++ b/tests/unittest.py
@@ -56,6 +56,7 @@ def around(target):
     def method_name(orig, *args, **kwargs):
         return orig(*args, **kwargs)
     """
+
     def _around(code):
         name = code.__name__
         orig = getattr(target, name)
@@ -89,6 +90,7 @@ def setUp(orig):
             old_level = logging.getLogger().level
 
             if old_level != level:
+
                 @around(self)
                 def tearDown(orig):
                     ret = orig()
@@ -117,8 +119,9 @@ def assert_dict(self, required, actual):
             actual (dict): The test result. Extra keys will not be checked.
         """
         for key in required:
-            self.assertEquals(required[key], actual[key],
-                              msg="%s mismatch. %s" % (key, actual))
+            self.assertEquals(
+                required[key], actual[key], msg="%s mismatch. %s" % (key, actual)
+            )
 
 
 def DEBUG(target):
diff --git a/tests/util/caches/test_descriptors.py b/tests/util/caches/test_descriptors.py
index ca8a7c907fbb..463a737efaf5 100644
--- a/tests/util/caches/test_descriptors.py
+++ b/tests/util/caches/test_descriptors.py
@@ -67,12 +67,8 @@ def record_callback(idx):
         self.assertIsNone(cache.get("key2", None))
 
         # both callbacks should have been callbacked
-        self.assertTrue(
-            callback_record[0], "Invalidation callback for key1 not called",
-        )
-        self.assertTrue(
-            callback_record[1], "Invalidation callback for key2 not called",
-        )
+        self.assertTrue(callback_record[0], "Invalidation callback for key1 not called")
+        self.assertTrue(callback_record[1], "Invalidation callback for key2 not called")
 
         # letting the other lookup complete should do nothing
         d1.callback("result1")
@@ -168,8 +164,7 @@ def do_lookup():
             with logcontext.LoggingContext() as c1:
                 c1.name = "c1"
                 r = yield obj.fn(1)
-                self.assertEqual(logcontext.LoggingContext.current_context(),
-                                 c1)
+                self.assertEqual(logcontext.LoggingContext.current_context(), c1)
             defer.returnValue(r)
 
         def check_result(r):
@@ -179,14 +174,18 @@ def check_result(r):
 
         # set off a deferred which will do a cache lookup
         d1 = do_lookup()
-        self.assertEqual(logcontext.LoggingContext.current_context(),
-                         logcontext.LoggingContext.sentinel)
+        self.assertEqual(
+            logcontext.LoggingContext.current_context(),
+            logcontext.LoggingContext.sentinel,
+        )
         d1.addCallback(check_result)
 
         # and another
         d2 = do_lookup()
-        self.assertEqual(logcontext.LoggingContext.current_context(),
-                         logcontext.LoggingContext.sentinel)
+        self.assertEqual(
+            logcontext.LoggingContext.current_context(),
+            logcontext.LoggingContext.sentinel,
+        )
         d2.addCallback(check_result)
 
         # let the lookup complete
@@ -224,15 +223,16 @@ def do_lookup():
                 except SynapseError:
                     pass
 
-                self.assertEqual(logcontext.LoggingContext.current_context(),
-                                 c1)
+                self.assertEqual(logcontext.LoggingContext.current_context(), c1)
 
         obj = Cls()
 
         # set off a deferred which will do a cache lookup
         d1 = do_lookup()
-        self.assertEqual(logcontext.LoggingContext.current_context(),
-                         logcontext.LoggingContext.sentinel)
+        self.assertEqual(
+            logcontext.LoggingContext.current_context(),
+            logcontext.LoggingContext.sentinel,
+        )
 
         return d1
 
@@ -288,14 +288,10 @@ def fn(self, arg1, arg2):
 
             @descriptors.cachedList("fn", "args1", inlineCallbacks=True)
             def list_fn(self, args1, arg2):
-                assert (
-                    logcontext.LoggingContext.current_context().request == "c1"
-                )
+                assert logcontext.LoggingContext.current_context().request == "c1"
                 # we want this to behave like an asynchronous function
                 yield run_on_reactor()
-                assert (
-                    logcontext.LoggingContext.current_context().request == "c1"
-                )
+                assert logcontext.LoggingContext.current_context().request == "c1"
                 defer.returnValue(self.mock(args1, arg2))
 
         with logcontext.LoggingContext() as c1:
@@ -308,10 +304,7 @@ def list_fn(self, args1, arg2):
                 logcontext.LoggingContext.sentinel,
             )
             r = yield d1
-            self.assertEqual(
-                logcontext.LoggingContext.current_context(),
-                c1
-            )
+            self.assertEqual(logcontext.LoggingContext.current_context(), c1)
             obj.mock.assert_called_once_with([10, 20], 2)
             self.assertEqual(r, {10: 'fish', 20: 'chips'})
             obj.mock.reset_mock()
@@ -337,6 +330,7 @@ def list_fn(self, args1, arg2):
     @defer.inlineCallbacks
     def test_invalidate(self):
         """Make sure that invalidation callbacks are called."""
+
         class Cls(object):
             def __init__(self):
                 self.mock = mock.Mock()
diff --git a/tests/util/test_dict_cache.py b/tests/util/test_dict_cache.py
index 26f2fa580094..34fdc9a43af8 100644
--- a/tests/util/test_dict_cache.py
+++ b/tests/util/test_dict_cache.py
@@ -20,7 +20,6 @@
 
 
 class DictCacheTestCase(unittest.TestCase):
-
     def setUp(self):
         self.cache = DictionaryCache("foobar")
 
@@ -41,9 +40,7 @@ def test_simple_cache_hit_partial(self):
         key = "test_simple_cache_hit_partial"
 
         seq = self.cache.sequence
-        test_value = {
-            "test": "test_simple_cache_hit_partial"
-        }
+        test_value = {"test": "test_simple_cache_hit_partial"}
         self.cache.update(seq, key, test_value)
 
         c = self.cache.get(key, ["test"])
@@ -53,9 +50,7 @@ def test_simple_cache_miss_partial(self):
         key = "test_simple_cache_miss_partial"
 
         seq = self.cache.sequence
-        test_value = {
-            "test": "test_simple_cache_miss_partial"
-        }
+        test_value = {"test": "test_simple_cache_miss_partial"}
         self.cache.update(seq, key, test_value)
 
         c = self.cache.get(key, ["test2"])
@@ -79,15 +74,11 @@ def test_multi_insert(self):
         key = "test_simple_cache_hit_miss_partial"
 
         seq = self.cache.sequence
-        test_value_1 = {
-            "test": "test_simple_cache_hit_miss_partial",
-        }
+        test_value_1 = {"test": "test_simple_cache_hit_miss_partial"}
         self.cache.update(seq, key, test_value_1, fetched_keys=set("test"))
 
         seq = self.cache.sequence
-        test_value_2 = {
-            "test2": "test_simple_cache_hit_miss_partial2",
-        }
+        test_value_2 = {"test2": "test_simple_cache_hit_miss_partial2"}
         self.cache.update(seq, key, test_value_2, fetched_keys=set("test2"))
 
         c = self.cache.get(key)
@@ -96,5 +87,5 @@ def test_multi_insert(self):
                 "test": "test_simple_cache_hit_miss_partial",
                 "test2": "test_simple_cache_hit_miss_partial2",
             },
-            c.value
+            c.value,
         )
diff --git a/tests/util/test_expiring_cache.py b/tests/util/test_expiring_cache.py
index d12b5e838b1e..5cbada4edaff 100644
--- a/tests/util/test_expiring_cache.py
+++ b/tests/util/test_expiring_cache.py
@@ -22,7 +22,6 @@
 
 
 class ExpiringCacheTestCase(unittest.TestCase):
-
     def test_get_set(self):
         clock = MockClock()
         cache = ExpiringCache("test", clock, max_len=1)
diff --git a/tests/util/test_file_consumer.py b/tests/util/test_file_consumer.py
index 7ce5f8c258f4..e90e08d1c042 100644
--- a/tests/util/test_file_consumer.py
+++ b/tests/util/test_file_consumer.py
@@ -27,7 +27,6 @@
 
 
 class FileConsumerTests(unittest.TestCase):
-
     @defer.inlineCallbacks
     def test_pull_consumer(self):
         string_file = StringIO()
@@ -87,7 +86,9 @@ def test_push_producer_feedback(self):
             producer = NonCallableMock(spec_set=["pauseProducing", "resumeProducing"])
 
             resume_deferred = defer.Deferred()
-            producer.resumeProducing.side_effect = lambda: resume_deferred.callback(None)
+            producer.resumeProducing.side_effect = lambda: resume_deferred.callback(
+                None
+            )
 
             consumer.registerProducer(producer, True)
 
diff --git a/tests/util/test_linearizer.py b/tests/util/test_linearizer.py
index 7f48a72de88f..61a55b461b80 100644
--- a/tests/util/test_linearizer.py
+++ b/tests/util/test_linearizer.py
@@ -26,7 +26,6 @@
 
 
 class LinearizerTestCase(unittest.TestCase):
-
     @defer.inlineCallbacks
     def test_linearizer(self):
         linearizer = Linearizer()
@@ -54,13 +53,11 @@ def test_lots_of_queued_things(self):
         def func(i, sleep=False):
             with logcontext.LoggingContext("func(%s)" % i) as lc:
                 with (yield linearizer.queue("")):
-                    self.assertEqual(
-                        logcontext.LoggingContext.current_context(), lc)
+                    self.assertEqual(logcontext.LoggingContext.current_context(), lc)
                     if sleep:
                         yield Clock(reactor).sleep(0)
 
-                self.assertEqual(
-                    logcontext.LoggingContext.current_context(), lc)
+                self.assertEqual(logcontext.LoggingContext.current_context(), lc)
 
         func(0, sleep=True)
         for i in range(1, 100):
diff --git a/tests/util/test_logcontext.py b/tests/util/test_logcontext.py
index c54001f7a424..4633db77b3fa 100644
--- a/tests/util/test_logcontext.py
+++ b/tests/util/test_logcontext.py
@@ -8,11 +8,8 @@
 
 
 class LoggingContextTestCase(unittest.TestCase):
-
     def _check_test_key(self, value):
-        self.assertEquals(
-            LoggingContext.current_context().request, value
-        )
+        self.assertEquals(LoggingContext.current_context().request, value)
 
     def test_with_context(self):
         with LoggingContext() as context_one:
@@ -50,6 +47,7 @@ def cb(res):
                 self._check_test_key("one")
                 callback_completed[0] = True
                 return res
+
             d.addCallback(cb)
 
             return d
@@ -74,8 +72,7 @@ def check_logcontext():
             # make sure that the context was reset before it got thrown back
             # into the reactor
             try:
-                self.assertIs(LoggingContext.current_context(),
-                              sentinel_context)
+                self.assertIs(LoggingContext.current_context(), sentinel_context)
                 d2.callback(None)
             except BaseException:
                 d2.errback(twisted.python.failure.Failure())
@@ -104,9 +101,7 @@ def test_run_in_background_with_chained_deferred(self):
         # a function which returns a deferred which looks like it has been
         # called, but is actually paused
         def testfunc():
-            return logcontext.make_deferred_yieldable(
-                _chained_deferred_function()
-            )
+            return logcontext.make_deferred_yieldable(_chained_deferred_function())
 
         return self._test_run_in_background(testfunc)
 
@@ -175,5 +170,6 @@ def cb(res):
         d2 = defer.Deferred()
         reactor.callLater(0, d2.callback, res)
         return d2
+
     d.addCallback(cb)
     return d
diff --git a/tests/util/test_lrucache.py b/tests/util/test_lrucache.py
index 9b36ef4482b1..786947375df2 100644
--- a/tests/util/test_lrucache.py
+++ b/tests/util/test_lrucache.py
@@ -23,7 +23,6 @@
 
 
 class LruCacheTestCase(unittest.TestCase):
-
     def test_get_set(self):
         cache = LruCache(1)
         cache["key"] = "value"
@@ -235,7 +234,6 @@ def test_eviction(self):
 
 
 class LruCacheSizedTestCase(unittest.TestCase):
-
     def test_evict(self):
         cache = LruCache(5, size_callback=len)
         cache["key1"] = [0]
diff --git a/tests/util/test_rwlock.py b/tests/util/test_rwlock.py
index 7cd470be6743..bd32e2cee7d1 100644
--- a/tests/util/test_rwlock.py
+++ b/tests/util/test_rwlock.py
@@ -20,7 +20,6 @@
 
 
 class ReadWriteLockTestCase(unittest.TestCase):
-
     def _assert_called_before_not_after(self, lst, first_false):
         for i, d in enumerate(lst[:first_false]):
             self.assertTrue(d.called, msg="%d was unexpectedly false" % i)
@@ -36,12 +35,12 @@ def test_rwlock(self):
         key = object()
 
         ds = [
-            rwlock.read(key),   # 0
-            rwlock.read(key),   # 1
+            rwlock.read(key),  # 0
+            rwlock.read(key),  # 1
             rwlock.write(key),  # 2
             rwlock.write(key),  # 3
-            rwlock.read(key),   # 4
-            rwlock.read(key),   # 5
+            rwlock.read(key),  # 4
+            rwlock.read(key),  # 5
             rwlock.write(key),  # 6
         ]
 
diff --git a/tests/util/test_snapshot_cache.py b/tests/util/test_snapshot_cache.py
index 0f5b32fcc065..1a44f7242551 100644
--- a/tests/util/test_snapshot_cache.py
+++ b/tests/util/test_snapshot_cache.py
@@ -22,7 +22,6 @@
 
 
 class SnapshotCacheTestCase(unittest.TestCase):
-
     def setUp(self):
         self.cache = SnapshotCache()
         self.cache.DURATION_MS = 1
diff --git a/tests/util/test_stream_change_cache.py b/tests/util/test_stream_change_cache.py
index 65b0f2e6fbc8..f2be63706bba 100644
--- a/tests/util/test_stream_change_cache.py
+++ b/tests/util/test_stream_change_cache.py
@@ -181,17 +181,8 @@ def test_get_entities_changed(self):
         # Query a subset of the entries mid-way through the stream. We should
         # only get back the subset.
         self.assertEqual(
-            cache.get_entities_changed(
-                [
-                    "bar@baz.net",
-                ],
-                stream_pos=2,
-            ),
-            set(
-                [
-                    "bar@baz.net",
-                ]
-            ),
+            cache.get_entities_changed(["bar@baz.net"], stream_pos=2),
+            set(["bar@baz.net"]),
         )
 
     def test_max_pos(self):
diff --git a/tests/utils.py b/tests/utils.py
index 8d73797971fa..8668b5478fd8 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -38,8 +38,9 @@
 
 
 @defer.inlineCallbacks
-def setup_test_homeserver(name="test", datastore=None, config=None, reactor=None,
-                          **kargs):
+def setup_test_homeserver(
+    name="test", datastore=None, config=None, reactor=None, **kargs
+):
     """Setup a homeserver suitable for running tests against. Keyword arguments
     are passed to the Homeserver constructor. If no datastore is supplied a
     datastore backed by an in-memory sqlite db will be given to the HS.
@@ -96,20 +97,12 @@ def setup_test_homeserver(name="test", datastore=None, config=None, reactor=None
     if USE_POSTGRES_FOR_TESTS:
         config.database_config = {
             "name": "psycopg2",
-            "args": {
-                "database": "synapse_test",
-                "cp_min": 1,
-                "cp_max": 5,
-            },
+            "args": {"database": "synapse_test", "cp_min": 1, "cp_max": 5},
         }
     else:
         config.database_config = {
             "name": "sqlite3",
-            "args": {
-                "database": ":memory:",
-                "cp_min": 1,
-                "cp_max": 1,
-            },
+            "args": {"database": ":memory:", "cp_min": 1, "cp_max": 1},
         }
 
     db_engine = create_engine(config.database_config)
@@ -121,7 +114,8 @@ def setup_test_homeserver(name="test", datastore=None, config=None, reactor=None
 
     if datastore is None:
         hs = HomeServer(
-            name, config=config,
+            name,
+            config=config,
             db_config=config.database_config,
             version_string="Synapse/tests",
             database_engine=db_engine,
@@ -143,7 +137,10 @@ def setup_test_homeserver(name="test", datastore=None, config=None, reactor=None
         hs.setup()
     else:
         hs = HomeServer(
-            name, db_pool=None, datastore=datastore, config=config,
+            name,
+            db_pool=None,
+            datastore=datastore,
+            config=config,
             version_string="Synapse/tests",
             database_engine=db_engine,
             room_list_handler=object(),
@@ -158,8 +155,9 @@ def setup_test_homeserver(name="test", datastore=None, config=None, reactor=None
     # because AuthHandler's constructor requires the HS, so we can't make one
     # beforehand and pass it in to the HS's constructor (chicken / egg)
     hs.get_auth_handler().hash = lambda p: hashlib.md5(p.encode('utf8')).hexdigest()
-    hs.get_auth_handler().validate_hash = lambda p, h: hashlib.md5(
-        p.encode('utf8')).hexdigest() == h
+    hs.get_auth_handler().validate_hash = (
+        lambda p, h: hashlib.md5(p.encode('utf8')).hexdigest() == h
+    )
 
     fed = kargs.get("resource_for_federation", None)
     if fed:
@@ -173,7 +171,7 @@ def setup_test_homeserver(name="test", datastore=None, config=None, reactor=None
                 sleep_limit=hs.config.federation_rc_sleep_limit,
                 sleep_msec=hs.config.federation_rc_sleep_delay,
                 reject_limit=hs.config.federation_rc_reject_limit,
-                concurrent_requests=hs.config.federation_rc_concurrent
+                concurrent_requests=hs.config.federation_rc_concurrent,
             ),
         )
 
@@ -199,7 +197,6 @@ def getRawHeaders(name, default=None):
 
 # This is a mock /resource/ not an entire server
 class MockHttpResource(HttpServer):
-
     def __init__(self, prefix=""):
         self.callbacks = []  # 3-tuple of method/pattern/function
         self.prefix = prefix
@@ -263,15 +260,9 @@ def trigger(self, http_method, path, content, mock_request, federation_auth=Fals
             matcher = pattern.match(path)
             if matcher:
                 try:
-                    args = [
-                        urlparse.unquote(u)
-                        for u in matcher.groups()
-                    ]
-
-                    (code, response) = yield func(
-                        mock_request,
-                        *args
-                    )
+                    args = [urlparse.unquote(u) for u in matcher.groups()]
+
+                    (code, response) = yield func(mock_request, *args)
                     defer.returnValue((code, response))
                 except CodeMessageException as e:
                     defer.returnValue((e.code, cs_error(e.msg, code=e.errcode)))
@@ -372,8 +363,7 @@ def time_bound_deferred(self, d, *args, **kwargs):
 
 def _format_call(args, kwargs):
     return ", ".join(
-        ["%r" % (a) for a in args] +
-        ["%s=%r" % (k, v) for k, v in kwargs.items()]
+        ["%r" % (a) for a in args] + ["%s=%r" % (k, v) for k, v in kwargs.items()]
     )
 
 
@@ -391,8 +381,9 @@ def __call__(self, *args, **kwargs):
         self.calls.append((args, kwargs))
 
         if not self.expectations:
-            raise ValueError("%r has no pending calls to handle call(%s)" % (
-                self, _format_call(args, kwargs))
+            raise ValueError(
+                "%r has no pending calls to handle call(%s)"
+                % (self, _format_call(args, kwargs))
             )
 
         for (call, result, d) in self.expectations:
@@ -400,9 +391,9 @@ def __call__(self, *args, **kwargs):
                 d.callback(None)
                 return result
 
-        failure = AssertionError("Was not expecting call(%s)" % (
-            _format_call(args, kwargs)
-        ))
+        failure = AssertionError(
+            "Was not expecting call(%s)" % (_format_call(args, kwargs))
+        )
 
         for _, _, d in self.expectations:
             try:
@@ -418,17 +409,19 @@ def expect_call_and_return(self, call, result):
     @defer.inlineCallbacks
     def await_calls(self, timeout=1000):
         deferred = defer.DeferredList(
-            [d for _, _, d in self.expectations],
-            fireOnOneErrback=True
+            [d for _, _, d in self.expectations], fireOnOneErrback=True
         )
 
         timer = reactor.callLater(
             timeout / 1000,
             deferred.errback,
-            AssertionError("%d pending calls left: %s" % (
-                len([e for e in self.expectations if not e[2].called]),
-                [e for e in self.expectations if not e[2].called]
-            ))
+            AssertionError(
+                "%d pending calls left: %s"
+                % (
+                    len([e for e in self.expectations if not e[2].called]),
+                    [e for e in self.expectations if not e[2].called],
+                )
+            ),
         )
 
         yield deferred
@@ -443,7 +436,6 @@ def assert_had_no_calls(self):
             self.calls = []
 
             raise AssertionError(
-                "Expected not to received any calls, got:\n" + "\n".join([
-                    "call(%s)" % _format_call(c[0], c[1]) for c in calls
-                ])
+                "Expected not to received any calls, got:\n"
+                + "\n".join(["call(%s)" % _format_call(c[0], c[1]) for c in calls])
             )

From dd16e7dfcc31b9bfa835d184510c9630de0b9734 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Fri, 10 Aug 2018 23:54:46 +1000
Subject: [PATCH 340/830] changelog

---
 changelog.d/3679.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3679.misc

diff --git a/changelog.d/3679.misc b/changelog.d/3679.misc
new file mode 100644
index 000000000000..1de0a0f2b41c
--- /dev/null
+++ b/changelog.d/3679.misc
@@ -0,0 +1 @@
+Synapse's tests are now formatted with the black autoformatter.

From c08f9d95b2051c6998a234e4e28e621c174b9a0e Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 10 Aug 2018 14:56:48 +0100
Subject: [PATCH 341/830] log *after* reloading log config

... because logging *before* reloading means the log message gets lost in the old MemoryLogger
---
 synapse/config/logger.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/synapse/config/logger.py b/synapse/config/logger.py
index a87b11a1dff0..cfc20dcccf31 100644
--- a/synapse/config/logger.py
+++ b/synapse/config/logger.py
@@ -193,9 +193,8 @@ def load_log_config():
 
         def sighup(signum, stack):
             # it might be better to use a file watcher or something for this.
-            logging.info("Reloading log config from %s due to SIGHUP",
-                         log_config)
             load_log_config()
+            logging.info("Reloaded log config from %s due to SIGHUP", log_config)
 
         load_log_config()
 

From 807449d8f212ff111acb44d8f418dcac7d69f713 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Sat, 11 Aug 2018 00:01:43 +1000
Subject: [PATCH 342/830] fix up a forced long line

---
 tests/crypto/test_keyring.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/crypto/test_keyring.py b/tests/crypto/test_keyring.py
index e40681ed1e71..0c6f510d113e 100644
--- a/tests/crypto/test_keyring.py
+++ b/tests/crypto/test_keyring.py
@@ -60,8 +60,9 @@ def setUp(self):
         self.hs = yield utils.setup_test_homeserver(
             handlers=None, http_client=self.http_client
         )
+        keys = self.mock_perspective_server.get_verify_keys()
         self.hs.config.perspectives = {
-            self.mock_perspective_server.server_name: self.mock_perspective_server.get_verify_keys()
+            self.mock_perspective_server.server_name: keys
         }
 
     def check_context(self, _, expected):

From 6c6aba76e1059f103ed48487e74ab4161347638b Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Fri, 10 Aug 2018 15:12:59 +0100
Subject: [PATCH 343/830] implementation of server notices to alert on hitting
 resource limits

---
 .../resource_limits_server_notices.py         |  77 ++++++-----
 .../server_notices/server_notices_sender.py   |  33 +++--
 .../test_resource_limits_server_notices.py    | 125 ++++++++++++++++++
 3 files changed, 190 insertions(+), 45 deletions(-)
 create mode 100644 tests/server_notices/test_resource_limits_server_notices.py

diff --git a/synapse/server_notices/resource_limits_server_notices.py b/synapse/server_notices/resource_limits_server_notices.py
index 33b1d80cd9df..017828f81e72 100644
--- a/synapse/server_notices/resource_limits_server_notices.py
+++ b/synapse/server_notices/resource_limits_server_notices.py
@@ -14,14 +14,9 @@
 # limitations under the License.
 import logging
 
-from six import iteritems, string_types
-
 from twisted.internet import defer
 
-from synapse.api.errors import SynapseError
-from synapse.api.urls import ConsentURIBuilder
-from synapse.config import ConfigError
-from synapse.types import get_localpart_from_id
+from synapse.api.errors import AuthError, SynapseError
 
 logger = logging.getLogger(__name__)
 
@@ -37,12 +32,12 @@ def __init__(self, hs):
         """
         self._server_notices_manager = hs.get_server_notices_manager()
         self._store = hs.get_datastore()
-        self._api = hs.get_api()
+        self.auth = hs.get_auth()
         self._server_notice_content = hs.config.user_consent_server_notice_content
-        self._limit_usage_by_mau = config.limit_usage_by_mau = False
-        self._hs_disabled.config.hs_disabled = False
+        self._limit_usage_by_mau = hs.config.limit_usage_by_mau = False
+        self._hs_disabled = hs.config.hs_disabled
 
-        self._notified = set()
+        self._notified_of_blocking = set()
         self._resouce_limited = False
         # Config checks?
 
@@ -56,29 +51,49 @@ def maybe_send_server_notice_to_user(self, user_id):
         Returns:
             Deferred
         """
-        if self._limit_usage_by_mau is False and self._hs_disabled is False:
-            # not enabled
+        if self._hs_disabled is True:
             return
 
-        timestamp = yield self.store.user_last_seen_monthly_active(user_id)
-        if timestamp is None:
-            # This user will be blocked from receiving the notice anyway
-            return
-        try:
-            yield self.api.check_auth_blocking()
-            if self._resouce_limited:
+        if self._limit_usage_by_mau is True:
+            timestamp = yield self._store.user_last_seen_monthly_active(user_id)
+            if timestamp is None:
+                # This user will be blocked from receiving the notice anyway.
+                # In practice, not sure we can ever get here
+                return
+            try:
+                yield self.auth.check_auth_blocking()
+                self._resouce_limited = False
                 # Need to start removing notices
-                pass
-        except AuthError as e:
-            # Need to start notifying of blocking
-            if not self._resouce_limited:
-                pass
+                if user_id in self._notified_of_blocking:
+                    # Send message to remove warning - needs updating
+                    content = "remove warning"
+                    self._send_server_notice(user_id, content)
+                    self._notified_of_blocking.remove(user_id)
 
-            # need to send a message.
-            try:
-                yield self._server_notices_manager.send_notice(
-                    user_id, content,
-                )
+            except AuthError:
+                # Need to start notifying of blocking
+
+                self._resouce_limited = True
+                if user_id not in self._notified_of_blocking:
+                    # Send message to add warning - needs updating
+                    content = "add warning"
+                    self._send_server_notice(user_id, content)
+                    self._notified_of_blocking.add(user_id)
 
-            except SynapseError as e:
-                logger.error("Error sending server notice about resource limits: %s", e)
+    @defer.inlineCallbacks
+    def _send_server_notice(self, user_id, content):
+        """Sends Server notice
+
+        Args:
+            user_id(str): The user to send to
+            content(str): The content of the message
+
+        Returns:
+            Deferred[]
+        """
+        try:
+            yield self._server_notices_manager.send_notice(
+                user_id, content,
+            )
+        except SynapseError as e:
+            logger.error("Error sending server notice about resource limits: %s", e)
diff --git a/synapse/server_notices/server_notices_sender.py b/synapse/server_notices/server_notices_sender.py
index 5d23965f34f8..6121b2f26778 100644
--- a/synapse/server_notices/server_notices_sender.py
+++ b/synapse/server_notices/server_notices_sender.py
@@ -12,7 +12,12 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from twisted.internet import defer
+
 from synapse.server_notices.consent_server_notices import ConsentServerNotices
+from synapse.server_notices.resource_limits_server_notices import (
+    ResourceLimitsServerNotices,
+)
 
 
 class ServerNoticesSender(object):
@@ -25,34 +30,34 @@ def __init__(self, hs):
         Args:
             hs (synapse.server.HomeServer):
         """
-        # todo: it would be nice to make this more dynamic
-        self._consent_server_notices = ConsentServerNotices(hs)
+        self._server_notices = (
+            ConsentServerNotices(hs),
+            ResourceLimitsServerNotices(hs)
+        )
 
+    @defer.inlineCallbacks
     def on_user_syncing(self, user_id):
         """Called when the user performs a sync operation.
 
         Args:
             user_id (str): mxid of user who synced
-
-        Returns:
-            Deferred
         """
-        return self._consent_server_notices.maybe_send_server_notice_to_user(
-            user_id,
-        )
+        for sn in self._server_notices:
+            yield sn.maybe_send_server_notice_to_user(
+                user_id,
+            )
 
+    @defer.inlineCallbacks
     def on_user_ip(self, user_id):
         """Called on the master when a worker process saw a client request.
 
         Args:
             user_id (str): mxid
-
-        Returns:
-            Deferred
         """
         # The synchrotrons use a stubbed version of ServerNoticesSender, so
         # we check for notices to send to the user in on_user_ip as well as
         # in on_user_syncing
-        return self._consent_server_notices.maybe_send_server_notice_to_user(
-            user_id,
-        )
+        for sn in self._server_notices:
+            yield sn.maybe_send_server_notice_to_user(
+                user_id,
+            )
diff --git a/tests/server_notices/test_resource_limits_server_notices.py b/tests/server_notices/test_resource_limits_server_notices.py
new file mode 100644
index 000000000000..a69253f1b6bf
--- /dev/null
+++ b/tests/server_notices/test_resource_limits_server_notices.py
@@ -0,0 +1,125 @@
+from mock import Mock
+
+from twisted.internet import defer
+
+from synapse.api.errors import AuthError
+from synapse.handlers.auth import AuthHandler
+from synapse.server_notices.resource_limits_server_notices import (
+    ResourceLimitsServerNotices,
+)
+
+from tests import unittest
+from tests.utils import setup_test_homeserver
+
+
+class AuthHandlers(object):
+    def __init__(self, hs):
+        self.auth_handler = AuthHandler(hs)
+
+
+class TestResourceLimitsServerNotices(unittest.TestCase):
+    @defer.inlineCallbacks
+    def setUp(self):
+        self.hs = yield setup_test_homeserver(handlers=None)
+        self.hs.handlers = AuthHandlers(self.hs)
+        self.auth_handler = self.hs.handlers.auth_handler
+        self.server_notices_sender = self.hs.get_server_notices_sender()
+
+        # relying on [1] is far from ideal, but the only case where
+        # ResourceLimitsServerNotices class needs to be isolated is this test,
+        # general code should never have a reason to do so ...
+        self._rlsn = self.server_notices_sender._server_notices[1]
+        if not isinstance(self._rlsn, ResourceLimitsServerNotices):
+            raise Exception("Failed to find reference to ResourceLimitsServerNotices")
+
+        self._rlsn._store.user_last_seen_monthly_active = Mock(
+            return_value=defer.succeed(1000)
+        )
+        self._send_notice = self._rlsn._server_notices_manager.send_notice
+        self._rlsn._server_notices_manager.send_notice = Mock()
+        self._send_notice = self._rlsn._server_notices_manager.send_notice
+
+        self._rlsn._limit_usage_by_mau = True
+        self.user_id = "user_id"
+
+    @defer.inlineCallbacks
+    def test_maybe_send_server_notice_to_user_flag_off(self):
+        """Tests cases where the flags indicate nothing to do"""
+        # test hs disabled case
+        self._hs_disabled = True
+
+        yield self._rlsn.maybe_send_server_notice_to_user("user_id")
+
+        self._send_notice.assert_not_called()
+        # Test when mau limiting disabled
+        self._hs_disabled = False
+        self._rlsn._limit_usage_by_mau = False
+        yield self._rlsn.maybe_send_server_notice_to_user("user_id")
+
+        self._send_notice.assert_not_called()
+
+    @defer.inlineCallbacks
+    def test_maybe_send_server_notice_to_user_remove_blocked_notice(self):
+        """Test when user has blocked notice, but should have it removed"""
+
+        self._rlsn._notified_of_blocking.add(self.user_id)
+        self._rlsn.auth.check_auth_blocking = Mock()
+
+        yield self._rlsn.maybe_send_server_notice_to_user(self.user_id)
+        # "remove warning" obviously aweful, but test will start failing when code
+        # actually sends a real event, and then it can be updated
+
+        self._send_notice.assert_called_once_with(self.user_id, "remove warning")
+        self.assertFalse(self.user_id in self._rlsn._notified_of_blocking)
+
+    @defer.inlineCallbacks
+    def test_maybe_send_server_notice_to_user_remove_blocked_notice_noop(self):
+        """Test when user has blocked notice, but notice ought to be there (NOOP)"""
+        self._rlsn._notified_of_blocking.add(self.user_id)
+        self._rlsn.auth.check_auth_blocking = Mock(
+            side_effect=AuthError(403, 'foo')
+        )
+
+        yield self._rlsn.maybe_send_server_notice_to_user("user_id")
+
+        self._send_notice.assert_not_called()
+        self.assertTrue(self.user_id in self._rlsn._notified_of_blocking)
+
+    @defer.inlineCallbacks
+    def test_maybe_send_server_notice_to_user_add_blocked_notice(self):
+        """Test when user does not have blocked notice, but should have one"""
+
+        self._rlsn.auth.check_auth_blocking = Mock(side_effect=AuthError(403, 'foo'))
+        yield self._rlsn.maybe_send_server_notice_to_user("user_id")
+
+        # "add warning" obviously awful, but test will start failing when code
+        # actually sends a real event, and then it can be updated
+        self._send_notice.assert_called_once_with(self.user_id, "add warning")
+        self.assertTrue(self.user_id in self._rlsn._notified_of_blocking)
+
+    @defer.inlineCallbacks
+    def test_maybe_send_server_notice_to_user_add_blocked_notice_noop(self):
+        """Test when user does not have blocked notice, nor should they (NOOP)"""
+
+        self._rlsn.auth.check_auth_blocking = Mock()
+
+        yield self._rlsn.maybe_send_server_notice_to_user(self.user_id)
+
+        self._send_notice.assert_not_called()
+        self.assertFalse(self.user_id in self._rlsn._notified_of_blocking)
+
+    @defer.inlineCallbacks
+    def test_maybe_send_server_notice_to_user_not_in_mau_cohort(self):
+
+        """Test when user is not part of the MAU cohort - this should not ever
+        happen - but ...
+        """
+
+        self._rlsn.auth.check_auth_blocking = Mock()
+        self._rlsn._store.user_last_seen_monthly_active = Mock(
+            return_value=defer.succeed(None)
+        )
+        yield self._rlsn.maybe_send_server_notice_to_user(self.user_id)
+
+        self._send_notice.assert_not_called()
+        self.assertFalse(self.user_id in self._rlsn._notified_of_blocking)

From c79c4c0a7d4887c24050df72417b1be9afe2992c Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Fri, 10 Aug 2018 15:25:47 +0100
Subject: [PATCH 344/830] server notices for resource limit blocking

---
 changelog.d/3680.feature | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3680.feature

diff --git a/changelog.d/3680.feature b/changelog.d/3680.feature
new file mode 100644
index 000000000000..4edaaf76a83e
--- /dev/null
+++ b/changelog.d/3680.feature
@@ -0,0 +1 @@
+Server notices for resource limit blocking

From 31fa743567c3f70c66c7af2f1332560b04b78f7a Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Sat, 11 Aug 2018 22:38:34 +0100
Subject: [PATCH 345/830] fix sqlite/postgres incompatibility in
 reap_monthly_active_users

---
 synapse/storage/monthly_active_users.py | 44 ++++++++++++++++---------
 1 file changed, 28 insertions(+), 16 deletions(-)

diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py
index d47dcef3a0a5..9a6e699386a1 100644
--- a/synapse/storage/monthly_active_users.py
+++ b/synapse/storage/monthly_active_users.py
@@ -64,23 +64,27 @@ def reap_monthly_active_users(self):
             Deferred[]
         """
         def _reap_users(txn):
+            # Purge stale users
 
             thirty_days_ago = (
                 int(self._clock.time_msec()) - (1000 * 60 * 60 * 24 * 30)
             )
-            # Purge stale users
-
-            # questionmarks is a hack to overcome sqlite not supporting
-            # tuples in 'WHERE IN %s'
-            questionmarks = '?' * len(self.reserved_users)
             query_args = [thirty_days_ago]
-            query_args.extend(self.reserved_users)
-
-            sql = """
-                DELETE FROM monthly_active_users
-                WHERE timestamp < ?
-                AND user_id NOT IN ({})
-                """.format(','.join(questionmarks))
+            base_sql = "DELETE FROM monthly_active_users WHERE timestamp < ?"
+
+            # Need if/else since 'AND user_id NOT IN ({})' fails on Postgres
+            # when len(reserved_users) == 0. Works fine on sqlite.
+            if len(self.reserved_users) > 0:
+                # questionmarks is a hack to overcome sqlite not supporting
+                # tuples in 'WHERE IN %s'
+                questionmarks = '?' * len(self.reserved_users)
+
+                query_args.extend(self.reserved_users)
+                sql = base_sql + """ AND user_id NOT IN ({})""".format(
+                    ','.join(questionmarks)
+                )
+            else:
+                sql = base_sql
 
             txn.execute(sql, query_args)
 
@@ -93,16 +97,24 @@ def _reap_users(txn):
             # negative LIMIT values. So there is no way to write it that both can
             # support
             query_args = [self.hs.config.max_mau_value]
-            query_args.extend(self.reserved_users)
-            sql = """
+
+            base_sql = """
                 DELETE FROM monthly_active_users
                 WHERE user_id NOT IN (
                     SELECT user_id FROM monthly_active_users
                     ORDER BY timestamp DESC
                     LIMIT ?
                     )
-                AND user_id NOT IN ({})
-                """.format(','.join(questionmarks))
+                """
+            # Need if/else since 'AND user_id NOT IN ({})' fails on Postgres
+            # when len(reserved_users) == 0. Works fine on sqlite.
+            if len(self.reserved_users) > 0:
+                query_args.extend(self.reserved_users)
+                sql = base_sql + """ AND user_id NOT IN ({})""".format(
+                    ','.join(questionmarks)
+                )
+            else:
+                sql = base_sql
             txn.execute(sql, query_args)
 
         yield self.runInteraction("reap_monthly_active_users", _reap_users)

From ac205a54b22478726cbab4427a3370a58737d71f Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Sat, 11 Aug 2018 22:50:09 +0100
Subject: [PATCH 346/830] Fixes test_reap_monthly_active_users so it passes
 under postgres

---
 changelog.d/3681.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3681.bugfix

diff --git a/changelog.d/3681.bugfix b/changelog.d/3681.bugfix
new file mode 100644
index 000000000000..d18a69cd0c58
--- /dev/null
+++ b/changelog.d/3681.bugfix
@@ -0,0 +1 @@
+Fixes test_reap_monthly_active_users so it passes under postgres

From 53ace904b2b8e2444bc518b2498947c869c8c13b Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Tue, 5 Dec 2017 01:29:25 +0000
Subject: [PATCH 347/830] total WIP skeleton for /room_keys API

---
 synapse/handlers/e2e_room_keys.py             |  60 ++++++++
 synapse/rest/client/v2_alpha/room_keys.py     |  56 ++++++++
 synapse/storage/e2e_room_keys.py              | 133 ++++++++++++++++++
 .../storage/schema/delta/46/e2e_room_keys.sql |  40 ++++++
 4 files changed, 289 insertions(+)
 create mode 100644 synapse/handlers/e2e_room_keys.py
 create mode 100644 synapse/rest/client/v2_alpha/room_keys.py
 create mode 100644 synapse/storage/e2e_room_keys.py
 create mode 100644 synapse/storage/schema/delta/46/e2e_room_keys.sql

diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py
new file mode 100644
index 000000000000..78c838a82950
--- /dev/null
+++ b/synapse/handlers/e2e_room_keys.py
@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+# Copyright 2017 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import ujson as json
+import logging
+
+from canonicaljson import encode_canonical_json
+from twisted.internet import defer
+
+from synapse.api.errors import SynapseError, CodeMessageException
+from synapse.types import get_domain_from_id
+from synapse.util.logcontext import preserve_fn, make_deferred_yieldable
+from synapse.util.retryutils import NotRetryingDestination
+
+logger = logging.getLogger(__name__)
+
+
+class E2eRoomKeysHandler(object):
+    def __init__(self, hs):
+        self.store = hs.get_datastore()
+
+    @defer.inlineCallbacks
+    def get_room_keys(self, user_id, version, room_id, session_id):
+        results = yield self.store.get_e2e_room_keys(user_id, version, room_id, session_id)
+        defer.returnValue(results)
+
+    @defer.inlineCallbacks
+    def upload_room_keys(self, user_id, version, room_keys):
+
+        # TODO: Validate the JSON to make sure it has the right keys.
+
+        # go through the room_keys
+        for room_id in room_keys['rooms']:
+            for session_id in room_keys['rooms'][room_id]['sessions']:
+                session = room_keys['rooms'][room_id]['sessions'][session_id]
+
+                # get a lock
+
+                # get the room_key for this particular row
+                yield self.store.get_e2e_room_key()
+
+                # check whether we merge or not
+                if()
+
+                # if so, we set it
+                yield self.store.set_e2e_room_key()
+
+                # release the lock
diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py
new file mode 100644
index 000000000000..9b93001919d0
--- /dev/null
+++ b/synapse/rest/client/v2_alpha/room_keys.py
@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+# Copyright 2017 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from twisted.internet import defer
+
+from synapse.api.errors import SynapseError
+from synapse.http.servlet import (
+    RestServlet, parse_json_object_from_request, parse_integer
+)
+from synapse.http.servlet import parse_string
+from synapse.types import StreamToken
+from ._base import client_v2_patterns
+
+logger = logging.getLogger(__name__)
+
+
+class RoomKeysUploadServlet(RestServlet):
+    PATTERNS = client_v2_patterns("/room_keys/keys(/(?P<room_id>[^/]+))?(/(?P<session_id>[^/]+))?$")
+
+    def __init__(self, hs):
+        """
+        Args:
+            hs (synapse.server.HomeServer): server
+        """
+        super(RoomKeysUploadServlet, self).__init__()
+        self.auth = hs.get_auth()
+        self.e2e_room_keys_handler = hs.get_e2e_room_keys_handler()
+
+    @defer.inlineCallbacks
+    def on_POST(self, request, room_id, session_id):
+        requester = yield self.auth.get_user_by_req(request, allow_guest=True)
+        user_id = requester.user.to_string()
+        body = parse_json_object_from_request(request)
+
+        result = yield self.e2e_room_keys_handler.upload_room_keys(
+            user_id, version, body
+        )
+        defer.returnValue((200, result))
+
+
+def register_servlets(hs, http_server):
+    RoomKeysUploadServlet(hs).register(http_server)
diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py
new file mode 100644
index 000000000000..9f6d47e1b63b
--- /dev/null
+++ b/synapse/storage/e2e_room_keys.py
@@ -0,0 +1,133 @@
+# -*- coding: utf-8 -*-
+# Copyright 2017 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from twisted.internet import defer
+
+from synapse.util.caches.descriptors import cached
+
+from canonicaljson import encode_canonical_json
+import ujson as json
+
+from ._base import SQLBaseStore
+
+
+class EndToEndRoomKeyStore(SQLBaseStore):
+
+    @defer.inlineCallbacks
+    def get_e2e_room_key(self, user_id, version, room_id, session_id):
+
+        row = yield self._simple_select_one(
+            table="e2e_room_keys",
+            keyvalues={
+                "user_id": user_id,
+                "version": version,
+                "room_id": room_id,
+                "session_id": session_id,
+            },
+            retcols=(
+                "first_message_index",
+                "forwarded_count",
+                "is_verified",
+                "session_data",
+            ),
+            desc="get_e2e_room_key",
+        )
+
+        defer.returnValue(row);
+
+    def set_e2e_room_key(self, user_id, version, room_id, session_id, room_key):
+
+        def _set_e2e_room_key_txn(txn):
+
+            self._simple_upsert(
+                txn,
+                table="e2e_room_keys",
+                keyvalues={
+                    "user_id": user_id,
+                    "room_id": room_id,
+                    "session_id": session_id,   
+                }
+                values=[
+                    {
+                        "version": version,
+                        "first_message_index": room_key['first_message_index'],
+                        "forwarded_count": room_key['forwarded_count'],
+                        "is_verified": room_key['is_verified'],
+                        "session_data": room_key['session_data'],
+                    }
+                ],
+                lock=False,
+            )
+
+            return True
+
+        return self.runInteraction(
+            "set_e2e_room_key", _set_e2e_room_key_txn
+        )
+
+
+    def set_e2e_room_keys(self, user_id, version, room_keys):
+
+        def _set_e2e_room_keys_txn(txn):
+
+            self._simple_insert_many_txn(
+                txn,
+                table="e2e_room_keys",
+                values=[
+                    {
+                        "user_id": user_id,
+                        "room_id": room_id,
+                        "session_id": session_id,
+                        "version": version,
+                        "first_message_index": room_keys['rooms'][room_id]['sessions'][session_id]['first_message_index'],
+                        "forwarded_count": room_keys['rooms'][room_id]['sessions'][session_id]['forwarded_count'],
+                        "is_verified": room_keys['rooms'][room_id]['sessions'][session_id]['is_verified'],
+                        "session_data": room_keys['rooms'][room_id]['sessions'][session_id]['session_data'],
+                    }
+                    for session_id in room_keys['rooms'][room_id]['sessions']
+                    for room_id in room_keys['rooms']
+                ]
+            )
+
+            return True
+
+        return self.runInteraction(
+            "set_e2e_room_keys", _set_e2e_room_keys_txn
+        )
+
+    @defer.inlineCallbacks
+    def get_e2e_room_keys(self, user_id, version, room_id, session_id):
+
+        keyvalues={
+            "user_id": user_id,
+            "version": version,
+        }
+        if room_id: keyvalues['room_id'] = room_id
+        if session_id: keyvalues['session_id'] = session_id
+
+        rows = yield self._simple_select_list(
+            table="e2e_room_keys",
+            keyvalues=keyvalues,
+            retcols=(
+                "first_message_index",
+                "forwarded_count",
+                "is_verified",
+                "session_data",
+            ),
+            desc="get_e2e_room_keys",
+        )
+
+        sessions = {}
+        sessions['rooms'][roomId]['sessions'][session_id] = row for row in rows;
+        defer.returnValue(sessions);
diff --git a/synapse/storage/schema/delta/46/e2e_room_keys.sql b/synapse/storage/schema/delta/46/e2e_room_keys.sql
new file mode 100644
index 000000000000..51b826e8b3df
--- /dev/null
+++ b/synapse/storage/schema/delta/46/e2e_room_keys.sql
@@ -0,0 +1,40 @@
+/* Copyright 2017 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- users' optionally backed up encrypted e2e sessions
+CREATE TABLE e2e_room_keys (
+    user_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    session_id TEXT NOT NULL,
+    version INT NOT NULL,
+    first_message_index INT,
+    forwarded_count INT,
+    is_verified BOOLEAN,
+    session_data TEXT NOT NULL
+);
+
+CREATE UNIQUE INDEX e2e_room_keys_user_idx ON e2e_room_keys(user_id);
+CREATE UNIQUE INDEX e2e_room_keys_room_idx ON e2e_room_keys(room_id);
+CREATE UNIQUE INDEX e2e_room_keys_session_idx ON e2e_room_keys(session_id);
+
+-- the versioning metadata about versions of users' encrypted e2e session backups
+CREATE TABLE e2e_room_key_versions (
+    user_id TEXT NOT NULL,
+    version INT NOT NULL,
+    algorithm TEXT NOT NULL,
+    dummy_session_data TEXT NOT NULL
+);
+
+CREATE UNIQUE INDEX e2e_room_key_user_idx ON e2e_room_keys(user_id);

From 0bc4627a731d0edd437905a5b07e85421c7553d8 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Tue, 5 Dec 2017 17:54:48 +0000
Subject: [PATCH 348/830] interim WIP checkin; doesn't build yet

---
 synapse/handlers/e2e_room_keys.py         | 46 +++++++++++++++--------
 synapse/rest/client/v2_alpha/room_keys.py | 37 ++++++++++++++++--
 synapse/storage/e2e_room_keys.py          | 20 ++++++++++
 3 files changed, 84 insertions(+), 19 deletions(-)

diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py
index 78c838a82950..15e3beb5ed62 100644
--- a/synapse/handlers/e2e_room_keys.py
+++ b/synapse/handlers/e2e_room_keys.py
@@ -20,8 +20,7 @@
 from twisted.internet import defer
 
 from synapse.api.errors import SynapseError, CodeMessageException
-from synapse.types import get_domain_from_id
-from synapse.util.logcontext import preserve_fn, make_deferred_yieldable
+from synapse.util.async import Linearizer
 from synapse.util.retryutils import NotRetryingDestination
 
 logger = logging.getLogger(__name__)
@@ -30,31 +29,48 @@
 class E2eRoomKeysHandler(object):
     def __init__(self, hs):
         self.store = hs.get_datastore()
+        self._upload_linearizer = async.Linearizer("upload_room_keys_lock")
 
     @defer.inlineCallbacks
     def get_room_keys(self, user_id, version, room_id, session_id):
         results = yield self.store.get_e2e_room_keys(user_id, version, room_id, session_id)
         defer.returnValue(results)
 
+    @defer.inlineCallbacks
+    def delete_room_keys(self, user_id, version, room_id, session_id):
+        yield self.store.delete_e2e_room_keys(user_id, version, room_id, session_id)
+
     @defer.inlineCallbacks
     def upload_room_keys(self, user_id, version, room_keys):
 
         # TODO: Validate the JSON to make sure it has the right keys.
 
-        # go through the room_keys
-        for room_id in room_keys['rooms']:
-            for session_id in room_keys['rooms'][room_id]['sessions']:
-                session = room_keys['rooms'][room_id]['sessions'][session_id]
-
-                # get a lock
+        # XXX: perhaps we should use a finer grained lock here?
+        with (yield self._upload_linearizer.queue(user_id):
 
-                # get the room_key for this particular row
-                yield self.store.get_e2e_room_key()
+            # go through the room_keys
+            for room_id in room_keys['rooms']:
+                for session_id in room_keys['rooms'][room_id]['sessions']:
+                    room_key = room_keys['rooms'][room_id]['sessions'][session_id]
 
-                # check whether we merge or not
-                if()
+                    # get the room_key for this particular row
+                    current_room_key = yield self.store.get_e2e_room_key(
+                        user_id, version, room_id, session_id
+                    )
 
-                # if so, we set it
-                yield self.store.set_e2e_room_key()
+                    # check whether we merge or not. spelling it out with if/elifs rather than
+                    # lots of booleans for legibility.
+                    replace = False
+                    if current_room_key:
+                        if room_key['is_verified'] and not current_room_key['is_verified']:
+                            replace = True
+                        elif room_key['first_message_index'] < current_room_key['first_message_index']:
+                            replace = True
+                        elif room_key['forwarded_count'] < room_key['forwarded_count']:
+                            replace = True
 
-                # release the lock
+                    # if so, we set the new room_key
+                    if replace:
+                        yield self.store.set_e2e_room_key(
+                            user_id, version, room_id, session_id, room_key
+                        )
diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py
index 9b93001919d0..7291018a480a 100644
--- a/synapse/rest/client/v2_alpha/room_keys.py
+++ b/synapse/rest/client/v2_alpha/room_keys.py
@@ -28,7 +28,7 @@
 logger = logging.getLogger(__name__)
 
 
-class RoomKeysUploadServlet(RestServlet):
+class RoomKeysServlet(RestServlet):
     PATTERNS = client_v2_patterns("/room_keys/keys(/(?P<room_id>[^/]+))?(/(?P<session_id>[^/]+))?$")
 
     def __init__(self, hs):
@@ -41,16 +41,45 @@ def __init__(self, hs):
         self.e2e_room_keys_handler = hs.get_e2e_room_keys_handler()
 
     @defer.inlineCallbacks
-    def on_POST(self, request, room_id, session_id):
-        requester = yield self.auth.get_user_by_req(request, allow_guest=True)
+    def on_PUT(self, request, room_id, session_id):
+        requester = yield self.auth.get_user_by_req(request, allow_guest=False)
         user_id = requester.user.to_string()
         body = parse_json_object_from_request(request)
+        version = request.args.get("version", None)
+
+        if session_id:
+            body = { "sessions": { session_id : body } }
+
+        if room_id:
+            body = { "rooms": { room_id : body } }
 
         result = yield self.e2e_room_keys_handler.upload_room_keys(
             user_id, version, body
         )
         defer.returnValue((200, result))
 
+    @defer.inlineCallbacks
+    def on_GET(self, request, room_id, session_id):
+        requester = yield self.auth.get_user_by_req(request, allow_guest=False)
+        user_id = requester.user.to_string()
+        version = request.args.get("version", None)
+
+        room_keys = yield self.e2e_room_keys_handler.get_room_keys(
+            user_id, version, room_id, session_id
+        )
+        defer.returnValue((200, room_keys))
+
+    @defer.inlineCallbacks
+    def on_DELETE(self, request, room_id, session_id):
+        requester = yield self.auth.get_user_by_req(request, allow_guest=False)
+        user_id = requester.user.to_string()
+        version = request.args.get("version", None)
+
+        yield self.e2e_room_keys_handler.delete_room_keys(
+            user_id, version, room_id, session_id
+        )
+        defer.returnValue((200, {}))
+
 
 def register_servlets(hs, http_server):
-    RoomKeysUploadServlet(hs).register(http_server)
+    RoomKeysServlet(hs).register(http_server)
diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py
index 9f6d47e1b63b..903dc083f84a 100644
--- a/synapse/storage/e2e_room_keys.py
+++ b/synapse/storage/e2e_room_keys.py
@@ -12,6 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 from twisted.internet import defer
 
 from synapse.util.caches.descriptors import cached
@@ -77,6 +78,9 @@ def _set_e2e_room_key_txn(txn):
         )
 
 
+    # XXX: this isn't currently used and isn't tested anywhere
+    # it could be used in future for bulk-uploading new versions of room_keys
+    # for a user or something though.
     def set_e2e_room_keys(self, user_id, version, room_keys):
 
         def _set_e2e_room_keys_txn(txn):
@@ -131,3 +135,19 @@ def get_e2e_room_keys(self, user_id, version, room_id, session_id):
         sessions = {}
         sessions['rooms'][roomId]['sessions'][session_id] = row for row in rows;
         defer.returnValue(sessions);
+
+    @defer.inlineCallbacks
+    def delete_e2e_room_keys(self, user_id, version, room_id, session_id):
+
+        keyvalues={
+            "user_id": user_id,
+            "version": version,
+        }
+        if room_id: keyvalues['room_id'] = room_id
+        if session_id: keyvalues['session_id'] = session_id
+
+        yield self._simple_delete(
+            table="e2e_room_keys",
+            keyvalues=keyvalues,
+            desc="delete_e2e_room_keys",
+        )

From 6b8c07abc293bd222051e769550753bc0fd6f667 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Tue, 5 Dec 2017 21:44:25 +0000
Subject: [PATCH 349/830] make it work and fix pep8

---
 synapse/handlers/e2e_room_keys.py             |  69 +++++++-----
 synapse/rest/__init__.py                      |   2 +
 synapse/rest/client/v2_alpha/room_keys.py     |  33 +++---
 synapse/server.py                             |   5 +
 synapse/storage/__init__.py                   |   2 +
 synapse/storage/e2e_room_keys.py              | 103 +++++++++++-------
 .../storage/schema/delta/46/e2e_room_keys.sql |   2 +-
 7 files changed, 134 insertions(+), 82 deletions(-)

diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py
index 15e3beb5ed62..93f4ad519481 100644
--- a/synapse/handlers/e2e_room_keys.py
+++ b/synapse/handlers/e2e_room_keys.py
@@ -13,15 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import ujson as json
 import logging
 
-from canonicaljson import encode_canonical_json
 from twisted.internet import defer
 
-from synapse.api.errors import SynapseError, CodeMessageException
+from synapse.api.errors import StoreError
 from synapse.util.async import Linearizer
-from synapse.util.retryutils import NotRetryingDestination
 
 logger = logging.getLogger(__name__)
 
@@ -29,11 +26,13 @@
 class E2eRoomKeysHandler(object):
     def __init__(self, hs):
         self.store = hs.get_datastore()
-        self._upload_linearizer = async.Linearizer("upload_room_keys_lock")
+        self._upload_linearizer = Linearizer("upload_room_keys_lock")
 
     @defer.inlineCallbacks
     def get_room_keys(self, user_id, version, room_id, session_id):
-        results = yield self.store.get_e2e_room_keys(user_id, version, room_id, session_id)
+        results = yield self.store.get_e2e_room_keys(
+            user_id, version, room_id, session_id
+        )
         defer.returnValue(results)
 
     @defer.inlineCallbacks
@@ -46,31 +45,49 @@ def upload_room_keys(self, user_id, version, room_keys):
         # TODO: Validate the JSON to make sure it has the right keys.
 
         # XXX: perhaps we should use a finer grained lock here?
-        with (yield self._upload_linearizer.queue(user_id):
+        with (yield self._upload_linearizer.queue(user_id)):
 
             # go through the room_keys
             for room_id in room_keys['rooms']:
                 for session_id in room_keys['rooms'][room_id]['sessions']:
                     room_key = room_keys['rooms'][room_id]['sessions'][session_id]
 
-                    # get the room_key for this particular row
-                    current_room_key = yield self.store.get_e2e_room_key(
-                        user_id, version, room_id, session_id
+                    yield self._upload_room_key(
+                        user_id, version, room_id, session_id, room_key
                     )
 
-                    # check whether we merge or not. spelling it out with if/elifs rather than
-                    # lots of booleans for legibility.
-                    replace = False
-                    if current_room_key:
-                        if room_key['is_verified'] and not current_room_key['is_verified']:
-                            replace = True
-                        elif room_key['first_message_index'] < current_room_key['first_message_index']:
-                            replace = True
-                        elif room_key['forwarded_count'] < room_key['forwarded_count']:
-                            replace = True
-
-                    # if so, we set the new room_key
-                    if replace:
-                        yield self.store.set_e2e_room_key(
-                            user_id, version, room_id, session_id, room_key
-                        )
+    @defer.inlineCallbacks
+    def _upload_room_key(self, user_id, version, room_id, session_id, room_key):
+        # get the room_key for this particular row
+        current_room_key = None
+        try:
+            current_room_key = yield self.store.get_e2e_room_key(
+                user_id, version, room_id, session_id
+            )
+        except StoreError as e:
+            if e.code == 404:
+                pass
+            else:
+                raise
+
+        # check whether we merge or not. spelling it out with if/elifs rather
+        # than lots of booleans for legibility.
+        upsert = True
+        if current_room_key:
+            if room_key['is_verified'] and not current_room_key['is_verified']:
+                pass
+            elif (
+                room_key['first_message_index'] <
+                current_room_key['first_message_index']
+            ):
+                pass
+            elif room_key['forwarded_count'] < room_key['forwarded_count']:
+                pass
+            else:
+                upsert = False
+
+        # if so, we set the new room_key
+        if upsert:
+            yield self.store.set_e2e_room_key(
+                user_id, version, room_id, session_id, room_key
+            )
diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py
index 3418f06fd6e5..4856822a5da5 100644
--- a/synapse/rest/__init__.py
+++ b/synapse/rest/__init__.py
@@ -46,6 +46,7 @@
     receipts,
     register,
     report_event,
+    room_keys,
     sendtodevice,
     sync,
     tags,
@@ -102,6 +103,7 @@ def register_servlets(client_resource, hs):
         auth.register_servlets(hs, client_resource)
         receipts.register_servlets(hs, client_resource)
         read_marker.register_servlets(hs, client_resource)
+        room_keys.register_servlets(hs, client_resource)
         keys.register_servlets(hs, client_resource)
         tokenrefresh.register_servlets(hs, client_resource)
         tags.register_servlets(hs, client_resource)
diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py
index 7291018a480a..010aed98f924 100644
--- a/synapse/rest/client/v2_alpha/room_keys.py
+++ b/synapse/rest/client/v2_alpha/room_keys.py
@@ -17,26 +17,25 @@
 
 from twisted.internet import defer
 
-from synapse.api.errors import SynapseError
 from synapse.http.servlet import (
-    RestServlet, parse_json_object_from_request, parse_integer
+    RestServlet, parse_json_object_from_request
 )
-from synapse.http.servlet import parse_string
-from synapse.types import StreamToken
 from ._base import client_v2_patterns
 
 logger = logging.getLogger(__name__)
 
 
 class RoomKeysServlet(RestServlet):
-    PATTERNS = client_v2_patterns("/room_keys/keys(/(?P<room_id>[^/]+))?(/(?P<session_id>[^/]+))?$")
+    PATTERNS = client_v2_patterns(
+        "/room_keys/keys(/(?P<room_id>[^/]+))?(/(?P<session_id>[^/]+))?$"
+    )
 
     def __init__(self, hs):
         """
         Args:
             hs (synapse.server.HomeServer): server
         """
-        super(RoomKeysUploadServlet, self).__init__()
+        super(RoomKeysServlet, self).__init__()
         self.auth = hs.get_auth()
         self.e2e_room_keys_handler = hs.get_e2e_room_keys_handler()
 
@@ -45,24 +44,32 @@ def on_PUT(self, request, room_id, session_id):
         requester = yield self.auth.get_user_by_req(request, allow_guest=False)
         user_id = requester.user.to_string()
         body = parse_json_object_from_request(request)
-        version = request.args.get("version", None)
+        version = request.args.get("version")[0]
 
         if session_id:
-            body = { "sessions": { session_id : body } }
+            body = {
+                "sessions": {
+                    session_id: body
+                }
+            }
 
         if room_id:
-            body = { "rooms": { room_id : body } }
+            body = {
+                "rooms": {
+                    room_id: body
+                }
+            }
 
-        result = yield self.e2e_room_keys_handler.upload_room_keys(
+        yield self.e2e_room_keys_handler.upload_room_keys(
             user_id, version, body
         )
-        defer.returnValue((200, result))
+        defer.returnValue((200, {}))
 
     @defer.inlineCallbacks
     def on_GET(self, request, room_id, session_id):
         requester = yield self.auth.get_user_by_req(request, allow_guest=False)
         user_id = requester.user.to_string()
-        version = request.args.get("version", None)
+        version = request.args.get("version")[0]
 
         room_keys = yield self.e2e_room_keys_handler.get_room_keys(
             user_id, version, room_id, session_id
@@ -73,7 +80,7 @@ def on_GET(self, request, room_id, session_id):
     def on_DELETE(self, request, room_id, session_id):
         requester = yield self.auth.get_user_by_req(request, allow_guest=False)
         user_id = requester.user.to_string()
-        version = request.args.get("version", None)
+        version = request.args.get("version")[0]
 
         yield self.e2e_room_keys_handler.delete_room_keys(
             user_id, version, room_id, session_id
diff --git a/synapse/server.py b/synapse/server.py
index 140be9ebe863..706cb1361f59 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -49,6 +49,7 @@
 from synapse.handlers.device import DeviceHandler
 from synapse.handlers.devicemessage import DeviceMessageHandler
 from synapse.handlers.e2e_keys import E2eKeysHandler
+from synapse.handlers.e2e_room_keys import E2eRoomKeysHandler
 from synapse.handlers.events import EventHandler, EventStreamHandler
 from synapse.handlers.groups_local import GroupsLocalHandler
 from synapse.handlers.initial_sync import InitialSyncHandler
@@ -127,6 +128,7 @@ def build_DEPENDENCY(self)
         'auth_handler',
         'device_handler',
         'e2e_keys_handler',
+        'e2e_room_keys_handler',
         'event_handler',
         'event_stream_handler',
         'initial_sync_handler',
@@ -288,6 +290,9 @@ def build_device_message_handler(self):
     def build_e2e_keys_handler(self):
         return E2eKeysHandler(self)
 
+    def build_e2e_room_keys_handler(self):
+        return E2eRoomKeysHandler(self)
+
     def build_application_service_api(self):
         return ApplicationServiceApi(self)
 
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 134e4a80f1f0..69cb28268ac0 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -30,6 +30,7 @@
 from .client_ips import ClientIpStore
 from .deviceinbox import DeviceInboxStore
 from .directory import DirectoryStore
+from .e2e_room_keys import EndToEndRoomKeyStore
 from .end_to_end_keys import EndToEndKeyStore
 from .engines import PostgresEngine
 from .event_federation import EventFederationStore
@@ -76,6 +77,7 @@ class DataStore(RoomMemberStore, RoomStore,
                 ApplicationServiceTransactionStore,
                 ReceiptsStore,
                 EndToEndKeyStore,
+                EndToEndRoomKeyStore,
                 SearchStore,
                 TagsStore,
                 AccountDataStore,
diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py
index 903dc083f84a..5982710bd574 100644
--- a/synapse/storage/e2e_room_keys.py
+++ b/synapse/storage/e2e_room_keys.py
@@ -15,11 +15,6 @@
 
 from twisted.internet import defer
 
-from synapse.util.caches.descriptors import cached
-
-from canonicaljson import encode_canonical_json
-import ujson as json
-
 from ._base import SQLBaseStore
 
 
@@ -45,29 +40,27 @@ def get_e2e_room_key(self, user_id, version, room_id, session_id):
             desc="get_e2e_room_key",
         )
 
-        defer.returnValue(row);
+        defer.returnValue(row)
 
     def set_e2e_room_key(self, user_id, version, room_id, session_id, room_key):
 
         def _set_e2e_room_key_txn(txn):
 
-            self._simple_upsert(
+            self._simple_upsert_txn(
                 txn,
                 table="e2e_room_keys",
                 keyvalues={
                     "user_id": user_id,
                     "room_id": room_id,
-                    "session_id": session_id,   
-                }
-                values=[
-                    {
-                        "version": version,
-                        "first_message_index": room_key['first_message_index'],
-                        "forwarded_count": room_key['forwarded_count'],
-                        "is_verified": room_key['is_verified'],
-                        "session_data": room_key['session_data'],
-                    }
-                ],
+                    "session_id": session_id,
+                },
+                values={
+                    "version": version,
+                    "first_message_index": room_key['first_message_index'],
+                    "forwarded_count": room_key['forwarded_count'],
+                    "is_verified": room_key['is_verified'],
+                    "session_data": room_key['session_data'],
+                },
                 lock=False,
             )
 
@@ -77,7 +70,6 @@ def _set_e2e_room_key_txn(txn):
             "set_e2e_room_key", _set_e2e_room_key_txn
         )
 
-
     # XXX: this isn't currently used and isn't tested anywhere
     # it could be used in future for bulk-uploading new versions of room_keys
     # for a user or something though.
@@ -85,23 +77,27 @@ def set_e2e_room_keys(self, user_id, version, room_keys):
 
         def _set_e2e_room_keys_txn(txn):
 
+            values = []
+            for room_id in room_keys['rooms']:
+                for session_id in room_keys['rooms'][room_id]['sessions']:
+                    session = room_keys['rooms'][room_id]['sessions'][session_id]
+                    values.append(
+                        {
+                            "user_id": user_id,
+                            "room_id": room_id,
+                            "session_id": session_id,
+                            "version": version,
+                            "first_message_index": session['first_message_index'],
+                            "forwarded_count": session['forwarded_count'],
+                            "is_verified": session['is_verified'],
+                            "session_data": session['session_data'],
+                        }
+                    )
+
             self._simple_insert_many_txn(
                 txn,
                 table="e2e_room_keys",
-                values=[
-                    {
-                        "user_id": user_id,
-                        "room_id": room_id,
-                        "session_id": session_id,
-                        "version": version,
-                        "first_message_index": room_keys['rooms'][room_id]['sessions'][session_id]['first_message_index'],
-                        "forwarded_count": room_keys['rooms'][room_id]['sessions'][session_id]['forwarded_count'],
-                        "is_verified": room_keys['rooms'][room_id]['sessions'][session_id]['is_verified'],
-                        "session_data": room_keys['rooms'][room_id]['sessions'][session_id]['session_data'],
-                    }
-                    for session_id in room_keys['rooms'][room_id]['sessions']
-                    for room_id in room_keys['rooms']
-                ]
+                values=values
             )
 
             return True
@@ -113,17 +109,22 @@ def _set_e2e_room_keys_txn(txn):
     @defer.inlineCallbacks
     def get_e2e_room_keys(self, user_id, version, room_id, session_id):
 
-        keyvalues={
+        keyvalues = {
             "user_id": user_id,
             "version": version,
         }
-        if room_id: keyvalues['room_id'] = room_id
-        if session_id: keyvalues['session_id'] = session_id
+        if room_id:
+            keyvalues['room_id'] = room_id
+        if session_id:
+            keyvalues['session_id'] = session_id
 
         rows = yield self._simple_select_list(
             table="e2e_room_keys",
             keyvalues=keyvalues,
             retcols=(
+                "user_id",
+                "room_id",
+                "session_id",
                 "first_message_index",
                 "forwarded_count",
                 "is_verified",
@@ -132,19 +133,37 @@ def get_e2e_room_keys(self, user_id, version, room_id, session_id):
             desc="get_e2e_room_keys",
         )
 
-        sessions = {}
-        sessions['rooms'][roomId]['sessions'][session_id] = row for row in rows;
-        defer.returnValue(sessions);
+        # perlesque autovivification from https://stackoverflow.com/a/19829714/6764493
+        class AutoVivification(dict):
+            def __getitem__(self, item):
+                try:
+                    return dict.__getitem__(self, item)
+                except KeyError:
+                    value = self[item] = type(self)()
+                    return value
+
+        sessions = AutoVivification()
+        for row in rows:
+            sessions['rooms'][row['room_id']]['sessions'][row['session_id']] = {
+                "first_message_index": row["first_message_index"],
+                "forwarded_count": row["forwarded_count"],
+                "is_verified": row["is_verified"],
+                "session_data": row["session_data"],
+            }
+
+        defer.returnValue(sessions)
 
     @defer.inlineCallbacks
     def delete_e2e_room_keys(self, user_id, version, room_id, session_id):
 
-        keyvalues={
+        keyvalues = {
             "user_id": user_id,
             "version": version,
         }
-        if room_id: keyvalues['room_id'] = room_id
-        if session_id: keyvalues['session_id'] = session_id
+        if room_id:
+            keyvalues['room_id'] = room_id
+        if session_id:
+            keyvalues['session_id'] = session_id
 
         yield self._simple_delete(
             table="e2e_room_keys",
diff --git a/synapse/storage/schema/delta/46/e2e_room_keys.sql b/synapse/storage/schema/delta/46/e2e_room_keys.sql
index 51b826e8b3df..6b344c5ad78d 100644
--- a/synapse/storage/schema/delta/46/e2e_room_keys.sql
+++ b/synapse/storage/schema/delta/46/e2e_room_keys.sql
@@ -29,7 +29,7 @@ CREATE UNIQUE INDEX e2e_room_keys_user_idx ON e2e_room_keys(user_id);
 CREATE UNIQUE INDEX e2e_room_keys_room_idx ON e2e_room_keys(room_id);
 CREATE UNIQUE INDEX e2e_room_keys_session_idx ON e2e_room_keys(session_id);
 
--- the versioning metadata about versions of users' encrypted e2e session backups
+-- the metadata for each generation of encrypted e2e session backups
 CREATE TABLE e2e_room_key_versions (
     user_id TEXT NOT NULL,
     version INT NOT NULL,

From cf1e2000f623afea8f3afb58e4a7659288c45777 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Tue, 5 Dec 2017 23:06:43 +0000
Subject: [PATCH 350/830] document the API

---
 synapse/rest/client/v2_alpha/room_keys.py | 133 ++++++++++++++++++++++
 1 file changed, 133 insertions(+)

diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py
index 010aed98f924..be82eccb2bc8 100644
--- a/synapse/rest/client/v2_alpha/room_keys.py
+++ b/synapse/rest/client/v2_alpha/room_keys.py
@@ -41,6 +41,79 @@ def __init__(self, hs):
 
     @defer.inlineCallbacks
     def on_PUT(self, request, room_id, session_id):
+        """
+        Uploads one or more encrypted E2E room keys for backup purposes.
+        room_id: the ID of the room the keys are for (optional)
+        session_id: the ID for the E2E room keys for the room (optional)
+        version: the version of the user's backup which this data is for.
+        the version must already have been created via the /change_secret API.
+
+        Each session has:
+         * first_message_index: a numeric index indicating the oldest message
+           encrypted by this session.
+         * forwarded_count: how many times the uploading client claims this key
+           has been shared (forwarded)
+         * is_verified: whether the client that uploaded the keys claims they
+           were sent by a device which they've verified
+         * session_data: base64-encrypted data describing the session.
+
+        Returns 200 OK on success with body {}
+
+        The API is designed to be otherwise agnostic to the room_key encryption
+        algorithm being used.  Sessions are merged with existing ones in the
+        backup using the heuristics:
+         * is_verified sessions always win over unverified sessions
+         * older first_message_index always win over newer sessions
+         * lower forwarded_count always wins over higher forwarded_count
+
+        We trust the clients not to lie and corrupt their own backups.
+
+        POST /room_keys/keys/!abc:matrix.org/c0ff33?version=1 HTTP/1.1
+        Content-Type: application/json
+
+        {
+            "first_message_index": 1,
+            "forwarded_count": 1,
+            "is_verified": false,
+            "session_data": "SSBBTSBBIEZJU0gK"
+        }
+
+        Or...
+
+        POST /room_keys/keys/!abc:matrix.org?version=1 HTTP/1.1
+        Content-Type: application/json
+
+        {
+            "sessions": {
+                "c0ff33": {
+                    "first_message_index": 1,
+                    "forwarded_count": 1,
+                    "is_verified": false,
+                    "session_data": "SSBBTSBBIEZJU0gK"
+                }
+            }
+        }
+
+        Or...
+
+        POST /room_keys/keys?version=1 HTTP/1.1
+        Content-Type: application/json
+
+        {
+            "rooms": {
+                "!abc:matrix.org": {
+                    "sessions": {
+                        "c0ff33": {
+                            "first_message_index": 1,
+                            "forwarded_count": 1,
+                            "is_verified": false,
+                            "session_data": "SSBBTSBBIEZJU0gK"
+                        }
+                    }
+                }
+            }
+        }
+        """
         requester = yield self.auth.get_user_by_req(request, allow_guest=False)
         user_id = requester.user.to_string()
         body = parse_json_object_from_request(request)
@@ -67,6 +140,57 @@ def on_PUT(self, request, room_id, session_id):
 
     @defer.inlineCallbacks
     def on_GET(self, request, room_id, session_id):
+        """
+        Retrieves one or more encrypted E2E room keys for backup purposes.
+        Symmetric with the PUT version of the API.
+
+        room_id: the ID of the room to retrieve the keys for (optional)
+        session_id: the ID for the E2E room keys to retrieve the keys for (optional)
+        version: the version of the user's backup which this data is for.
+        the version must already have been created via the /change_secret API.
+
+        Returns as follows:
+
+        GET /room_keys/keys/!abc:matrix.org/c0ff33?version=1 HTTP/1.1
+        {
+            "first_message_index": 1,
+            "forwarded_count": 1,
+            "is_verified": false,
+            "session_data": "SSBBTSBBIEZJU0gK"
+        }
+
+        Or...
+
+        GET /room_keys/keys/!abc:matrix.org?version=1 HTTP/1.1
+        {
+            "sessions": {
+                "c0ff33": {
+                    "first_message_index": 1,
+                    "forwarded_count": 1,
+                    "is_verified": false,
+                    "session_data": "SSBBTSBBIEZJU0gK"
+                }
+            }
+        }
+
+        Or...
+
+        GET /room_keys/keys?version=1 HTTP/1.1
+        {
+            "rooms": {
+                "!abc:matrix.org": {
+                    "sessions": {
+                        "c0ff33": {
+                            "first_message_index": 1,
+                            "forwarded_count": 1,
+                            "is_verified": false,
+                            "session_data": "SSBBTSBBIEZJU0gK"
+                        }
+                    }
+                }
+            }
+        }
+        """
         requester = yield self.auth.get_user_by_req(request, allow_guest=False)
         user_id = requester.user.to_string()
         version = request.args.get("version")[0]
@@ -78,6 +202,15 @@ def on_GET(self, request, room_id, session_id):
 
     @defer.inlineCallbacks
     def on_DELETE(self, request, room_id, session_id):
+        """
+        Deletes one or more encrypted E2E room keys for a user for backup purposes.
+
+        room_id: the ID of the room whose keys to delete (optional)
+        session_id: the ID for the E2E session to delete (optional)
+        version: the version of the user's backup which this data is for.
+        the version must already have been created via the /change_secret API.
+        """
+
         requester = yield self.auth.get_user_by_req(request, allow_guest=False)
         user_id = requester.user.to_string()
         version = request.args.get("version")[0]

From 8ae64b270f7742abfe4cf0b8d140c81993464ea4 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Wed, 6 Dec 2017 01:02:57 +0000
Subject: [PATCH 351/830] implement /room_keys/version too (untested)

---
 synapse/api/errors.py                         | 25 +++++++++
 synapse/handlers/e2e_room_keys.py             | 47 ++++++++++++++--
 synapse/rest/client/v2_alpha/room_keys.py     | 47 ++++++++++++++++
 synapse/storage/e2e_room_keys.py              | 56 +++++++++++++++++++
 .../storage/schema/delta/46/e2e_room_keys.sql |  2 +-
 5 files changed, 171 insertions(+), 6 deletions(-)

diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index b41d595059a3..8c97e91ba194 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -56,6 +56,7 @@ class Codes(object):
     CONSENT_NOT_GIVEN = "M_CONSENT_NOT_GIVEN"
     CANNOT_LEAVE_SERVER_NOTICE_ROOM = "M_CANNOT_LEAVE_SERVER_NOTICE_ROOM"
     MAU_LIMIT_EXCEEDED = "M_MAU_LIMIT_EXCEEDED"
+    WRONG_ROOM_KEYS_VERSION = "M_WRONG_ROOM_KEYS_VERSION"
 
 
 class CodeMessageException(RuntimeError):
@@ -285,6 +286,30 @@ def error_dict(self):
         )
 
 
+class RoomKeysVersionError(SynapseError):
+    """A client has tried to upload to a non-current version of the room_keys store
+    """
+    def __init__(self, code=403, msg="Wrong room_keys version", current_version=None,
+                 errcode=Codes.WRONG_ROOM_KEYS_VERSION):
+        super(RoomKeysVersionError, self).__init__(code, msg, errcode)
+        self.current_version = current_version
+
+    def error_dict(self):
+        return cs_error(
+            self.msg,
+            self.errcode,
+            current_version=self.current_version,
+        )
+
+
+def cs_exception(exception):
+    if isinstance(exception, CodeMessageException):
+        return exception.error_dict()
+    else:
+        logger.error("Unknown exception type: %s", type(exception))
+        return {}
+
+
 def cs_error(msg, code=Codes.UNKNOWN, **kwargs):
     """ Utility method for constructing an error response for client-server
     interactions.
diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py
index 93f4ad519481..4333ca610cbe 100644
--- a/synapse/handlers/e2e_room_keys.py
+++ b/synapse/handlers/e2e_room_keys.py
@@ -17,7 +17,7 @@
 
 from twisted.internet import defer
 
-from synapse.api.errors import StoreError
+from synapse.api.errors import StoreError, SynapseError, RoomKeysVersionError
 from synapse.util.async import Linearizer
 
 logger = logging.getLogger(__name__)
@@ -30,10 +30,13 @@ def __init__(self, hs):
 
     @defer.inlineCallbacks
     def get_room_keys(self, user_id, version, room_id, session_id):
-        results = yield self.store.get_e2e_room_keys(
-            user_id, version, room_id, session_id
-        )
-        defer.returnValue(results)
+        # we deliberately take the lock to get keys so that changing the version
+        # works atomically
+        with (yield self._upload_linearizer.queue(user_id)):
+            results = yield self.store.get_e2e_room_keys(
+                user_id, version, room_id, session_id
+            )
+            defer.returnValue(results)
 
     @defer.inlineCallbacks
     def delete_room_keys(self, user_id, version, room_id, session_id):
@@ -44,6 +47,16 @@ def upload_room_keys(self, user_id, version, room_keys):
 
         # TODO: Validate the JSON to make sure it has the right keys.
 
+        # Check that the version we're trying to upload is the current version
+        try:
+            version_info = yield self.get_version_info(user_id, version)
+        except StoreError as e:
+            if e.code == 404:
+                raise SynapseError(404, "Version '%d' not found" % (version,))
+
+        if version_info.version != version:
+            raise RoomKeysVersionError(current_version=version_info.version)
+
         # XXX: perhaps we should use a finer grained lock here?
         with (yield self._upload_linearizer.queue(user_id)):
 
@@ -91,3 +104,27 @@ def _upload_room_key(self, user_id, version, room_id, session_id, room_key):
             yield self.store.set_e2e_room_key(
                 user_id, version, room_id, session_id, room_key
             )
+
+    @defer.inlineCallbacks
+    def create_version(self, user_id, version, version_info):
+
+        # TODO: Validate the JSON to make sure it has the right keys.
+
+        # lock everyone out until we've switched version
+        with (yield self._upload_linearizer.queue(user_id)):
+            yield self.store.create_version(
+                user_id, version, version_info
+            )
+
+    @defer.inlineCallbacks
+    def get_version_info(self, user_id, version):
+        with (yield self._upload_linearizer.queue(user_id)):
+            results = yield self.store.get_e2e_room_key_version(
+                user_id, version
+            )
+            defer.returnValue(results)
+
+    @defer.inlineCallbacks
+    def delete_version(self, user_id, version):
+        with (yield self._upload_linearizer.queue(user_id)):
+            yield self.store.delete_e2e_room_key_version(user_id, version)
diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py
index be82eccb2bc8..4d76e1d824dc 100644
--- a/synapse/rest/client/v2_alpha/room_keys.py
+++ b/synapse/rest/client/v2_alpha/room_keys.py
@@ -221,5 +221,52 @@ def on_DELETE(self, request, room_id, session_id):
         defer.returnValue((200, {}))
 
 
+class RoomKeysVersionServlet(RestServlet):
+    PATTERNS = client_v2_patterns(
+        "/room_keys/version(/(?P<version>[^/]+))?$"
+    )
+
+    def __init__(self, hs):
+        """
+        Args:
+            hs (synapse.server.HomeServer): server
+        """
+        super(RoomKeysVersionServlet, self).__init__()
+        self.auth = hs.get_auth()
+        self.e2e_room_keys_handler = hs.get_e2e_room_keys_handler()
+
+    @defer.inlineCallbacks
+    def on_POST(self, request, version):
+        requester = yield self.auth.get_user_by_req(request, allow_guest=False)
+        user_id = requester.user.to_string()
+        info = parse_json_object_from_request(request)
+
+        new_version = yield self.e2e_room_keys_handler.create_version(
+            user_id, version, info
+        )
+        defer.returnValue((200, {"version": new_version}))
+
+    @defer.inlineCallbacks
+    def on_GET(self, request, version):
+        requester = yield self.auth.get_user_by_req(request, allow_guest=False)
+        user_id = requester.user.to_string()
+
+        info = yield self.e2e_room_keys_handler.get_version_info(
+            user_id, version
+        )
+        defer.returnValue((200, info))
+
+    @defer.inlineCallbacks
+    def on_DELETE(self, request, version):
+        requester = yield self.auth.get_user_by_req(request, allow_guest=False)
+        user_id = requester.user.to_string()
+
+        yield self.e2e_room_keys_handler.delete_version(
+            user_id, version
+        )
+        defer.returnValue((200, {}))
+
+
 def register_servlets(hs, http_server):
     RoomKeysServlet(hs).register(http_server)
+    RoomKeysVersionServlet(hs).register(http_server)
diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py
index 5982710bd574..994878acf684 100644
--- a/synapse/storage/e2e_room_keys.py
+++ b/synapse/storage/e2e_room_keys.py
@@ -170,3 +170,59 @@ def delete_e2e_room_keys(self, user_id, version, room_id, session_id):
             keyvalues=keyvalues,
             desc="delete_e2e_room_keys",
         )
+
+    @defer.inlineCallbacks
+    def get_e2e_room_key_version(self, user_id, version):
+
+        row = yield self._simple_select_one(
+            table="e2e_room_key_versions",
+            keyvalues={
+                "user_id": user_id,
+                "version": version,
+            },
+            retcols=(
+                "user_id",
+                "version",
+                "algorithm",
+                "auth_data",
+            ),
+            desc="get_e2e_room_key_version_info",
+        )
+
+        defer.returnValue(row)
+
+    def create_e2e_room_key_version(self, user_id, version, info):
+
+        def _create_e2e_room_key_version_txn(txn):
+
+            self._simple_insert_txn(
+                txn,
+                table="e2e_room_key_versions",
+                values={
+                    "user_id": user_id,
+                    "version": version,
+                    "algorithm": info["algorithm"],
+                    "auth_data": info["auth_data"],
+                },
+                lock=False,
+            )
+
+            return True
+
+        return self.runInteraction(
+            "create_e2e_room_key_version_txn", _create_e2e_room_key_version_txn
+        )
+
+    @defer.inlineCallbacks
+    def delete_e2e_room_key_version(self, user_id, version):
+
+        keyvalues = {
+            "user_id": user_id,
+            "version": version,
+        }
+
+        yield self._simple_delete(
+            table="e2e_room_key_versions",
+            keyvalues=keyvalues,
+            desc="delete_e2e_room_key_version",
+        )
diff --git a/synapse/storage/schema/delta/46/e2e_room_keys.sql b/synapse/storage/schema/delta/46/e2e_room_keys.sql
index 6b344c5ad78d..463f828c6670 100644
--- a/synapse/storage/schema/delta/46/e2e_room_keys.sql
+++ b/synapse/storage/schema/delta/46/e2e_room_keys.sql
@@ -34,7 +34,7 @@ CREATE TABLE e2e_room_key_versions (
     user_id TEXT NOT NULL,
     version INT NOT NULL,
     algorithm TEXT NOT NULL,
-    dummy_session_data TEXT NOT NULL
+    auth_data TEXT NOT NULL
 );
 
 CREATE UNIQUE INDEX e2e_room_key_user_idx ON e2e_room_keys(user_id);

From 69e51c7ba48a84b48ab64c8c290a232d14193a18 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Wed, 6 Dec 2017 10:02:49 +0100
Subject: [PATCH 352/830] make /room_keys/version work

---
 synapse/handlers/e2e_room_keys.py             | 18 +++++++++------
 synapse/rest/client/v2_alpha/room_keys.py     |  9 +++++++-
 synapse/storage/e2e_room_keys.py              | 22 ++++++++++++++-----
 .../storage/schema/delta/46/e2e_room_keys.sql |  4 ++--
 4 files changed, 38 insertions(+), 15 deletions(-)

diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py
index 4333ca610cbe..bd58be655853 100644
--- a/synapse/handlers/e2e_room_keys.py
+++ b/synapse/handlers/e2e_room_keys.py
@@ -48,13 +48,16 @@ def upload_room_keys(self, user_id, version, room_keys):
         # TODO: Validate the JSON to make sure it has the right keys.
 
         # Check that the version we're trying to upload is the current version
+
         try:
             version_info = yield self.get_version_info(user_id, version)
         except StoreError as e:
             if e.code == 404:
-                raise SynapseError(404, "Version '%d' not found" % (version,))
+                raise SynapseError(404, "Version '%s' not found" % (version,))
+            else:
+                raise e
 
-        if version_info.version != version:
+        if version_info['version'] != version:
             raise RoomKeysVersionError(current_version=version_info.version)
 
         # XXX: perhaps we should use a finer grained lock here?
@@ -81,7 +84,7 @@ def _upload_room_key(self, user_id, version, room_id, session_id, room_key):
             if e.code == 404:
                 pass
             else:
-                raise
+                raise e
 
         # check whether we merge or not. spelling it out with if/elifs rather
         # than lots of booleans for legibility.
@@ -106,20 +109,21 @@ def _upload_room_key(self, user_id, version, room_id, session_id, room_key):
             )
 
     @defer.inlineCallbacks
-    def create_version(self, user_id, version, version_info):
+    def create_version(self, user_id, version_info):
 
         # TODO: Validate the JSON to make sure it has the right keys.
 
         # lock everyone out until we've switched version
         with (yield self._upload_linearizer.queue(user_id)):
-            yield self.store.create_version(
-                user_id, version, version_info
+            new_version = yield self.store.create_e2e_room_key_version(
+                user_id, version_info
             )
+            defer.returnValue(new_version)
 
     @defer.inlineCallbacks
     def get_version_info(self, user_id, version):
         with (yield self._upload_linearizer.queue(user_id)):
-            results = yield self.store.get_e2e_room_key_version(
+            results = yield self.store.get_e2e_room_key_version_info(
                 user_id, version
             )
             defer.returnValue(results)
diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py
index 4d76e1d824dc..128b732fb15c 100644
--- a/synapse/rest/client/v2_alpha/room_keys.py
+++ b/synapse/rest/client/v2_alpha/room_keys.py
@@ -17,6 +17,7 @@
 
 from twisted.internet import defer
 
+from synapse.api.errors import SynapseError
 from synapse.http.servlet import (
     RestServlet, parse_json_object_from_request
 )
@@ -237,15 +238,21 @@ def __init__(self, hs):
 
     @defer.inlineCallbacks
     def on_POST(self, request, version):
+        if version:
+            raise SynapseError(405, "Cannot POST to a specific version")
+
         requester = yield self.auth.get_user_by_req(request, allow_guest=False)
         user_id = requester.user.to_string()
         info = parse_json_object_from_request(request)
 
         new_version = yield self.e2e_room_keys_handler.create_version(
-            user_id, version, info
+            user_id, info
         )
         defer.returnValue((200, {"version": new_version}))
 
+    # we deliberately don't have a PUT /version, as these things really should
+    # be immutable to avoid people footgunning
+
     @defer.inlineCallbacks
     def on_GET(self, request, version):
         requester = yield self.auth.get_user_by_req(request, allow_guest=False)
diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py
index 994878acf684..8efca11a8cc5 100644
--- a/synapse/storage/e2e_room_keys.py
+++ b/synapse/storage/e2e_room_keys.py
@@ -172,7 +172,7 @@ def delete_e2e_room_keys(self, user_id, version, room_id, session_id):
         )
 
     @defer.inlineCallbacks
-    def get_e2e_room_key_version(self, user_id, version):
+    def get_e2e_room_key_version_info(self, user_id, version):
 
         row = yield self._simple_select_one(
             table="e2e_room_key_versions",
@@ -191,23 +191,35 @@ def get_e2e_room_key_version(self, user_id, version):
 
         defer.returnValue(row)
 
-    def create_e2e_room_key_version(self, user_id, version, info):
+    def create_e2e_room_key_version(self, user_id, info):
+        """Atomically creates a new version of this user's e2e_room_keys store
+        with the given version info.
+        """
 
         def _create_e2e_room_key_version_txn(txn):
 
+            txn.execute(
+                "SELECT MAX(version) FROM e2e_room_key_versions WHERE user_id=?",
+                (user_id,)
+            )
+            current_version = txn.fetchone()[0]
+            if current_version is None:
+                current_version = 0
+
+            new_version = current_version + 1
+
             self._simple_insert_txn(
                 txn,
                 table="e2e_room_key_versions",
                 values={
                     "user_id": user_id,
-                    "version": version,
+                    "version": new_version,
                     "algorithm": info["algorithm"],
                     "auth_data": info["auth_data"],
                 },
-                lock=False,
             )
 
-            return True
+            return new_version
 
         return self.runInteraction(
             "create_e2e_room_key_version_txn", _create_e2e_room_key_version_txn
diff --git a/synapse/storage/schema/delta/46/e2e_room_keys.sql b/synapse/storage/schema/delta/46/e2e_room_keys.sql
index 463f828c6670..0d2a85fbe67e 100644
--- a/synapse/storage/schema/delta/46/e2e_room_keys.sql
+++ b/synapse/storage/schema/delta/46/e2e_room_keys.sql
@@ -18,7 +18,7 @@ CREATE TABLE e2e_room_keys (
     user_id TEXT NOT NULL,
     room_id TEXT NOT NULL,
     session_id TEXT NOT NULL,
-    version INT NOT NULL,
+    version TEXT NOT NULL,
     first_message_index INT,
     forwarded_count INT,
     is_verified BOOLEAN,
@@ -32,7 +32,7 @@ CREATE UNIQUE INDEX e2e_room_keys_session_idx ON e2e_room_keys(session_id);
 -- the metadata for each generation of encrypted e2e session backups
 CREATE TABLE e2e_room_key_versions (
     user_id TEXT NOT NULL,
-    version INT NOT NULL,
+    version TEXT NOT NULL,
     algorithm TEXT NOT NULL,
     auth_data TEXT NOT NULL
 );

From 0abb205b47158a4160ddceb317c0245d640b6e3f Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Mon, 18 Dec 2017 01:52:46 +0000
Subject: [PATCH 353/830] blindly incorporate PR review - needs testing &
 fixing

---
 synapse/api/errors.py                         | 11 ++-
 synapse/handlers/e2e_room_keys.py             | 88 ++++++++++++-------
 synapse/rest/client/v2_alpha/room_keys.py     |  2 +
 synapse/storage/e2e_room_keys.py              | 69 ++++++---------
 .../storage/schema/delta/46/e2e_room_keys.sql |  8 +-
 5 files changed, 99 insertions(+), 79 deletions(-)

diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index 8c97e91ba194..d37bcb4082ba 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -289,9 +289,14 @@ def error_dict(self):
 class RoomKeysVersionError(SynapseError):
     """A client has tried to upload to a non-current version of the room_keys store
     """
-    def __init__(self, code=403, msg="Wrong room_keys version", current_version=None,
-                 errcode=Codes.WRONG_ROOM_KEYS_VERSION):
-        super(RoomKeysVersionError, self).__init__(code, msg, errcode)
+    def __init__(self, current_version):
+        """
+        Args:
+            current_version (str): the current version of the store they should have used
+        """
+        super(RoomKeysVersionError, self).__init__(
+            403, "Wrong room_keys version", Codes.WRONG_ROOM_KEYS_VERSION
+        )
         self.current_version = current_version
 
     def error_dict(self):
diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py
index bd58be655853..dda31fdd24dc 100644
--- a/synapse/handlers/e2e_room_keys.py
+++ b/synapse/handlers/e2e_room_keys.py
@@ -24,8 +24,21 @@
 
 
 class E2eRoomKeysHandler(object):
+    """
+    Implements an optional realtime backup mechanism for encrypted E2E megolm room keys.
+    This gives a way for users to store and recover their megolm keys if they lose all
+    their clients. It should also extend easily to future room key mechanisms.
+    The actual payload of the encrypted keys is completely opaque to the handler.
+    """
+
     def __init__(self, hs):
         self.store = hs.get_datastore()
+
+        # Used to lock whenever a client is uploading key data.  This prevents collisions
+        # between clients trying to upload the details of a new session, given all
+        # clients belonging to a user will receive and try to upload a new session at
+        # roughly the same time.  Also used to lock out uploads when the key is being
+        # changed.
         self._upload_linearizer = Linearizer("upload_room_keys_lock")
 
     @defer.inlineCallbacks
@@ -40,33 +53,34 @@ def get_room_keys(self, user_id, version, room_id, session_id):
 
     @defer.inlineCallbacks
     def delete_room_keys(self, user_id, version, room_id, session_id):
-        yield self.store.delete_e2e_room_keys(user_id, version, room_id, session_id)
+        # lock for consistency with uploading
+        with (yield self._upload_linearizer.queue(user_id)):
+            yield self.store.delete_e2e_room_keys(user_id, version, room_id, session_id)
 
     @defer.inlineCallbacks
     def upload_room_keys(self, user_id, version, room_keys):
 
         # TODO: Validate the JSON to make sure it has the right keys.
 
-        # Check that the version we're trying to upload is the current version
-
-        try:
-            version_info = yield self.get_version_info(user_id, version)
-        except StoreError as e:
-            if e.code == 404:
-                raise SynapseError(404, "Version '%s' not found" % (version,))
-            else:
-                raise e
-
-        if version_info['version'] != version:
-            raise RoomKeysVersionError(current_version=version_info.version)
-
         # XXX: perhaps we should use a finer grained lock here?
         with (yield self._upload_linearizer.queue(user_id)):
-
-            # go through the room_keys
-            for room_id in room_keys['rooms']:
-                for session_id in room_keys['rooms'][room_id]['sessions']:
-                    room_key = room_keys['rooms'][room_id]['sessions'][session_id]
+            # Check that the version we're trying to upload is the current version
+            try:
+                version_info = yield self.get_version_info(user_id, version)
+            except StoreError as e:
+                if e.code == 404:
+                    raise SynapseError(404, "Version '%s' not found" % (version,))
+                else:
+                    raise e
+
+            if version_info['version'] != version:
+                raise RoomKeysVersionError(current_version=version_info.version)
+
+            # go through the room_keys.
+            # XXX: this should/could be done concurrently, given we're in a lock.
+            for room_id, room in room_keys['rooms'].iteritems():
+                for session_id, session in room['sessions'].iteritems():
+                    room_key = session[session_id]
 
                     yield self._upload_room_key(
                         user_id, version, room_id, session_id, room_key
@@ -86,10 +100,29 @@ def _upload_room_key(self, user_id, version, room_id, session_id, room_key):
             else:
                 raise e
 
-        # check whether we merge or not. spelling it out with if/elifs rather
-        # than lots of booleans for legibility.
-        upsert = True
+        if _should_replace_room_key(current_room_key, room_key):
+            yield self.store.set_e2e_room_key(
+                user_id, version, room_id, session_id, room_key
+            )
+
+    def _should_replace_room_key(current_room_key, room_key):
+        """
+        Determine whether to replace the current_room_key in our backup for this
+        session (if any) with a new room_key that has been uploaded.
+
+        Args:
+            current_room_key (dict): Optional, the current room_key dict if any
+            room_key (dict): The new room_key dict which may or may not be fit to
+                replace the current_room_key
+
+        Returns:
+            True if current_room_key should be replaced by room_key in the backup
+        """
+
         if current_room_key:
+            # spelt out with if/elifs rather than nested boolean expressions
+            # purely for legibility.
+
             if room_key['is_verified'] and not current_room_key['is_verified']:
                 pass
             elif (
@@ -97,16 +130,11 @@ def _upload_room_key(self, user_id, version, room_id, session_id, room_key):
                 current_room_key['first_message_index']
             ):
                 pass
-            elif room_key['forwarded_count'] < room_key['forwarded_count']:
+            elif room_key['forwarded_count'] < current_room_key['forwarded_count']:
                 pass
             else:
-                upsert = False
-
-        # if so, we set the new room_key
-        if upsert:
-            yield self.store.set_e2e_room_key(
-                user_id, version, room_id, session_id, room_key
-            )
+                return False
+        return True
 
     @defer.inlineCallbacks
     def create_version(self, user_id, version_info):
diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py
index 128b732fb15c..70b7b4573f57 100644
--- a/synapse/rest/client/v2_alpha/room_keys.py
+++ b/synapse/rest/client/v2_alpha/room_keys.py
@@ -68,6 +68,8 @@ def on_PUT(self, request, room_id, session_id):
          * lower forwarded_count always wins over higher forwarded_count
 
         We trust the clients not to lie and corrupt their own backups.
+        It also means that if your access_token is stolen, the attacker could
+        delete your backup.
 
         POST /room_keys/keys/!abc:matrix.org/c0ff33?version=1 HTTP/1.1
         Content-Type: application/json
diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py
index 8efca11a8cc5..c11417c41539 100644
--- a/synapse/storage/e2e_room_keys.py
+++ b/synapse/storage/e2e_room_keys.py
@@ -44,30 +44,21 @@ def get_e2e_room_key(self, user_id, version, room_id, session_id):
 
     def set_e2e_room_key(self, user_id, version, room_id, session_id, room_key):
 
-        def _set_e2e_room_key_txn(txn):
-
-            self._simple_upsert_txn(
-                txn,
-                table="e2e_room_keys",
-                keyvalues={
-                    "user_id": user_id,
-                    "room_id": room_id,
-                    "session_id": session_id,
-                },
-                values={
-                    "version": version,
-                    "first_message_index": room_key['first_message_index'],
-                    "forwarded_count": room_key['forwarded_count'],
-                    "is_verified": room_key['is_verified'],
-                    "session_data": room_key['session_data'],
-                },
-                lock=False,
-            )
-
-            return True
-
-        return self.runInteraction(
-            "set_e2e_room_key", _set_e2e_room_key_txn
+        yield self._simple_upsert(
+            table="e2e_room_keys",
+            keyvalues={
+                "user_id": user_id,
+                "room_id": room_id,
+                "session_id": session_id,
+            },
+            values={
+                "version": version,
+                "first_message_index": room_key['first_message_index'],
+                "forwarded_count": room_key['forwarded_count'],
+                "is_verified": room_key['is_verified'],
+                "session_data": room_key['session_data'],
+            },
+            lock=False,
         )
 
     # XXX: this isn't currently used and isn't tested anywhere
@@ -107,7 +98,9 @@ def _set_e2e_room_keys_txn(txn):
         )
 
     @defer.inlineCallbacks
-    def get_e2e_room_keys(self, user_id, version, room_id, session_id):
+    def get_e2e_room_keys(
+        self, user_id, version, room_id=room_id, session_id=session_id
+    ):
 
         keyvalues = {
             "user_id": user_id,
@@ -115,8 +108,8 @@ def get_e2e_room_keys(self, user_id, version, room_id, session_id):
         }
         if room_id:
             keyvalues['room_id'] = room_id
-        if session_id:
-            keyvalues['session_id'] = session_id
+            if session_id:
+                keyvalues['session_id'] = session_id
 
         rows = yield self._simple_select_list(
             table="e2e_room_keys",
@@ -133,18 +126,10 @@ def get_e2e_room_keys(self, user_id, version, room_id, session_id):
             desc="get_e2e_room_keys",
         )
 
-        # perlesque autovivification from https://stackoverflow.com/a/19829714/6764493
-        class AutoVivification(dict):
-            def __getitem__(self, item):
-                try:
-                    return dict.__getitem__(self, item)
-                except KeyError:
-                    value = self[item] = type(self)()
-                    return value
-
-        sessions = AutoVivification()
+        sessions = {}
         for row in rows:
-            sessions['rooms'][row['room_id']]['sessions'][row['session_id']] = {
+            room_entry = sessions['rooms'].setdefault(row['room_id'], {"sessions": {}})
+            room_entry['sessions'][row['session_id']] = {
                 "first_message_index": row["first_message_index"],
                 "forwarded_count": row["forwarded_count"],
                 "is_verified": row["is_verified"],
@@ -154,7 +139,9 @@ def __getitem__(self, item):
         defer.returnValue(sessions)
 
     @defer.inlineCallbacks
-    def delete_e2e_room_keys(self, user_id, version, room_id, session_id):
+    def delete_e2e_room_keys(
+        self, user_id, version, room_id=room_id, session_id=session_id
+    ):
 
         keyvalues = {
             "user_id": user_id,
@@ -162,8 +149,8 @@ def delete_e2e_room_keys(self, user_id, version, room_id, session_id):
         }
         if room_id:
             keyvalues['room_id'] = room_id
-        if session_id:
-            keyvalues['session_id'] = session_id
+            if session_id:
+                keyvalues['session_id'] = session_id
 
         yield self._simple_delete(
             table="e2e_room_keys",
diff --git a/synapse/storage/schema/delta/46/e2e_room_keys.sql b/synapse/storage/schema/delta/46/e2e_room_keys.sql
index 0d2a85fbe67e..16499ac34c23 100644
--- a/synapse/storage/schema/delta/46/e2e_room_keys.sql
+++ b/synapse/storage/schema/delta/46/e2e_room_keys.sql
@@ -25,16 +25,14 @@ CREATE TABLE e2e_room_keys (
     session_data TEXT NOT NULL
 );
 
-CREATE UNIQUE INDEX e2e_room_keys_user_idx ON e2e_room_keys(user_id);
-CREATE UNIQUE INDEX e2e_room_keys_room_idx ON e2e_room_keys(room_id);
-CREATE UNIQUE INDEX e2e_room_keys_session_idx ON e2e_room_keys(session_id);
+CREATE UNIQUE INDEX e2e_room_keys_idx ON e2e_room_keys(user_id, room_id, session_id);
 
 -- the metadata for each generation of encrypted e2e session backups
-CREATE TABLE e2e_room_key_versions (
+CREATE TABLE e2e_room_keys_versions (
     user_id TEXT NOT NULL,
     version TEXT NOT NULL,
     algorithm TEXT NOT NULL,
     auth_data TEXT NOT NULL
 );
 
-CREATE UNIQUE INDEX e2e_room_key_user_idx ON e2e_room_keys(user_id);
+CREATE UNIQUE INDEX e2e_room_keys_versions_user_idx ON e2e_room_keys_versions(user_id);

From cac02537998718c05a561918269745161378dd6f Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Mon, 18 Dec 2017 01:58:53 +0000
Subject: [PATCH 354/830] rename room_key_version table correctly, and fix opt
 args

---
 synapse/handlers/e2e_room_keys.py |  6 +++---
 synapse/storage/e2e_room_keys.py  | 26 +++++++++++++-------------
 2 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py
index dda31fdd24dc..87be081b1cfd 100644
--- a/synapse/handlers/e2e_room_keys.py
+++ b/synapse/handlers/e2e_room_keys.py
@@ -143,7 +143,7 @@ def create_version(self, user_id, version_info):
 
         # lock everyone out until we've switched version
         with (yield self._upload_linearizer.queue(user_id)):
-            new_version = yield self.store.create_e2e_room_key_version(
+            new_version = yield self.store.create_e2e_room_keys_version(
                 user_id, version_info
             )
             defer.returnValue(new_version)
@@ -151,7 +151,7 @@ def create_version(self, user_id, version_info):
     @defer.inlineCallbacks
     def get_version_info(self, user_id, version):
         with (yield self._upload_linearizer.queue(user_id)):
-            results = yield self.store.get_e2e_room_key_version_info(
+            results = yield self.store.get_e2e_room_keys_version_info(
                 user_id, version
             )
             defer.returnValue(results)
@@ -159,4 +159,4 @@ def get_version_info(self, user_id, version):
     @defer.inlineCallbacks
     def delete_version(self, user_id, version):
         with (yield self._upload_linearizer.queue(user_id)):
-            yield self.store.delete_e2e_room_key_version(user_id, version)
+            yield self.store.delete_e2e_room_keys_version(user_id, version)
diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py
index c11417c41539..7e1cb13e7427 100644
--- a/synapse/storage/e2e_room_keys.py
+++ b/synapse/storage/e2e_room_keys.py
@@ -99,7 +99,7 @@ def _set_e2e_room_keys_txn(txn):
 
     @defer.inlineCallbacks
     def get_e2e_room_keys(
-        self, user_id, version, room_id=room_id, session_id=session_id
+        self, user_id, version, room_id=None, session_id=None
     ):
 
         keyvalues = {
@@ -140,7 +140,7 @@ def get_e2e_room_keys(
 
     @defer.inlineCallbacks
     def delete_e2e_room_keys(
-        self, user_id, version, room_id=room_id, session_id=session_id
+        self, user_id, version, room_id=None, session_id=None
     ):
 
         keyvalues = {
@@ -159,10 +159,10 @@ def delete_e2e_room_keys(
         )
 
     @defer.inlineCallbacks
-    def get_e2e_room_key_version_info(self, user_id, version):
+    def get_e2e_room_keys_version_info(self, user_id, version):
 
         row = yield self._simple_select_one(
-            table="e2e_room_key_versions",
+            table="e2e_room_keys_versions",
             keyvalues={
                 "user_id": user_id,
                 "version": version,
@@ -173,20 +173,20 @@ def get_e2e_room_key_version_info(self, user_id, version):
                 "algorithm",
                 "auth_data",
             ),
-            desc="get_e2e_room_key_version_info",
+            desc="get_e2e_room_keys_version_info",
         )
 
         defer.returnValue(row)
 
-    def create_e2e_room_key_version(self, user_id, info):
+    def create_e2e_room_keys_version(self, user_id, info):
         """Atomically creates a new version of this user's e2e_room_keys store
         with the given version info.
         """
 
-        def _create_e2e_room_key_version_txn(txn):
+        def _create_e2e_room_keys_version_txn(txn):
 
             txn.execute(
-                "SELECT MAX(version) FROM e2e_room_key_versions WHERE user_id=?",
+                "SELECT MAX(version) FROM e2e_room_keys_versions WHERE user_id=?",
                 (user_id,)
             )
             current_version = txn.fetchone()[0]
@@ -197,7 +197,7 @@ def _create_e2e_room_key_version_txn(txn):
 
             self._simple_insert_txn(
                 txn,
-                table="e2e_room_key_versions",
+                table="e2e_room_keys_versions",
                 values={
                     "user_id": user_id,
                     "version": new_version,
@@ -209,11 +209,11 @@ def _create_e2e_room_key_version_txn(txn):
             return new_version
 
         return self.runInteraction(
-            "create_e2e_room_key_version_txn", _create_e2e_room_key_version_txn
+            "create_e2e_room_keys_version_txn", _create_e2e_room_keys_version_txn
         )
 
     @defer.inlineCallbacks
-    def delete_e2e_room_key_version(self, user_id, version):
+    def delete_e2e_room_keys_version(self, user_id, version):
 
         keyvalues = {
             "user_id": user_id,
@@ -221,7 +221,7 @@ def delete_e2e_room_key_version(self, user_id, version):
         }
 
         yield self._simple_delete(
-            table="e2e_room_key_versions",
+            table="e2e_room_keys_versions",
             keyvalues=keyvalues,
-            desc="delete_e2e_room_key_version",
+            desc="delete_e2e_room_keys_version",
         )

From ca0b052307de8868d3e337f1ace5667dad740ab1 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Sun, 24 Dec 2017 15:03:44 +0000
Subject: [PATCH 355/830] fix factoring out of _should_replace_room_key

---
 synapse/handlers/e2e_room_keys.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py
index 87be081b1cfd..b67d6a2a7e68 100644
--- a/synapse/handlers/e2e_room_keys.py
+++ b/synapse/handlers/e2e_room_keys.py
@@ -100,15 +100,16 @@ def _upload_room_key(self, user_id, version, room_id, session_id, room_key):
             else:
                 raise e
 
-        if _should_replace_room_key(current_room_key, room_key):
+        if E2eRoomKeysHandler._should_replace_room_key(current_room_key, room_key):
             yield self.store.set_e2e_room_key(
                 user_id, version, room_id, session_id, room_key
             )
 
+    @staticmethod
     def _should_replace_room_key(current_room_key, room_key):
         """
-        Determine whether to replace the current_room_key in our backup for this
-        session (if any) with a new room_key that has been uploaded.
+        Determine whether to replace a given current_room_key (if any)
+        with a newly uploaded room_key backup
 
         Args:
             current_room_key (dict): Optional, the current room_key dict if any

From 8d14598e90396c52c9394cf807861921e053d8da Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Sun, 24 Dec 2017 16:44:18 +0000
Subject: [PATCH 356/830] add storage docstring; remove unused
 set_e2e_room_keys

---
 synapse/storage/e2e_room_keys.py | 119 +++++++++++++++++++++----------
 1 file changed, 83 insertions(+), 36 deletions(-)

diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py
index 7e1cb13e7427..45d2c9b4333c 100644
--- a/synapse/storage/e2e_room_keys.py
+++ b/synapse/storage/e2e_room_keys.py
@@ -22,6 +22,22 @@ class EndToEndRoomKeyStore(SQLBaseStore):
 
     @defer.inlineCallbacks
     def get_e2e_room_key(self, user_id, version, room_id, session_id):
+        """Get the encrypted E2E room key for a given session from a given
+        backup version of room_keys.  We only store the 'best' room key for a given
+        session at a given time, as determined by the handler.
+
+        Args:
+            user_id(str): the user whose backup we're querying
+            version(str): the version ID of the backup for the set of keys we're querying
+            room_id(str): the ID of the room whose keys we're querying.
+                This is a bit redundant as it's implied by the session_id, but
+                we include for consistency with the rest of the API.
+            session_id(str): the session whose room_key we're querying.
+
+        Returns:
+            A deferred dict giving the session_data and message metadata for
+            this room key.
+        """
 
         row = yield self._simple_select_one(
             table="e2e_room_keys",
@@ -43,6 +59,17 @@ def get_e2e_room_key(self, user_id, version, room_id, session_id):
         defer.returnValue(row)
 
     def set_e2e_room_key(self, user_id, version, room_id, session_id, room_key):
+        """Replaces or inserts the encrypted E2E room key for a given session in
+        a given backup
+
+        Args:
+            user_id(str): the user whose backup we're setting
+            version(str): the version ID of the backup we're updating
+            room_id(str): the ID of the room whose keys we're setting
+            session_id(str): the session whose room_key we're setting
+        Raises:
+            StoreError if stuff goes wrong, probably
+        """
 
         yield self._simple_upsert(
             table="e2e_room_keys",
@@ -61,46 +88,27 @@ def set_e2e_room_key(self, user_id, version, room_id, session_id, room_key):
             lock=False,
         )
 
-    # XXX: this isn't currently used and isn't tested anywhere
-    # it could be used in future for bulk-uploading new versions of room_keys
-    # for a user or something though.
-    def set_e2e_room_keys(self, user_id, version, room_keys):
-
-        def _set_e2e_room_keys_txn(txn):
-
-            values = []
-            for room_id in room_keys['rooms']:
-                for session_id in room_keys['rooms'][room_id]['sessions']:
-                    session = room_keys['rooms'][room_id]['sessions'][session_id]
-                    values.append(
-                        {
-                            "user_id": user_id,
-                            "room_id": room_id,
-                            "session_id": session_id,
-                            "version": version,
-                            "first_message_index": session['first_message_index'],
-                            "forwarded_count": session['forwarded_count'],
-                            "is_verified": session['is_verified'],
-                            "session_data": session['session_data'],
-                        }
-                    )
-
-            self._simple_insert_many_txn(
-                txn,
-                table="e2e_room_keys",
-                values=values
-            )
-
-            return True
-
-        return self.runInteraction(
-            "set_e2e_room_keys", _set_e2e_room_keys_txn
-        )
-
     @defer.inlineCallbacks
     def get_e2e_room_keys(
         self, user_id, version, room_id=None, session_id=None
     ):
+        """Bulk get the E2E room keys for a given backup, optionally filtered to a given
+        room, or a given session.
+
+        Args:
+            user_id(str): the user whose backup we're querying
+            version(str): the version ID of the backup for the set of keys we're querying
+            room_id(str): Optional. the ID of the room whose keys we're querying, if any.
+                If not specified, we return the keys for all the rooms in the backup.
+            session_id(str): Optional. the session whose room_key we're querying, if any.
+                If specified, we also require the room_id to be specified.
+                If not specified, we return all the keys in this version of
+                the backup (or for the specified room)
+
+        Returns:
+            A deferred list of dicts giving the session_data and message metadata for
+            these room keys.
+        """
 
         keyvalues = {
             "user_id": user_id,
@@ -142,6 +150,22 @@ def get_e2e_room_keys(
     def delete_e2e_room_keys(
         self, user_id, version, room_id=None, session_id=None
     ):
+        """Bulk delete the E2E room keys for a given backup, optionally filtered to a given
+        room or a given session.
+
+        Args:
+            user_id(str): the user whose backup we're deleting from
+            version(str): the version ID of the backup for the set of keys we're deleting
+            room_id(str): Optional. the ID of the room whose keys we're deleting, if any.
+                If not specified, we delete the keys for all the rooms in the backup.
+            session_id(str): Optional. the session whose room_key we're querying, if any.
+                If specified, we also require the room_id to be specified.
+                If not specified, we delete all the keys in this version of
+                the backup (or for the specified room)
+
+        Returns:
+            A deferred of the deletion transaction
+        """
 
         keyvalues = {
             "user_id": user_id,
@@ -160,6 +184,15 @@ def delete_e2e_room_keys(
 
     @defer.inlineCallbacks
     def get_e2e_room_keys_version_info(self, user_id, version):
+        """Get info etadata about a given version of our room_keys backup
+
+        Args:
+            user_id(str): the user whose backup we're querying
+            version(str): the version ID of the backup we're querying about
+
+        Returns:
+            A deferred dict giving the info metadata for this backup version
+        """
 
         row = yield self._simple_select_one(
             table="e2e_room_keys_versions",
@@ -181,6 +214,13 @@ def get_e2e_room_keys_version_info(self, user_id, version):
     def create_e2e_room_keys_version(self, user_id, info):
         """Atomically creates a new version of this user's e2e_room_keys store
         with the given version info.
+
+        Args:
+            user_id(str): the user whose backup we're creating a version
+            info(dict): the info about the backup version to be created
+
+        Returns:
+            A deferred string for the newly created version ID
         """
 
         def _create_e2e_room_keys_version_txn(txn):
@@ -214,6 +254,13 @@ def _create_e2e_room_keys_version_txn(txn):
 
     @defer.inlineCallbacks
     def delete_e2e_room_keys_version(self, user_id, version):
+        """Delete a given backup version of the user's room keys.
+        Doesn't delete their actual key data.
+
+        Args:
+            user_id(str): the user whose backup version we're deleting
+            version(str): the ID of the backup version we're deleting
+        """
 
         keyvalues = {
             "user_id": user_id,

From 9f500cb39efa608c02f212711a5ad2177757bf4b Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Sun, 24 Dec 2017 17:42:17 +0000
Subject: [PATCH 357/830] more docstring for the e2e_room_keys rest

---
 synapse/handlers/e2e_room_keys.py         |  2 -
 synapse/rest/client/v2_alpha/room_keys.py | 51 +++++++++++++++++++++++
 2 files changed, 51 insertions(+), 2 deletions(-)

diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py
index b67d6a2a7e68..7a940d1c2132 100644
--- a/synapse/handlers/e2e_room_keys.py
+++ b/synapse/handlers/e2e_room_keys.py
@@ -59,7 +59,6 @@ def delete_room_keys(self, user_id, version, room_id, session_id):
 
     @defer.inlineCallbacks
     def upload_room_keys(self, user_id, version, room_keys):
-
         # TODO: Validate the JSON to make sure it has the right keys.
 
         # XXX: perhaps we should use a finer grained lock here?
@@ -139,7 +138,6 @@ def _should_replace_room_key(current_room_key, room_key):
 
     @defer.inlineCallbacks
     def create_version(self, user_id, version_info):
-
         # TODO: Validate the JSON to make sure it has the right keys.
 
         # lock everyone out until we've switched version
diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py
index 70b7b4573f57..04547c7d4371 100644
--- a/synapse/rest/client/v2_alpha/room_keys.py
+++ b/synapse/rest/client/v2_alpha/room_keys.py
@@ -208,6 +208,10 @@ def on_DELETE(self, request, room_id, session_id):
         """
         Deletes one or more encrypted E2E room keys for a user for backup purposes.
 
+        DELETE /room_keys/keys/!abc:matrix.org/c0ff33?version=1
+        HTTP/1.1 200 OK
+        {}
+
         room_id: the ID of the room whose keys to delete (optional)
         session_id: the ID for the E2E session to delete (optional)
         version: the version of the user's backup which this data is for.
@@ -240,6 +244,33 @@ def __init__(self, hs):
 
     @defer.inlineCallbacks
     def on_POST(self, request, version):
+        """
+        Create a new backup version for this user's room_keys with the given
+        info.  The version is allocated by the server and returned to the user
+        in the response.  This API is intended to be used whenever the user
+        changes the encryption key for their backups, ensuring that backups
+        encrypted with different keys don't collide.
+
+        The algorithm passed in the version info is a reverse-DNS namespaced
+        identifier to describe the format of the encrypted backupped keys.
+
+        The auth_data is { user_id: "user_id", nonce: <random string> }
+        encrypted using the algorithm and current encryption key described above.
+
+        POST /room_keys/version
+        Content-Type: application/json
+        {
+            "algorithm": "m.megolm_backup.v1",
+            "auth_data": "dGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgZW5jcnlwdGVkIGpzb24K"
+        }
+
+        HTTP/1.1 200 OK
+        Content-Type: application/json
+        {
+            "version": 12345
+        }
+        """
+
         if version:
             raise SynapseError(405, "Cannot POST to a specific version")
 
@@ -257,6 +288,17 @@ def on_POST(self, request, version):
 
     @defer.inlineCallbacks
     def on_GET(self, request, version):
+        """
+        Retrieve the version information about a given version of the user's
+        room_keys backup.
+
+        GET /room_keys/version/12345 HTTP/1.1
+        {
+            "algorithm": "m.megolm_backup.v1",
+            "auth_data": "dGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgZW5jcnlwdGVkIGpzb24K"
+        }
+        """
+
         requester = yield self.auth.get_user_by_req(request, allow_guest=False)
         user_id = requester.user.to_string()
 
@@ -267,6 +309,15 @@ def on_GET(self, request, version):
 
     @defer.inlineCallbacks
     def on_DELETE(self, request, version):
+        """
+        Delete the information about a given version of the user's
+        room_keys backup.  Doesn't delete the actual room data.
+
+        DELETE /room_keys/version/12345 HTTP/1.1
+        HTTP/1.1 200 OK
+        {}
+        """
+
         requester = yield self.auth.get_user_by_req(request, allow_guest=False)
         user_id = requester.user.to_string()
 

From 9f0791b7bd030a70182cd9b33bbe2f78dba705dd Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Wed, 27 Dec 2017 23:35:10 +0000
Subject: [PATCH 358/830] add a tonne of docstring; make upload_room_keys
 properly assert version

---
 synapse/storage/e2e_room_keys.py | 51 +++++++++++++++++++-------------
 1 file changed, 31 insertions(+), 20 deletions(-)

diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py
index 45d2c9b4333c..e04e6a3690e1 100644
--- a/synapse/storage/e2e_room_keys.py
+++ b/synapse/storage/e2e_room_keys.py
@@ -67,6 +67,7 @@ def set_e2e_room_key(self, user_id, version, room_id, session_id, room_key):
             version(str): the version ID of the backup we're updating
             room_id(str): the ID of the room whose keys we're setting
             session_id(str): the session whose room_key we're setting
+            room_key(dict): the room_key being set
         Raises:
             StoreError if stuff goes wrong, probably
         """
@@ -182,35 +183,46 @@ def delete_e2e_room_keys(
             desc="delete_e2e_room_keys",
         )
 
-    @defer.inlineCallbacks
-    def get_e2e_room_keys_version_info(self, user_id, version):
-        """Get info etadata about a given version of our room_keys backup
+    def get_e2e_room_keys_version_info(self, user_id, version=None):
+        """Get info metadata about a version of our room_keys backup.
 
         Args:
             user_id(str): the user whose backup we're querying
-            version(str): the version ID of the backup we're querying about
-
+            version(str): Optional. the version ID of the backup we're querying about
+                If missing, we return the information about the current version.
+        Raises:
+            StoreError: with code 404 if there are no e2e_room_keys_versions present
         Returns:
             A deferred dict giving the info metadata for this backup version
         """
 
-        row = yield self._simple_select_one(
-            table="e2e_room_keys_versions",
-            keyvalues={
-                "user_id": user_id,
-                "version": version,
-            },
-            retcols=(
-                "user_id",
-                "version",
-                "algorithm",
-                "auth_data",
-            ),
+        def _get_e2e_room_keys_version_info_txn(txn):
+            if version is None:
+                txn.execute(
+                    "SELECT MAX(version) FROM e2e_room_keys_versions WHERE user_id=?",
+                    (user_id,)
+                )
+                version = txn.fetchone()[0]
+
+            return self._simple_select_one_txn(
+                table="e2e_room_keys_versions",
+                keyvalues={
+                    "user_id": user_id,
+                    "version": version,
+                },
+                retcols=(
+                    "user_id",
+                    "version",
+                    "algorithm",
+                    "auth_data",
+                ),
+            )
+
+        return self.runInteraction(
             desc="get_e2e_room_keys_version_info",
+            _get_e2e_room_keys_version_info_txn
         )
 
-        defer.returnValue(row)
-
     def create_e2e_room_keys_version(self, user_id, info):
         """Atomically creates a new version of this user's e2e_room_keys store
         with the given version info.
@@ -224,7 +236,6 @@ def create_e2e_room_keys_version(self, user_id, info):
         """
 
         def _create_e2e_room_keys_version_txn(txn):
-
             txn.execute(
                 "SELECT MAX(version) FROM e2e_room_keys_versions WHERE user_id=?",
                 (user_id,)

From 14b3da63a339333292a83410c0ba3148bcb644ba Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Wed, 27 Dec 2017 23:37:44 +0000
Subject: [PATCH 359/830] add a tonne of docstring; make upload_room_keys
 properly assert version

---
 synapse/handlers/e2e_room_keys.py         | 111 ++++++++++++++++++++--
 synapse/rest/client/v2_alpha/room_keys.py |  11 ++-
 2 files changed, 113 insertions(+), 9 deletions(-)

diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py
index 7a940d1c2132..2fa025bfc74c 100644
--- a/synapse/handlers/e2e_room_keys.py
+++ b/synapse/handlers/e2e_room_keys.py
@@ -42,7 +42,16 @@ def __init__(self, hs):
         self._upload_linearizer = Linearizer("upload_room_keys_lock")
 
     @defer.inlineCallbacks
-    def get_room_keys(self, user_id, version, room_id, session_id):
+    def get_room_keys(self, user_id, version, room_id=None, session_id=None):
+        """Bulk get the E2E room keys for a given backup, optionally filtered to a given
+        room, or a given session.
+        See EndToEndRoomKeyStore.get_e2e_room_keys for full details.
+
+        Returns:
+            A deferred list of dicts giving the session_data and message metadata for
+            these room keys.
+        """
+
         # we deliberately take the lock to get keys so that changing the version
         # works atomically
         with (yield self._upload_linearizer.queue(user_id)):
@@ -52,20 +61,56 @@ def get_room_keys(self, user_id, version, room_id, session_id):
             defer.returnValue(results)
 
     @defer.inlineCallbacks
-    def delete_room_keys(self, user_id, version, room_id, session_id):
+    def delete_room_keys(self, user_id, version, room_id=None, session_id=None):
+        """Bulk delete the E2E room keys for a given backup, optionally filtered to a given
+        room or a given session.
+        See EndToEndRoomKeyStore.delete_e2e_room_keys for full details.
+
+        Returns:
+            A deferred of the deletion transaction
+        """
+
         # lock for consistency with uploading
         with (yield self._upload_linearizer.queue(user_id)):
             yield self.store.delete_e2e_room_keys(user_id, version, room_id, session_id)
 
     @defer.inlineCallbacks
     def upload_room_keys(self, user_id, version, room_keys):
+        """Bulk upload a list of room keys into a given backup version, asserting
+        that the given version is the current backup version.  room_keys are merged
+        into the current backup as described in RoomKeysServlet.on_PUT().
+
+        Args:
+            user_id(str): the user whose backup we're setting
+            version(str): the version ID of the backup we're updating
+            room_keys(dict): a nested dict describing the room_keys we're setting:
+
+        {
+            "rooms": {
+                "!abc:matrix.org": {
+                    "sessions": {
+                        "c0ff33": {
+                            "first_message_index": 1,
+                            "forwarded_count": 1,
+                            "is_verified": false,
+                            "session_data": "SSBBTSBBIEZJU0gK"
+                        }
+                    }
+                }
+            }
+        }
+
+        Raises:
+            SynapseError: with code 404 if there are no versions defined
+            RoomKeysVersionError: if the uploaded version is not the current version
+        """
+
         # TODO: Validate the JSON to make sure it has the right keys.
 
         # XXX: perhaps we should use a finer grained lock here?
         with (yield self._upload_linearizer.queue(user_id)):
             # Check that the version we're trying to upload is the current version
-            try:
-                version_info = yield self.get_version_info(user_id, version)
+            version_info = yield self.get_current_version_info(user_id)
             except StoreError as e:
                 if e.code == 404:
                     raise SynapseError(404, "Version '%s' not found" % (version,))
@@ -87,6 +132,17 @@ def upload_room_keys(self, user_id, version, room_keys):
 
     @defer.inlineCallbacks
     def _upload_room_key(self, user_id, version, room_id, session_id, room_key):
+        """Upload a given room_key for a given room and session into a given
+        version of the backup.  Merges the key with any which might already exist.
+
+        Args: 
+            user_id(str): the user whose backup we're setting
+            version(str): the version ID of the backup we're updating
+            room_id(str): the ID of the room whose keys we're setting
+            session_id(str): the session whose room_key we're setting
+            room_key(dict): the room_key being set
+        """
+
         # get the room_key for this particular row
         current_room_key = None
         try:
@@ -138,6 +194,23 @@ def _should_replace_room_key(current_room_key, room_key):
 
     @defer.inlineCallbacks
     def create_version(self, user_id, version_info):
+        """Create a new backup version.  This automatically becomes the new
+        backup version for the user's keys; previous backups will no longer be
+        writeable to.
+
+        Args: 
+            user_id(str): the user whose backup version we're creating
+            version_info(dict): metadata about the new version being created
+
+        {
+            "algorithm": "m.megolm_backup.v1",
+            "auth_data": "dGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgZW5jcnlwdGVkIGpzb24K"
+        }
+
+        Returns:
+            A deferred of a string that gives the new version number.
+        """
+
         # TODO: Validate the JSON to make sure it has the right keys.
 
         # lock everyone out until we've switched version
@@ -148,14 +221,36 @@ def create_version(self, user_id, version_info):
             defer.returnValue(new_version)
 
     @defer.inlineCallbacks
-    def get_version_info(self, user_id, version):
+    def get_current_version_info(self, user_id):
+        """Get the user's current backup version.
+
+        Args:
+            user_id(str): the user whose current backup version we're querying
+        Raises:
+            StoreError: code 404 if there is no current backup version
+        Returns:
+            A deferred of a info dict that gives the info about the new version.
+
+        {
+            "algorithm": "m.megolm_backup.v1",
+            "auth_data": "dGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgZW5jcnlwdGVkIGpzb24K"
+        }
+        """
+
         with (yield self._upload_linearizer.queue(user_id)):
-            results = yield self.store.get_e2e_room_keys_version_info(
-                user_id, version
-            )
+            results = yield self.store.get_e2e_room_keys_version_info(user_id)
             defer.returnValue(results)
 
     @defer.inlineCallbacks
     def delete_version(self, user_id, version):
+        """Deletes a given version of the user's e2e_room_keys backup
+
+        Args:
+            user_id(str): the user whose current backup version we're deleting
+            version(str): the version id of the backup being deleted
+        Raises:
+            StoreError: code 404 if this backup version doesn't exist
+        """
+
         with (yield self._upload_linearizer.queue(user_id)):
             yield self.store.delete_e2e_room_keys_version(user_id, version)
diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py
index 04547c7d4371..d3f857aba2e0 100644
--- a/synapse/rest/client/v2_alpha/room_keys.py
+++ b/synapse/rest/client/v2_alpha/room_keys.py
@@ -47,7 +47,7 @@ def on_PUT(self, request, room_id, session_id):
         room_id: the ID of the room the keys are for (optional)
         session_id: the ID for the E2E room keys for the room (optional)
         version: the version of the user's backup which this data is for.
-        the version must already have been created via the /change_secret API.
+        the version must already have been created via the /room_keys/version API.
 
         Each session has:
          * first_message_index: a numeric index indicating the oldest message
@@ -59,6 +59,9 @@ def on_PUT(self, request, room_id, session_id):
          * session_data: base64-encrypted data describing the session.
 
         Returns 200 OK on success with body {}
+        Returns 403 Forbidden if the version in question is not the most recently
+        created version (i.e. if this is an old client trying to write to a stale backup)
+        Returns 404 Not Found if the version in question doesn't exist
 
         The API is designed to be otherwise agnostic to the room_key encryption
         algorithm being used.  Sessions are merged with existing ones in the
@@ -251,6 +254,9 @@ def on_POST(self, request, version):
         changes the encryption key for their backups, ensuring that backups
         encrypted with different keys don't collide.
 
+        It takes out an exclusive lock on this user's room_key backups, to ensure
+        clients only upload to the current backup.
+
         The algorithm passed in the version info is a reverse-DNS namespaced
         identifier to describe the format of the encrypted backupped keys.
 
@@ -292,6 +298,9 @@ def on_GET(self, request, version):
         Retrieve the version information about a given version of the user's
         room_keys backup.
 
+        It takes out an exclusive lock on this user's room_key backups, to ensure
+        clients only upload to the current backup.
+
         GET /room_keys/version/12345 HTTP/1.1
         {
             "algorithm": "m.megolm_backup.v1",

From 234611f3472b229d9e3a9ec7a27c51446f6a61ba Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Wed, 27 Dec 2017 23:42:08 +0000
Subject: [PATCH 360/830] fix typos

---
 synapse/handlers/e2e_room_keys.py | 3 ++-
 synapse/storage/e2e_room_keys.py  | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py
index 2fa025bfc74c..6446c3c6c353 100644
--- a/synapse/handlers/e2e_room_keys.py
+++ b/synapse/handlers/e2e_room_keys.py
@@ -110,7 +110,8 @@ def upload_room_keys(self, user_id, version, room_keys):
         # XXX: perhaps we should use a finer grained lock here?
         with (yield self._upload_linearizer.queue(user_id)):
             # Check that the version we're trying to upload is the current version
-            version_info = yield self.get_current_version_info(user_id)
+            try:
+                version_info = yield self.get_current_version_info(user_id)
             except StoreError as e:
                 if e.code == 404:
                     raise SynapseError(404, "Version '%s' not found" % (version,))
diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py
index e04e6a3690e1..b51faa1204e0 100644
--- a/synapse/storage/e2e_room_keys.py
+++ b/synapse/storage/e2e_room_keys.py
@@ -219,7 +219,7 @@ def _get_e2e_room_keys_version_info_txn(txn):
             )
 
         return self.runInteraction(
-            desc="get_e2e_room_keys_version_info",
+            "get_e2e_room_keys_version_info",
             _get_e2e_room_keys_version_info_txn
         )
 

From 982edca38026e2bb9085e77b4c25af60c4793f34 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Wed, 27 Dec 2017 23:58:51 +0000
Subject: [PATCH 361/830] fix flakes

---
 synapse/handlers/e2e_room_keys.py | 4 ++--
 synapse/storage/e2e_room_keys.py  | 6 ++++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py
index 6446c3c6c353..fdae69c60084 100644
--- a/synapse/handlers/e2e_room_keys.py
+++ b/synapse/handlers/e2e_room_keys.py
@@ -136,7 +136,7 @@ def _upload_room_key(self, user_id, version, room_id, session_id, room_key):
         """Upload a given room_key for a given room and session into a given
         version of the backup.  Merges the key with any which might already exist.
 
-        Args: 
+        Args:
             user_id(str): the user whose backup we're setting
             version(str): the version ID of the backup we're updating
             room_id(str): the ID of the room whose keys we're setting
@@ -199,7 +199,7 @@ def create_version(self, user_id, version_info):
         backup version for the user's keys; previous backups will no longer be
         writeable to.
 
-        Args: 
+        Args:
             user_id(str): the user whose backup version we're creating
             version_info(dict): metadata about the new version being created
 
diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py
index b51faa1204e0..7ab75070a26f 100644
--- a/synapse/storage/e2e_room_keys.py
+++ b/synapse/storage/e2e_room_keys.py
@@ -202,13 +202,15 @@ def _get_e2e_room_keys_version_info_txn(txn):
                     "SELECT MAX(version) FROM e2e_room_keys_versions WHERE user_id=?",
                     (user_id,)
                 )
-                version = txn.fetchone()[0]
+                this_version = txn.fetchone()[0]
+            else:
+                this_version = version
 
             return self._simple_select_one_txn(
                 table="e2e_room_keys_versions",
                 keyvalues={
                     "user_id": user_id,
-                    "version": version,
+                    "version": this_version,
                 },
                 retcols=(
                     "user_id",

From 5e42c45c96bb62a86604118250e9cb6c57b94254 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Sun, 31 Dec 2017 14:10:31 +0000
Subject: [PATCH 362/830] switch get_current_version_info back to being
 get_version_info

---
 synapse/handlers/e2e_room_keys.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py
index fdae69c60084..f08d80da3e5e 100644
--- a/synapse/handlers/e2e_room_keys.py
+++ b/synapse/handlers/e2e_room_keys.py
@@ -111,7 +111,7 @@ def upload_room_keys(self, user_id, version, room_keys):
         with (yield self._upload_linearizer.queue(user_id)):
             # Check that the version we're trying to upload is the current version
             try:
-                version_info = yield self.get_current_version_info(user_id)
+                version_info = yield self.get_version_info(user_id)
             except StoreError as e:
                 if e.code == 404:
                     raise SynapseError(404, "Version '%s' not found" % (version,))
@@ -222,17 +222,20 @@ def create_version(self, user_id, version_info):
             defer.returnValue(new_version)
 
     @defer.inlineCallbacks
-    def get_current_version_info(self, user_id):
-        """Get the user's current backup version.
+    def get_version_info(self, user_id, version=None):
+        """Get the info about a given version of the user's backup
 
         Args:
             user_id(str): the user whose current backup version we're querying
+            version(str): Optional; if None gives the most recent version
+                otherwise a historical one.
         Raises:
-            StoreError: code 404 if there is no current backup version
+            StoreError: code 404 if the requested backup version doesn't exist
         Returns:
             A deferred of a info dict that gives the info about the new version.
 
         {
+            "version": "1234",
             "algorithm": "m.megolm_backup.v1",
             "auth_data": "dGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgZW5jcnlwdGVkIGpzb24K"
         }

From 93d174bcc4cf773bea2534d3683a1e91e5489c89 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Sun, 31 Dec 2017 14:10:49 +0000
Subject: [PATCH 363/830] improve docstring

---
 synapse/rest/client/v2_alpha/room_keys.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py
index d3f857aba2e0..ca69ced1e31c 100644
--- a/synapse/rest/client/v2_alpha/room_keys.py
+++ b/synapse/rest/client/v2_alpha/room_keys.py
@@ -296,13 +296,17 @@ def on_POST(self, request, version):
     def on_GET(self, request, version):
         """
         Retrieve the version information about a given version of the user's
-        room_keys backup.
+        room_keys backup.  If the version part is missing, returns info about the
+        most current backup version (if any)
 
         It takes out an exclusive lock on this user's room_key backups, to ensure
         clients only upload to the current backup.
 
+        Returns 404 is the given version does not exist.
+
         GET /room_keys/version/12345 HTTP/1.1
         {
+            "version": "12345",
             "algorithm": "m.megolm_backup.v1",
             "auth_data": "dGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgZW5jcnlwdGVkIGpzb24K"
         }

From b5eee511c73fdd9b5d1a7453433513791192a250 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Sun, 31 Dec 2017 14:11:02 +0000
Subject: [PATCH 364/830] don't needlessly return user_id

---
 synapse/storage/e2e_room_keys.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py
index 7ab75070a26f..3c720f3b3e7c 100644
--- a/synapse/storage/e2e_room_keys.py
+++ b/synapse/storage/e2e_room_keys.py
@@ -213,7 +213,6 @@ def _get_e2e_room_keys_version_info_txn(txn):
                     "version": this_version,
                 },
                 retcols=(
-                    "user_id",
                     "version",
                     "algorithm",
                     "auth_data",

From 174be586e5fca46013a4a8f03b4337f46b2502aa Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Sun, 31 Dec 2017 14:11:15 +0000
Subject: [PATCH 365/830] first cut at a UT

---
 tests/handlers/test_e2e_room_keys.py | 141 +++++++++++++++++++++++++++
 1 file changed, 141 insertions(+)
 create mode 100644 tests/handlers/test_e2e_room_keys.py

diff --git a/tests/handlers/test_e2e_room_keys.py b/tests/handlers/test_e2e_room_keys.py
new file mode 100644
index 000000000000..9d3bef6db2f1
--- /dev/null
+++ b/tests/handlers/test_e2e_room_keys.py
@@ -0,0 +1,141 @@
+# -*- coding: utf-8 -*-
+# Copyright 2016 OpenMarket Ltd
+# Copyright 2017 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+from synapse.api import errors
+from twisted.internet import defer
+
+import synapse.api.errors
+import synapse.handlers.e2e_room_keys
+
+import synapse.storage
+from tests import unittest, utils
+
+
+class E2eRoomKeysHandlerTestCase(unittest.TestCase):
+    def __init__(self, *args, **kwargs):
+        super(E2eRoomKeysHandlerTestCase, self).__init__(*args, **kwargs)
+        self.hs = None       # type: synapse.server.HomeServer
+        self.handler = None  # type: synapse.handlers.e2e_keys.E2eRoomKeysHandler
+
+    @defer.inlineCallbacks
+    def setUp(self):
+        self.hs = yield utils.setup_test_homeserver(
+            handlers=None,
+            replication_layer=mock.Mock(),
+        )
+        self.handler = synapse.handlers.e2e_keys.E2eRoomKeysHandler(self.hs)
+
+
+    @defer.inlineCallbacks
+    def test_get_missing_current_version_info(self):
+        """Check that we get a 404 if we ask for info about the current version
+        if there is no version.
+        """
+        local_user = "@boris:" + self.hs.hostname
+        try:
+            res = yield self.handler.get_version_info(local_user);
+        except errors.SynapseError as e:
+            self.assertEqual(e.code, 404);
+        self.assertEqual(res, None);
+
+    @defer.inlineCallbacks
+    def test_get_missing_version_info(self):
+        """Check that we get a 404 if we ask for info about a specific version
+        if it doesn't exist.
+        """
+        local_user = "@boris:" + self.hs.hostname
+        try:
+            res = yield self.handler.get_version_info(local_user, "mrflibble");
+        except errors.SynapseError as e:
+            self.assertEqual(e.code, 404);
+        self.assertEqual(res, None);
+
+    @defer.inlineCallbacks
+    def test_create_version(self):
+        """Check that we can create and then retrieve versions.
+        """
+        local_user = "@boris:" + self.hs.hostname
+        res = yield self.handler.create_version(user_id, {
+            "algorithm": "m.megolm_backup.v1",
+            "auth_data": "first_version_auth_data",
+        });
+        self.assertEqual(res, "1");
+
+        # check we can retrieve it as the current version
+        res = yield self.handler.get_version_info(local_user);
+        self.assertDictEqual(res, {
+            "version": "1",
+            "algorithm": "m.megolm_backup.v1",
+            "auth_data": "first_version_auth_data",
+        });
+
+        # check we can retrieve it as a specific version
+        res = yield self.handler.get_version_info(local_user, "1");
+        self.assertDictEqual(res, {
+            "version": "1",
+            "algorithm": "m.megolm_backup.v1",
+            "auth_data": "first_version_auth_data",
+        });
+
+        # upload a new one...
+        res = yield self.handler.create_version(user_id, {
+            "algorithm": "m.megolm_backup.v1",
+            "auth_data": "second_version_auth_data",
+        });
+        self.assertEqual(res, "2");
+
+        # check we can retrieve it as the current version
+        res = yield self.handler.get_version_info(local_user);
+        self.assertDictEqual(res, {
+            "version": "2",
+            "algorithm": "m.megolm_backup.v1",
+            "auth_data": "second_version_auth_data",
+        });
+
+    @defer.inlineCallbacks
+    def test_delete_version(self):
+        """Check that we can create and then delete versions.
+        """
+        local_user = "@boris:" + self.hs.hostname
+        res = yield self.handler.create_version(user_id, {
+            "algorithm": "m.megolm_backup.v1",
+            "auth_data": "first_version_auth_data",
+        });
+        self.assertEqual(res, "1");
+
+        # check we can delete it
+        yield self.handler.delete_version(local_user, "1");
+
+        # check that it's gone
+        try:
+            res = yield self.handler.get_version_info(local_user, "1");
+        except errors.SynapseError as e:
+            self.assertEqual(e.code, 404);
+        self.assertEqual(res, None);        
+
+
+    @defer.inlineCallbacks
+    def test_get_room_keys(self):
+        pass
+
+    @defer.inlineCallbacks
+    def test_upload_room_keys(self):
+        pass
+
+    @defer.inlineCallbacks
+    def test_delete_room_keys(self):
+        pass

From 15d513f16fd272fb3763a42f05a8f296d4e1abf0 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Sun, 31 Dec 2017 14:35:25 +0000
Subject: [PATCH 366/830] fix idiocies and so make tests pass

---
 synapse/storage/e2e_room_keys.py              |  5 +++--
 .../storage/schema/delta/46/e2e_room_keys.sql |  2 +-
 tests/handlers/test_e2e_room_keys.py          | 19 +++++++++++--------
 3 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py
index 3c720f3b3e7c..e4d56b7c3757 100644
--- a/synapse/storage/e2e_room_keys.py
+++ b/synapse/storage/e2e_room_keys.py
@@ -207,6 +207,7 @@ def _get_e2e_room_keys_version_info_txn(txn):
                 this_version = version
 
             return self._simple_select_one_txn(
+                txn,
                 table="e2e_room_keys_versions",
                 keyvalues={
                     "user_id": user_id,
@@ -243,9 +244,9 @@ def _create_e2e_room_keys_version_txn(txn):
             )
             current_version = txn.fetchone()[0]
             if current_version is None:
-                current_version = 0
+                current_version = '0'
 
-            new_version = current_version + 1
+            new_version = str(int(current_version) + 1)
 
             self._simple_insert_txn(
                 txn,
diff --git a/synapse/storage/schema/delta/46/e2e_room_keys.sql b/synapse/storage/schema/delta/46/e2e_room_keys.sql
index 16499ac34c23..4531fd56ee8a 100644
--- a/synapse/storage/schema/delta/46/e2e_room_keys.sql
+++ b/synapse/storage/schema/delta/46/e2e_room_keys.sql
@@ -35,4 +35,4 @@ CREATE TABLE e2e_room_keys_versions (
     auth_data TEXT NOT NULL
 );
 
-CREATE UNIQUE INDEX e2e_room_keys_versions_user_idx ON e2e_room_keys_versions(user_id);
+CREATE UNIQUE INDEX e2e_room_keys_versions_idx ON e2e_room_keys_versions(user_id, version);
diff --git a/tests/handlers/test_e2e_room_keys.py b/tests/handlers/test_e2e_room_keys.py
index 9d3bef6db2f1..6f4b3a147a03 100644
--- a/tests/handlers/test_e2e_room_keys.py
+++ b/tests/handlers/test_e2e_room_keys.py
@@ -37,7 +37,7 @@ def setUp(self):
             handlers=None,
             replication_layer=mock.Mock(),
         )
-        self.handler = synapse.handlers.e2e_keys.E2eRoomKeysHandler(self.hs)
+        self.handler = synapse.handlers.e2e_room_keys.E2eRoomKeysHandler(self.hs)
 
 
     @defer.inlineCallbacks
@@ -46,6 +46,7 @@ def test_get_missing_current_version_info(self):
         if there is no version.
         """
         local_user = "@boris:" + self.hs.hostname
+        res = None
         try:
             res = yield self.handler.get_version_info(local_user);
         except errors.SynapseError as e:
@@ -58,6 +59,7 @@ def test_get_missing_version_info(self):
         if it doesn't exist.
         """
         local_user = "@boris:" + self.hs.hostname
+        res = None
         try:
             res = yield self.handler.get_version_info(local_user, "mrflibble");
         except errors.SynapseError as e:
@@ -69,7 +71,7 @@ def test_create_version(self):
         """Check that we can create and then retrieve versions.
         """
         local_user = "@boris:" + self.hs.hostname
-        res = yield self.handler.create_version(user_id, {
+        res = yield self.handler.create_version(local_user, {
             "algorithm": "m.megolm_backup.v1",
             "auth_data": "first_version_auth_data",
         });
@@ -92,7 +94,7 @@ def test_create_version(self):
         });
 
         # upload a new one...
-        res = yield self.handler.create_version(user_id, {
+        res = yield self.handler.create_version(local_user, {
             "algorithm": "m.megolm_backup.v1",
             "auth_data": "second_version_auth_data",
         });
@@ -111,7 +113,7 @@ def test_delete_version(self):
         """Check that we can create and then delete versions.
         """
         local_user = "@boris:" + self.hs.hostname
-        res = yield self.handler.create_version(user_id, {
+        res = yield self.handler.create_version(local_user, {
             "algorithm": "m.megolm_backup.v1",
             "auth_data": "first_version_auth_data",
         });
@@ -121,21 +123,22 @@ def test_delete_version(self):
         yield self.handler.delete_version(local_user, "1");
 
         # check that it's gone
+        res = None        
         try:
             res = yield self.handler.get_version_info(local_user, "1");
         except errors.SynapseError as e:
             self.assertEqual(e.code, 404);
-        self.assertEqual(res, None);        
+        self.assertEqual(res, None);
 
 
     @defer.inlineCallbacks
     def test_get_room_keys(self):
-        pass
+        yield None
 
     @defer.inlineCallbacks
     def test_upload_room_keys(self):
-        pass
+        yield None
 
     @defer.inlineCallbacks
     def test_delete_room_keys(self):
-        pass
+        yield None

From f6a3067868e7da2222484feb9aa421b1dcfecd0a Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Sun, 31 Dec 2017 14:42:10 +0000
Subject: [PATCH 367/830] linting

---
 tests/handlers/test_e2e_room_keys.py | 48 +++++++++++++---------------
 1 file changed, 23 insertions(+), 25 deletions(-)

diff --git a/tests/handlers/test_e2e_room_keys.py b/tests/handlers/test_e2e_room_keys.py
index 6f4b3a147a03..afe6ecf27bd4 100644
--- a/tests/handlers/test_e2e_room_keys.py
+++ b/tests/handlers/test_e2e_room_keys.py
@@ -39,7 +39,6 @@ def setUp(self):
         )
         self.handler = synapse.handlers.e2e_room_keys.E2eRoomKeysHandler(self.hs)
 
-
     @defer.inlineCallbacks
     def test_get_missing_current_version_info(self):
         """Check that we get a 404 if we ask for info about the current version
@@ -48,10 +47,10 @@ def test_get_missing_current_version_info(self):
         local_user = "@boris:" + self.hs.hostname
         res = None
         try:
-            res = yield self.handler.get_version_info(local_user);
+            res = yield self.handler.get_version_info(local_user)
         except errors.SynapseError as e:
-            self.assertEqual(e.code, 404);
-        self.assertEqual(res, None);
+            self.assertEqual(e.code, 404)
+        self.assertEqual(res, None)
 
     @defer.inlineCallbacks
     def test_get_missing_version_info(self):
@@ -61,10 +60,10 @@ def test_get_missing_version_info(self):
         local_user = "@boris:" + self.hs.hostname
         res = None
         try:
-            res = yield self.handler.get_version_info(local_user, "mrflibble");
+            res = yield self.handler.get_version_info(local_user, "mrflibble")
         except errors.SynapseError as e:
-            self.assertEqual(e.code, 404);
-        self.assertEqual(res, None);
+            self.assertEqual(e.code, 404)
+        self.assertEqual(res, None)
 
     @defer.inlineCallbacks
     def test_create_version(self):
@@ -74,39 +73,39 @@ def test_create_version(self):
         res = yield self.handler.create_version(local_user, {
             "algorithm": "m.megolm_backup.v1",
             "auth_data": "first_version_auth_data",
-        });
-        self.assertEqual(res, "1");
+        })
+        self.assertEqual(res, "1")
 
         # check we can retrieve it as the current version
-        res = yield self.handler.get_version_info(local_user);
+        res = yield self.handler.get_version_info(local_user)
         self.assertDictEqual(res, {
             "version": "1",
             "algorithm": "m.megolm_backup.v1",
             "auth_data": "first_version_auth_data",
-        });
+        })
 
         # check we can retrieve it as a specific version
-        res = yield self.handler.get_version_info(local_user, "1");
+        res = yield self.handler.get_version_info(local_user, "1")
         self.assertDictEqual(res, {
             "version": "1",
             "algorithm": "m.megolm_backup.v1",
             "auth_data": "first_version_auth_data",
-        });
+        })
 
         # upload a new one...
         res = yield self.handler.create_version(local_user, {
             "algorithm": "m.megolm_backup.v1",
             "auth_data": "second_version_auth_data",
-        });
-        self.assertEqual(res, "2");
+        })
+        self.assertEqual(res, "2")
 
         # check we can retrieve it as the current version
-        res = yield self.handler.get_version_info(local_user);
+        res = yield self.handler.get_version_info(local_user)
         self.assertDictEqual(res, {
             "version": "2",
             "algorithm": "m.megolm_backup.v1",
             "auth_data": "second_version_auth_data",
-        });
+        })
 
     @defer.inlineCallbacks
     def test_delete_version(self):
@@ -116,20 +115,19 @@ def test_delete_version(self):
         res = yield self.handler.create_version(local_user, {
             "algorithm": "m.megolm_backup.v1",
             "auth_data": "first_version_auth_data",
-        });
-        self.assertEqual(res, "1");
+        })
+        self.assertEqual(res, "1")
 
         # check we can delete it
-        yield self.handler.delete_version(local_user, "1");
+        yield self.handler.delete_version(local_user, "1")
 
         # check that it's gone
-        res = None        
+        res = None
         try:
-            res = yield self.handler.get_version_info(local_user, "1");
+            res = yield self.handler.get_version_info(local_user, "1")
         except errors.SynapseError as e:
-            self.assertEqual(e.code, 404);
-        self.assertEqual(res, None);
-
+            self.assertEqual(e.code, 404)
+        self.assertEqual(res, None)
 
     @defer.inlineCallbacks
     def test_get_room_keys(self):

From fe87890b18f57f0268bd65aeca881e7817bbe9e4 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Sun, 31 Dec 2017 17:47:11 +0000
Subject: [PATCH 368/830] implement remaining tests and make them work

---
 synapse/handlers/e2e_room_keys.py         |  35 ++-
 synapse/rest/client/v2_alpha/room_keys.py |   6 +
 synapse/storage/e2e_room_keys.py          |   3 +-
 tests/handlers/test_e2e_room_keys.py      | 276 ++++++++++++++++++++--
 4 files changed, 287 insertions(+), 33 deletions(-)

diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py
index f08d80da3e5e..09c2888db638 100644
--- a/synapse/handlers/e2e_room_keys.py
+++ b/synapse/handlers/e2e_room_keys.py
@@ -58,6 +58,10 @@ def get_room_keys(self, user_id, version, room_id=None, session_id=None):
             results = yield self.store.get_e2e_room_keys(
                 user_id, version, room_id, session_id
             )
+
+            if results['rooms'] == {}:
+                raise SynapseError(404, "No room_keys found")
+
             defer.returnValue(results)
 
     @defer.inlineCallbacks
@@ -109,9 +113,10 @@ def upload_room_keys(self, user_id, version, room_keys):
 
         # XXX: perhaps we should use a finer grained lock here?
         with (yield self._upload_linearizer.queue(user_id)):
+
             # Check that the version we're trying to upload is the current version
             try:
-                version_info = yield self.get_version_info(user_id)
+                version_info = yield self._get_version_info_unlocked(user_id)
             except StoreError as e:
                 if e.code == 404:
                     raise SynapseError(404, "Version '%s' not found" % (version,))
@@ -119,16 +124,23 @@ def upload_room_keys(self, user_id, version, room_keys):
                     raise e
 
             if version_info['version'] != version:
-                raise RoomKeysVersionError(current_version=version_info.version)
+                # Check that the version we're trying to upload actually exists
+                try:
+                    version_info = yield self._get_version_info_unlocked(user_id, version)
+                    # if we get this far, the version must exist
+                    raise RoomKeysVersionError(current_version=version_info['version'])
+                except StoreError as e:
+                    if e.code == 404:
+                        raise SynapseError(404, "Version '%s' not found" % (version,))
+                    else:
+                        raise e
 
             # go through the room_keys.
             # XXX: this should/could be done concurrently, given we're in a lock.
             for room_id, room in room_keys['rooms'].iteritems():
                 for session_id, session in room['sessions'].iteritems():
-                    room_key = session[session_id]
-
                     yield self._upload_room_key(
-                        user_id, version, room_id, session_id, room_key
+                        user_id, version, room_id, session_id, session
                     )
 
     @defer.inlineCallbacks
@@ -242,8 +254,17 @@ def get_version_info(self, user_id, version=None):
         """
 
         with (yield self._upload_linearizer.queue(user_id)):
-            results = yield self.store.get_e2e_room_keys_version_info(user_id)
-            defer.returnValue(results)
+            res = yield self._get_version_info_unlocked(user_id, version)
+            defer.returnValue(res)
+
+    @defer.inlineCallbacks
+    def _get_version_info_unlocked(self, user_id, version=None):
+        """Get the info about a given version of the user's backup
+        without obtaining the upload_linearizer lock.  For params see get_version_info
+        """
+
+        results = yield self.store.get_e2e_room_keys_version_info(user_id, version)
+        defer.returnValue(results)
 
     @defer.inlineCallbacks
     def delete_version(self, user_id, version):
diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py
index ca69ced1e31c..8f10e4e1cd72 100644
--- a/synapse/rest/client/v2_alpha/room_keys.py
+++ b/synapse/rest/client/v2_alpha/room_keys.py
@@ -204,6 +204,12 @@ def on_GET(self, request, room_id, session_id):
         room_keys = yield self.e2e_room_keys_handler.get_room_keys(
             user_id, version, room_id, session_id
         )
+
+        if session_id:
+            room_keys = room_keys['rooms'][room_id]['sessions'][session_id]
+        elif room_id:
+            room_keys = room_keys['rooms'][room_id]
+
         defer.returnValue((200, room_keys))
 
     @defer.inlineCallbacks
diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py
index e4d56b7c3757..8e8e4e457ce4 100644
--- a/synapse/storage/e2e_room_keys.py
+++ b/synapse/storage/e2e_room_keys.py
@@ -58,6 +58,7 @@ def get_e2e_room_key(self, user_id, version, room_id, session_id):
 
         defer.returnValue(row)
 
+    @defer.inlineCallbacks
     def set_e2e_room_key(self, user_id, version, room_id, session_id, room_key):
         """Replaces or inserts the encrypted E2E room key for a given session in
         a given backup
@@ -135,7 +136,7 @@ def get_e2e_room_keys(
             desc="get_e2e_room_keys",
         )
 
-        sessions = {}
+        sessions = { 'rooms': {} }
         for row in rows:
             room_entry = sessions['rooms'].setdefault(row['room_id'], {"sessions": {}})
             room_entry['sessions'][row['session_id']] = {
diff --git a/tests/handlers/test_e2e_room_keys.py b/tests/handlers/test_e2e_room_keys.py
index afe6ecf27bd4..3cbfd6f9d0c2 100644
--- a/tests/handlers/test_e2e_room_keys.py
+++ b/tests/handlers/test_e2e_room_keys.py
@@ -17,6 +17,7 @@
 import mock
 from synapse.api import errors
 from twisted.internet import defer
+import copy
 
 import synapse.api.errors
 import synapse.handlers.e2e_room_keys
@@ -25,6 +26,22 @@
 from tests import unittest, utils
 
 
+# sample room_key data for use in the tests
+room_keys = {
+    "rooms": {
+        "!abc:matrix.org": {
+            "sessions": {
+                "c0ff33": {
+                    "first_message_index": 1,
+                    "forwarded_count": 1,
+                    "is_verified": False,
+                    "session_data": "SSBBTSBBIEZJU0gK"
+                }
+            }
+        }
+    }
+}
+
 class E2eRoomKeysHandlerTestCase(unittest.TestCase):
     def __init__(self, *args, **kwargs):
         super(E2eRoomKeysHandlerTestCase, self).__init__(*args, **kwargs)
@@ -38,46 +55,44 @@ def setUp(self):
             replication_layer=mock.Mock(),
         )
         self.handler = synapse.handlers.e2e_room_keys.E2eRoomKeysHandler(self.hs)
+        self.local_user = "@boris:" + self.hs.hostname;
 
     @defer.inlineCallbacks
     def test_get_missing_current_version_info(self):
         """Check that we get a 404 if we ask for info about the current version
         if there is no version.
         """
-        local_user = "@boris:" + self.hs.hostname
         res = None
         try:
-            res = yield self.handler.get_version_info(local_user)
+            yield self.handler.get_version_info(self.local_user)
         except errors.SynapseError as e:
-            self.assertEqual(e.code, 404)
-        self.assertEqual(res, None)
+            res = e.code
+        self.assertEqual(res, 404)
 
     @defer.inlineCallbacks
     def test_get_missing_version_info(self):
         """Check that we get a 404 if we ask for info about a specific version
         if it doesn't exist.
         """
-        local_user = "@boris:" + self.hs.hostname
         res = None
         try:
-            res = yield self.handler.get_version_info(local_user, "mrflibble")
+            yield self.handler.get_version_info(self.local_user, "bogus_version")
         except errors.SynapseError as e:
-            self.assertEqual(e.code, 404)
-        self.assertEqual(res, None)
+            res = e.code
+        self.assertEqual(res, 404)
 
     @defer.inlineCallbacks
     def test_create_version(self):
         """Check that we can create and then retrieve versions.
         """
-        local_user = "@boris:" + self.hs.hostname
-        res = yield self.handler.create_version(local_user, {
+        res = yield self.handler.create_version(self.local_user, {
             "algorithm": "m.megolm_backup.v1",
             "auth_data": "first_version_auth_data",
         })
         self.assertEqual(res, "1")
 
         # check we can retrieve it as the current version
-        res = yield self.handler.get_version_info(local_user)
+        res = yield self.handler.get_version_info(self.local_user)
         self.assertDictEqual(res, {
             "version": "1",
             "algorithm": "m.megolm_backup.v1",
@@ -85,7 +100,7 @@ def test_create_version(self):
         })
 
         # check we can retrieve it as a specific version
-        res = yield self.handler.get_version_info(local_user, "1")
+        res = yield self.handler.get_version_info(self.local_user, "1")
         self.assertDictEqual(res, {
             "version": "1",
             "algorithm": "m.megolm_backup.v1",
@@ -93,14 +108,14 @@ def test_create_version(self):
         })
 
         # upload a new one...
-        res = yield self.handler.create_version(local_user, {
+        res = yield self.handler.create_version(self.local_user, {
             "algorithm": "m.megolm_backup.v1",
             "auth_data": "second_version_auth_data",
         })
         self.assertEqual(res, "2")
 
         # check we can retrieve it as the current version
-        res = yield self.handler.get_version_info(local_user)
+        res = yield self.handler.get_version_info(self.local_user)
         self.assertDictEqual(res, {
             "version": "2",
             "algorithm": "m.megolm_backup.v1",
@@ -111,32 +126,243 @@ def test_create_version(self):
     def test_delete_version(self):
         """Check that we can create and then delete versions.
         """
-        local_user = "@boris:" + self.hs.hostname
-        res = yield self.handler.create_version(local_user, {
+        res = yield self.handler.create_version(self.local_user, {
             "algorithm": "m.megolm_backup.v1",
             "auth_data": "first_version_auth_data",
         })
         self.assertEqual(res, "1")
 
         # check we can delete it
-        yield self.handler.delete_version(local_user, "1")
+        yield self.handler.delete_version(self.local_user, "1")
 
         # check that it's gone
         res = None
         try:
-            res = yield self.handler.get_version_info(local_user, "1")
+            yield self.handler.get_version_info(self.local_user, "1")
         except errors.SynapseError as e:
-            self.assertEqual(e.code, 404)
-        self.assertEqual(res, None)
+            res = e.code
+        self.assertEqual(res, 404)
 
     @defer.inlineCallbacks
-    def test_get_room_keys(self):
-        yield None
+    def test_get_missing_room_keys(self):
+        """Check that we get a 404 on querying missing room_keys
+        """
+        res = None
+        try:
+            yield self.handler.get_room_keys(self.local_user, "bogus_version")
+        except errors.SynapseError as e:
+            res = e.code
+        self.assertEqual(res, 404)
+
+        # check we also get a 404 even if the version is valid
+        version = yield self.handler.create_version(self.local_user, {
+            "algorithm": "m.megolm_backup.v1",
+            "auth_data": "first_version_auth_data",
+        })
+        self.assertEqual(version, "1")
+
+        res = None
+        try:
+            yield self.handler.get_room_keys(self.local_user, version)
+        except errors.SynapseError as e:
+            res = e.code
+        self.assertEqual(res, 404)
+
+    # TODO: test the locking semantics when uploading room_keys,
+    # although this is probably best done in sytest
 
     @defer.inlineCallbacks
-    def test_upload_room_keys(self):
-        yield None
+    def test_upload_room_keys_no_versions(self):
+        """Check that we get a 404 on uploading keys when no versions are defined
+        """
+        res = None
+        try:
+            yield self.handler.upload_room_keys(self.local_user, "no_version", room_keys)
+        except errors.SynapseError as e:
+            res = e.code
+        self.assertEqual(res, 404)
+
+    @defer.inlineCallbacks
+    def test_upload_room_keys_bogus_version(self):
+        """Check that we get a 404 on uploading keys when an nonexistent version is specified
+        """
+        version = yield self.handler.create_version(self.local_user, {
+            "algorithm": "m.megolm_backup.v1",
+            "auth_data": "first_version_auth_data",
+        })
+        self.assertEqual(version, "1")
+
+        res = None
+        try:
+            yield self.handler.upload_room_keys(self.local_user, "bogus_version", room_keys)
+        except errors.SynapseError as e:
+            res = e.code
+        self.assertEqual(res, 404)
+
+    @defer.inlineCallbacks
+    def test_upload_room_keys_wrong_version(self):
+        """Check that we get a 403 on uploading keys for an old version
+        """
+        version = yield self.handler.create_version(self.local_user, {
+            "algorithm": "m.megolm_backup.v1",
+            "auth_data": "first_version_auth_data",
+        })
+        self.assertEqual(version, "1")
+
+        version = yield self.handler.create_version(self.local_user, {
+            "algorithm": "m.megolm_backup.v1",
+            "auth_data": "second_version_auth_data",
+        })
+        self.assertEqual(version, "2")
+
+        res = None
+        try:
+            yield self.handler.upload_room_keys(self.local_user, "1", room_keys)
+        except errors.SynapseError as e:
+            res = e.code
+        self.assertEqual(res, 403)
+
+    @defer.inlineCallbacks
+    def test_upload_room_keys_insert(self):
+        """Check that we can insert and retrieve keys for a session
+        """
+        version = yield self.handler.create_version(self.local_user, {
+            "algorithm": "m.megolm_backup.v1",
+            "auth_data": "first_version_auth_data",
+        })
+        self.assertEqual(version, "1")
+
+        yield self.handler.upload_room_keys(self.local_user, version, room_keys)
+
+        res = yield self.handler.get_room_keys(self.local_user, version)
+        self.assertDictEqual(res, room_keys)
+
+        # check getting room_keys for a given room
+        res = yield self.handler.get_room_keys(
+            self.local_user,
+            version,
+            room_id="!abc:matrix.org"
+        )
+        self.assertDictEqual(res, room_keys)
+
+        # check getting room_keys for a given session_id
+        res = yield self.handler.get_room_keys(
+            self.local_user,
+            version,
+            room_id="!abc:matrix.org",
+            session_id="c0ff33",
+        )
+        self.assertDictEqual(res, room_keys)
+
+    @defer.inlineCallbacks
+    def test_upload_room_keys_merge(self):
+        """Check that we can upload a new room_key for an existing session and
+        have it correctly merged"""
+        version = yield self.handler.create_version(self.local_user, {
+            "algorithm": "m.megolm_backup.v1",
+            "auth_data": "first_version_auth_data",
+        })
+        self.assertEqual(version, "1")
+
+        yield self.handler.upload_room_keys(self.local_user, version, room_keys)
+
+        new_room_keys = copy.deepcopy(room_keys)
+
+        # test that increasing the message_index doesn't replace the existing session
+        new_room_keys['rooms']['!abc:matrix.org']['sessions']['c0ff33']['first_message_index'] = 2
+        new_room_keys['rooms']['!abc:matrix.org']['sessions']['c0ff33']['session_data'] = 'new'
+        yield self.handler.upload_room_keys(self.local_user, version, new_room_keys)
+
+        res = yield self.handler.get_room_keys(self.local_user, version)
+        self.assertEqual(
+            res['rooms']['!abc:matrix.org']['sessions']['c0ff33']['session_data'],
+            "SSBBTSBBIEZJU0gK"
+        )
+
+        # test that marking the session as verified however /does/ replace it
+        new_room_keys['rooms']['!abc:matrix.org']['sessions']['c0ff33']['is_verified'] = True
+        yield self.handler.upload_room_keys(self.local_user, version, new_room_keys)
+
+        res = yield self.handler.get_room_keys(self.local_user, version)
+        self.assertEqual(
+            res['rooms']['!abc:matrix.org']['sessions']['c0ff33']['session_data'],
+            "new"
+        )
+
+        # test that a session with a higher forwarded_count doesn't replace one
+        # with a lower forwarding count
+        new_room_keys['rooms']['!abc:matrix.org']['sessions']['c0ff33']['forwarded_count'] = 2
+        new_room_keys['rooms']['!abc:matrix.org']['sessions']['c0ff33']['session_data'] = 'other'
+        yield self.handler.upload_room_keys(self.local_user, version, new_room_keys)
+
+        res = yield self.handler.get_room_keys(self.local_user, version)
+        self.assertEqual(
+            res['rooms']['!abc:matrix.org']['sessions']['c0ff33']['session_data'],
+            "new"
+        )
+
+        # TODO: check edge cases as well as the common variations here
 
     @defer.inlineCallbacks
     def test_delete_room_keys(self):
-        yield None
+        """Check that we can insert and delete keys for a session
+        """
+        version = yield self.handler.create_version(self.local_user, {
+            "algorithm": "m.megolm_backup.v1",
+            "auth_data": "first_version_auth_data",
+        })
+        self.assertEqual(version, "1")
+
+        # check for bulk-delete
+        yield self.handler.upload_room_keys(self.local_user, version, room_keys)
+        yield self.handler.delete_room_keys(self.local_user, version)
+        res = None
+        try:
+            yield self.handler.get_room_keys(
+                self.local_user,
+                version,
+                room_id="!abc:matrix.org",
+                session_id="c0ff33",
+            )
+        except errors.SynapseError as e:
+            res = e.code
+        self.assertEqual(res, 404)
+
+        # check for bulk-delete per room
+        yield self.handler.upload_room_keys(self.local_user, version, room_keys)
+        yield self.handler.delete_room_keys(
+            self.local_user,
+            version,
+            room_id="!abc:matrix.org",
+        )
+        res = None
+        try:
+            yield self.handler.get_room_keys(
+                self.local_user,
+                version,
+                room_id="!abc:matrix.org",
+                session_id="c0ff33",
+            )
+        except errors.SynapseError as e:
+            res = e.code
+        self.assertEqual(res, 404)
+
+        # check for bulk-delete per session
+        yield self.handler.upload_room_keys(self.local_user, version, room_keys)
+        yield self.handler.delete_room_keys(
+            self.local_user,
+            version,
+            room_id="!abc:matrix.org",
+            session_id="c0ff33",
+        )
+        res = None
+        try:
+            yield self.handler.get_room_keys(
+                self.local_user,
+                version,
+                room_id="!abc:matrix.org",
+                session_id="c0ff33",
+            )
+        except errors.SynapseError as e:
+            res = e.code
+        self.assertEqual(res, 404)

From edc427a35187e462458d188039e761ffdb7e486d Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Sun, 31 Dec 2017 17:50:55 +0000
Subject: [PATCH 369/830] flake8

---
 synapse/storage/e2e_room_keys.py     |  2 +-
 tests/handlers/test_e2e_room_keys.py | 21 +++++++++++++--------
 2 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py
index 8e8e4e457ce4..c2f226396d51 100644
--- a/synapse/storage/e2e_room_keys.py
+++ b/synapse/storage/e2e_room_keys.py
@@ -136,7 +136,7 @@ def get_e2e_room_keys(
             desc="get_e2e_room_keys",
         )
 
-        sessions = { 'rooms': {} }
+        sessions = {'rooms': {}}
         for row in rows:
             room_entry = sessions['rooms'].setdefault(row['room_id'], {"sessions": {}})
             room_entry['sessions'][row['session_id']] = {
diff --git a/tests/handlers/test_e2e_room_keys.py b/tests/handlers/test_e2e_room_keys.py
index 3cbfd6f9d0c2..6e43543ed96d 100644
--- a/tests/handlers/test_e2e_room_keys.py
+++ b/tests/handlers/test_e2e_room_keys.py
@@ -42,6 +42,7 @@
     }
 }
 
+
 class E2eRoomKeysHandlerTestCase(unittest.TestCase):
     def __init__(self, *args, **kwargs):
         super(E2eRoomKeysHandlerTestCase, self).__init__(*args, **kwargs)
@@ -55,7 +56,7 @@ def setUp(self):
             replication_layer=mock.Mock(),
         )
         self.handler = synapse.handlers.e2e_room_keys.E2eRoomKeysHandler(self.hs)
-        self.local_user = "@boris:" + self.hs.hostname;
+        self.local_user = "@boris:" + self.hs.hostname
 
     @defer.inlineCallbacks
     def test_get_missing_current_version_info(self):
@@ -184,7 +185,8 @@ def test_upload_room_keys_no_versions(self):
 
     @defer.inlineCallbacks
     def test_upload_room_keys_bogus_version(self):
-        """Check that we get a 404 on uploading keys when an nonexistent version is specified
+        """Check that we get a 404 on uploading keys when an nonexistent version
+        is specified
         """
         version = yield self.handler.create_version(self.local_user, {
             "algorithm": "m.megolm_backup.v1",
@@ -194,7 +196,9 @@ def test_upload_room_keys_bogus_version(self):
 
         res = None
         try:
-            yield self.handler.upload_room_keys(self.local_user, "bogus_version", room_keys)
+            yield self.handler.upload_room_keys(
+                self.local_user, "bogus_version", room_keys
+            )
         except errors.SynapseError as e:
             res = e.code
         self.assertEqual(res, 404)
@@ -267,10 +271,11 @@ def test_upload_room_keys_merge(self):
         yield self.handler.upload_room_keys(self.local_user, version, room_keys)
 
         new_room_keys = copy.deepcopy(room_keys)
+        new_room_key = new_room_keys['rooms']['!abc:matrix.org']['sessions']['c0ff33']
 
         # test that increasing the message_index doesn't replace the existing session
-        new_room_keys['rooms']['!abc:matrix.org']['sessions']['c0ff33']['first_message_index'] = 2
-        new_room_keys['rooms']['!abc:matrix.org']['sessions']['c0ff33']['session_data'] = 'new'
+        new_room_key['first_message_index'] = 2
+        new_room_key['session_data'] = 'new'
         yield self.handler.upload_room_keys(self.local_user, version, new_room_keys)
 
         res = yield self.handler.get_room_keys(self.local_user, version)
@@ -280,7 +285,7 @@ def test_upload_room_keys_merge(self):
         )
 
         # test that marking the session as verified however /does/ replace it
-        new_room_keys['rooms']['!abc:matrix.org']['sessions']['c0ff33']['is_verified'] = True
+        new_room_key['is_verified'] = True
         yield self.handler.upload_room_keys(self.local_user, version, new_room_keys)
 
         res = yield self.handler.get_room_keys(self.local_user, version)
@@ -291,8 +296,8 @@ def test_upload_room_keys_merge(self):
 
         # test that a session with a higher forwarded_count doesn't replace one
         # with a lower forwarding count
-        new_room_keys['rooms']['!abc:matrix.org']['sessions']['c0ff33']['forwarded_count'] = 2
-        new_room_keys['rooms']['!abc:matrix.org']['sessions']['c0ff33']['session_data'] = 'other'
+        new_room_key['forwarded_count'] = 2
+        new_room_key['session_data'] = 'other'
         yield self.handler.upload_room_keys(self.local_user, version, new_room_keys)
 
         res = yield self.handler.get_room_keys(self.local_user, version)

From 72788cf9c1f018a5346a8f9204c6bfe058289fd2 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Sun, 7 Jan 2018 23:45:26 +0000
Subject: [PATCH 370/830] support DELETE /version with no args

---
 synapse/handlers/e2e_room_keys.py         | 2 +-
 synapse/rest/client/v2_alpha/room_keys.py | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py
index 09c2888db638..a43fc7fc7e5b 100644
--- a/synapse/handlers/e2e_room_keys.py
+++ b/synapse/handlers/e2e_room_keys.py
@@ -267,7 +267,7 @@ def _get_version_info_unlocked(self, user_id, version=None):
         defer.returnValue(results)
 
     @defer.inlineCallbacks
-    def delete_version(self, user_id, version):
+    def delete_version(self, user_id, version=None):
         """Deletes a given version of the user's e2e_room_keys backup
 
         Args:
diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py
index 8f10e4e1cd72..63b1f62f90e3 100644
--- a/synapse/rest/client/v2_alpha/room_keys.py
+++ b/synapse/rest/client/v2_alpha/room_keys.py
@@ -308,7 +308,7 @@ def on_GET(self, request, version):
         It takes out an exclusive lock on this user's room_key backups, to ensure
         clients only upload to the current backup.
 
-        Returns 404 is the given version does not exist.
+        Returns 404 if the given version does not exist.
 
         GET /room_keys/version/12345 HTTP/1.1
         {
@@ -330,7 +330,8 @@ def on_GET(self, request, version):
     def on_DELETE(self, request, version):
         """
         Delete the information about a given version of the user's
-        room_keys backup.  Doesn't delete the actual room data.
+        room_keys backup.  If the version part is missing, deletes the most
+        current backup version (if any). Doesn't delete the actual room data.
 
         DELETE /room_keys/version/12345 HTTP/1.1
         HTTP/1.1 200 OK

From 66a4ca1d28c2c2e22a0343e6db0f5a2bce9ec987 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Sun, 7 Jan 2018 23:45:55 +0000
Subject: [PATCH 371/830] 404 nicely if you try to interact with a missing
 current version

---
 synapse/storage/e2e_room_keys.py     | 51 +++++++++++++++++++---------
 tests/handlers/test_e2e_room_keys.py | 22 ++++++++++++
 2 files changed, 57 insertions(+), 16 deletions(-)

diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py
index c2f226396d51..d82f223e8665 100644
--- a/synapse/storage/e2e_room_keys.py
+++ b/synapse/storage/e2e_room_keys.py
@@ -184,6 +184,17 @@ def delete_e2e_room_keys(
             desc="delete_e2e_room_keys",
         )
 
+    @staticmethod
+    def _get_current_version(txn, user_id):
+        txn.execute(
+            "SELECT MAX(version) FROM e2e_room_keys_versions WHERE user_id=?",
+            (user_id,)
+        )
+        row = txn.fetchone()
+        if not row:
+            raise StoreError(404, 'No current backup version')
+        return row[0]
+
     def get_e2e_room_keys_version_info(self, user_id, version=None):
         """Get info metadata about a version of our room_keys backup.
 
@@ -199,11 +210,7 @@ def get_e2e_room_keys_version_info(self, user_id, version=None):
 
         def _get_e2e_room_keys_version_info_txn(txn):
             if version is None:
-                txn.execute(
-                    "SELECT MAX(version) FROM e2e_room_keys_versions WHERE user_id=?",
-                    (user_id,)
-                )
-                this_version = txn.fetchone()[0]
+                this_version = self._get_current_version(txn, user_id)
             else:
                 this_version = version
 
@@ -266,23 +273,35 @@ def _create_e2e_room_keys_version_txn(txn):
             "create_e2e_room_keys_version_txn", _create_e2e_room_keys_version_txn
         )
 
-    @defer.inlineCallbacks
-    def delete_e2e_room_keys_version(self, user_id, version):
+    def delete_e2e_room_keys_version(self, user_id, version=None):
         """Delete a given backup version of the user's room keys.
         Doesn't delete their actual key data.
 
         Args:
             user_id(str): the user whose backup version we're deleting
-            version(str): the ID of the backup version we're deleting
+            version(str): Optional. the version ID of the backup version we're deleting
+                If missing, we delete the current backup version info.
+        Raises:
+            StoreError: with code 404 if there are no e2e_room_keys_versions present,
+                or if the version requested doesn't exist.
         """
 
-        keyvalues = {
-            "user_id": user_id,
-            "version": version,
-        }
+        def _delete_e2e_room_keys_version_txn(txn):
+            if version is None:
+                this_version = self._get_current_version(txn, user_id)
+            else:
+                this_version = version
 
-        yield self._simple_delete(
-            table="e2e_room_keys_versions",
-            keyvalues=keyvalues,
-            desc="delete_e2e_room_keys_version",
+            return self._simple_delete_one_txn(
+                txn,
+                table="e2e_room_keys_versions",
+                keyvalues={
+                    "user_id": user_id,
+                    "version": this_version,
+                },
+            )
+
+        return self.runInteraction(
+            "delete_e2e_room_keys_version",
+            _delete_e2e_room_keys_version_txn
         )
diff --git a/tests/handlers/test_e2e_room_keys.py b/tests/handlers/test_e2e_room_keys.py
index 6e43543ed96d..8bfffb5c0e51 100644
--- a/tests/handlers/test_e2e_room_keys.py
+++ b/tests/handlers/test_e2e_room_keys.py
@@ -123,6 +123,28 @@ def test_create_version(self):
             "auth_data": "second_version_auth_data",
         })
 
+    @defer.inlineCallbacks
+    def test_delete_missing_version(self):
+        """Check that we get a 404 on deleting nonexistent versions
+        """
+        res = None
+        try:
+            yield self.handler.delete_version(self.local_user, "1")
+        except errors.SynapseError as e:
+            res = e.code
+        self.assertEqual(res, 404)
+
+    @defer.inlineCallbacks
+    def test_delete_missing_current_version(self):
+        """Check that we get a 404 on deleting nonexistent current version
+        """
+        res = None
+        try:
+            yield self.handler.delete_version(self.local_user)
+        except errors.SynapseError as e:
+            res = e.code
+        self.assertEqual(res, 404)
+
     @defer.inlineCallbacks
     def test_delete_version(self):
         """Check that we can create and then delete versions.

From 54ac18e8327e689b9f91b93ad8caac2f6b3dd29c Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Sun, 7 Jan 2018 23:48:22 +0000
Subject: [PATCH 372/830] use parse_string

---
 synapse/rest/client/v2_alpha/room_keys.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py
index 63b1f62f90e3..afb8a36dd0df 100644
--- a/synapse/rest/client/v2_alpha/room_keys.py
+++ b/synapse/rest/client/v2_alpha/room_keys.py
@@ -123,7 +123,7 @@ def on_PUT(self, request, room_id, session_id):
         requester = yield self.auth.get_user_by_req(request, allow_guest=False)
         user_id = requester.user.to_string()
         body = parse_json_object_from_request(request)
-        version = request.args.get("version")[0]
+        version = parse_string(request, "version")
 
         if session_id:
             body = {
@@ -199,7 +199,7 @@ def on_GET(self, request, room_id, session_id):
         """
         requester = yield self.auth.get_user_by_req(request, allow_guest=False)
         user_id = requester.user.to_string()
-        version = request.args.get("version")[0]
+        version = parse_string(request, "version")
 
         room_keys = yield self.e2e_room_keys_handler.get_room_keys(
             user_id, version, room_id, session_id
@@ -229,7 +229,7 @@ def on_DELETE(self, request, room_id, session_id):
 
         requester = yield self.auth.get_user_by_req(request, allow_guest=False)
         user_id = requester.user.to_string()
-        version = request.args.get("version")[0]
+        version = parse_string(request, "version")
 
         yield self.e2e_room_keys_handler.delete_room_keys(
             user_id, version, room_id, session_id

From f0cede5556a54ece6cd5289856bb230f8009c4fa Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Sun, 7 Jan 2018 23:58:32 +0000
Subject: [PATCH 373/830] missing import

---
 synapse/storage/e2e_room_keys.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py
index d82f223e8665..089989fcfa5f 100644
--- a/synapse/storage/e2e_room_keys.py
+++ b/synapse/storage/e2e_room_keys.py
@@ -14,6 +14,7 @@
 # limitations under the License.
 
 from twisted.internet import defer
+from synapse.api.errors import StoreError
 
 from ._base import SQLBaseStore
 

From 4f7064f6b5a1822067bfbf358317bc1dbb51b9c6 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Sun, 7 Jan 2018 23:59:07 +0000
Subject: [PATCH 374/830] missing import

---
 synapse/rest/client/v2_alpha/room_keys.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py
index afb8a36dd0df..9f0172e6f5ee 100644
--- a/synapse/rest/client/v2_alpha/room_keys.py
+++ b/synapse/rest/client/v2_alpha/room_keys.py
@@ -19,7 +19,7 @@
 
 from synapse.api.errors import SynapseError
 from synapse.http.servlet import (
-    RestServlet, parse_json_object_from_request
+    RestServlet, parse_json_object_from_request, parse_string
 )
 from ._base import client_v2_patterns
 

From 99dd975dae7baaaef2a3b0a92fa51965b121ae34 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Mon, 13 Aug 2018 16:47:46 +1000
Subject: [PATCH 375/830] Run tests under PostgreSQL (#3423)

---
 .travis.yml                                 |  10 ++
 changelog.d/3423.misc                       |   1 +
 synapse/handlers/presence.py                |   5 +
 synapse/storage/client_ips.py               |   5 +
 tests/__init__.py                           |   3 +
 tests/api/test_auth.py                      |   2 +-
 tests/api/test_filtering.py                 |   5 +-
 tests/crypto/test_keyring.py                |   6 +-
 tests/handlers/test_auth.py                 |   2 +-
 tests/handlers/test_device.py               |   2 +-
 tests/handlers/test_directory.py            |   1 +
 tests/handlers/test_e2e_keys.py             |   2 +-
 tests/handlers/test_profile.py              |   1 +
 tests/handlers/test_register.py             |   1 +
 tests/handlers/test_typing.py               |   1 +
 tests/replication/slave/storage/_base.py    |   1 +
 tests/rest/client/v1/test_admin.py          |   2 +-
 tests/rest/client/v1/test_events.py         |   1 +
 tests/rest/client/v1/test_profile.py        |   1 +
 tests/rest/client/v1/test_register.py       |   2 +-
 tests/rest/client/v1/test_rooms.py          |   1 +
 tests/rest/client/v1/test_typing.py         |   1 +
 tests/rest/client/v2_alpha/test_filter.py   |   2 +-
 tests/rest/client/v2_alpha/test_register.py |   2 +-
 tests/rest/client/v2_alpha/test_sync.py     |   2 +-
 tests/server.py                             |  10 +-
 tests/storage/test_appservice.py            |  13 +-
 tests/storage/test_background_update.py     |   4 +-
 tests/storage/test_client_ips.py            |   2 +-
 tests/storage/test_devices.py               |   2 +-
 tests/storage/test_directory.py             |   2 +-
 tests/storage/test_end_to_end_keys.py       |   3 +-
 tests/storage/test_event_federation.py      |   2 +-
 tests/storage/test_event_push_actions.py    |   2 +-
 tests/storage/test_keys.py                  |   2 +-
 tests/storage/test_monthly_active_users.py  |   2 +-
 tests/storage/test_presence.py              |   2 +-
 tests/storage/test_profile.py               |   2 +-
 tests/storage/test_redaction.py             |   2 +-
 tests/storage/test_registration.py          |   2 +-
 tests/storage/test_room.py                  |   4 +-
 tests/storage/test_roommember.py            |   2 +-
 tests/storage/test_state.py                 |   2 +-
 tests/storage/test_user_directory.py        |   2 +-
 tests/test_federation.py                    |   5 +-
 tests/test_server.py                        |   2 +-
 tests/test_visibility.py                    |   2 +-
 tests/utils.py                              | 133 +++++++++++++++++---
 tox.ini                                     |  20 ++-
 49 files changed, 227 insertions(+), 59 deletions(-)
 create mode 100644 changelog.d/3423.misc

diff --git a/.travis.yml b/.travis.yml
index b34b17af758f..318701c9f89f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,6 +8,9 @@ before_script:
   - git remote set-branches --add origin develop
   - git fetch origin develop
 
+services:
+  - postgresql
+
 matrix:
   fast_finish: true
   include:
@@ -20,6 +23,9 @@ matrix:
   - python: 2.7
     env: TOX_ENV=py27
 
+  - python: 2.7
+    env: TOX_ENV=py27-postgres TRIAL_FLAGS="-j 4"
+
   - python: 3.6
     env: TOX_ENV=py36
 
@@ -29,6 +35,10 @@ matrix:
   - python: 3.6
     env: TOX_ENV=check-newsfragment
 
+  allow_failures:
+  - python: 2.7
+    env: TOX_ENV=py27-postgres TRIAL_FLAGS="-j 4"
+
 install:
   - pip install tox
 
diff --git a/changelog.d/3423.misc b/changelog.d/3423.misc
new file mode 100644
index 000000000000..51768c6d14ef
--- /dev/null
+++ b/changelog.d/3423.misc
@@ -0,0 +1 @@
+The test suite now can run under PostgreSQL.
diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index 20fc3b03230d..3671d24f60d0 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -95,6 +95,7 @@ def __init__(self, hs):
         Args:
             hs (synapse.server.HomeServer):
         """
+        self.hs = hs
         self.is_mine = hs.is_mine
         self.is_mine_id = hs.is_mine_id
         self.clock = hs.get_clock()
@@ -230,6 +231,10 @@ def _on_shutdown(self):
         earlier than they should when synapse is restarted. This affect of this
         is some spurious presence changes that will self-correct.
         """
+        # If the DB pool has already terminated, don't try updating
+        if not self.hs.get_db_pool().running:
+            return
+
         logger.info(
             "Performing _on_shutdown. Persisting %d unpersisted changes",
             len(self.user_to_current_state)
diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py
index 2489527f2cc8..8fc678fa67be 100644
--- a/synapse/storage/client_ips.py
+++ b/synapse/storage/client_ips.py
@@ -96,6 +96,11 @@ def insert_client_ip(self, user_id, access_token, ip, user_agent, device_id,
         self._batch_row_update[key] = (user_agent, device_id, now)
 
     def _update_client_ips_batch(self):
+
+        # If the DB pool has already terminated, don't try updating
+        if not self.hs.get_db_pool().running:
+            return
+
         def update():
             to_update = self._batch_row_update
             self._batch_row_update = {}
diff --git a/tests/__init__.py b/tests/__init__.py
index 24006c949e48..9d9ca2282928 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -15,4 +15,7 @@
 
 from twisted.trial import util
 
+from tests import utils
+
 util.DEFAULT_TIMEOUT_DURATION = 10
+utils.setupdb()
diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py
index f8e28876bb52..a65689ba896e 100644
--- a/tests/api/test_auth.py
+++ b/tests/api/test_auth.py
@@ -39,7 +39,7 @@ def setUp(self):
         self.state_handler = Mock()
         self.store = Mock()
 
-        self.hs = yield setup_test_homeserver(handlers=None)
+        self.hs = yield setup_test_homeserver(self.addCleanup, handlers=None)
         self.hs.get_datastore = Mock(return_value=self.store)
         self.hs.handlers = TestHandlers(self.hs)
         self.auth = Auth(self.hs)
diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py
index 1c2d71052cd0..48b2d3d663bb 100644
--- a/tests/api/test_filtering.py
+++ b/tests/api/test_filtering.py
@@ -46,7 +46,10 @@ def setUp(self):
         self.mock_http_client.put_json = DeferredMockCallable()
 
         hs = yield setup_test_homeserver(
-            handlers=None, http_client=self.mock_http_client, keyring=Mock()
+            self.addCleanup,
+            handlers=None,
+            http_client=self.mock_http_client,
+            keyring=Mock(),
         )
 
         self.filtering = hs.get_filtering()
diff --git a/tests/crypto/test_keyring.py b/tests/crypto/test_keyring.py
index 0c6f510d113e..8299dc72c87a 100644
--- a/tests/crypto/test_keyring.py
+++ b/tests/crypto/test_keyring.py
@@ -58,12 +58,10 @@ def setUp(self):
         self.mock_perspective_server = MockPerspectiveServer()
         self.http_client = Mock()
         self.hs = yield utils.setup_test_homeserver(
-            handlers=None, http_client=self.http_client
+            self.addCleanup, handlers=None, http_client=self.http_client
         )
         keys = self.mock_perspective_server.get_verify_keys()
-        self.hs.config.perspectives = {
-            self.mock_perspective_server.server_name: keys
-        }
+        self.hs.config.perspectives = {self.mock_perspective_server.server_name: keys}
 
     def check_context(self, _, expected):
         self.assertEquals(
diff --git a/tests/handlers/test_auth.py b/tests/handlers/test_auth.py
index ede01f8099a6..56c0f87fb772 100644
--- a/tests/handlers/test_auth.py
+++ b/tests/handlers/test_auth.py
@@ -35,7 +35,7 @@ def __init__(self, hs):
 class AuthTestCase(unittest.TestCase):
     @defer.inlineCallbacks
     def setUp(self):
-        self.hs = yield setup_test_homeserver(handlers=None)
+        self.hs = yield setup_test_homeserver(self.addCleanup, handlers=None)
         self.hs.handlers = AuthHandlers(self.hs)
         self.auth_handler = self.hs.handlers.auth_handler
         self.macaroon_generator = self.hs.get_macaroon_generator()
diff --git a/tests/handlers/test_device.py b/tests/handlers/test_device.py
index d70d64550403..56e7acd37cfb 100644
--- a/tests/handlers/test_device.py
+++ b/tests/handlers/test_device.py
@@ -34,7 +34,7 @@ def __init__(self, *args, **kwargs):
 
     @defer.inlineCallbacks
     def setUp(self):
-        hs = yield utils.setup_test_homeserver()
+        hs = yield utils.setup_test_homeserver(self.addCleanup)
         self.handler = hs.get_device_handler()
         self.store = hs.get_datastore()
         self.clock = hs.get_clock()
diff --git a/tests/handlers/test_directory.py b/tests/handlers/test_directory.py
index 06de9f5eca25..ec7355688b25 100644
--- a/tests/handlers/test_directory.py
+++ b/tests/handlers/test_directory.py
@@ -46,6 +46,7 @@ def register_query_handler(query_type, handler):
         self.mock_registry.register_query_handler = register_query_handler
 
         hs = yield setup_test_homeserver(
+            self.addCleanup,
             http_client=None,
             resource_for_federation=Mock(),
             federation_client=self.mock_federation,
diff --git a/tests/handlers/test_e2e_keys.py b/tests/handlers/test_e2e_keys.py
index 57ab228455e4..8dccc6826eb2 100644
--- a/tests/handlers/test_e2e_keys.py
+++ b/tests/handlers/test_e2e_keys.py
@@ -34,7 +34,7 @@ def __init__(self, *args, **kwargs):
     @defer.inlineCallbacks
     def setUp(self):
         self.hs = yield utils.setup_test_homeserver(
-            handlers=None, federation_client=mock.Mock()
+            self.addCleanup, handlers=None, federation_client=mock.Mock()
         )
         self.handler = synapse.handlers.e2e_keys.E2eKeysHandler(self.hs)
 
diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py
index 9268a6fe2b89..62dc69003c85 100644
--- a/tests/handlers/test_profile.py
+++ b/tests/handlers/test_profile.py
@@ -48,6 +48,7 @@ def register_query_handler(query_type, handler):
         self.mock_registry.register_query_handler = register_query_handler
 
         hs = yield setup_test_homeserver(
+            self.addCleanup,
             http_client=None,
             handlers=None,
             resource_for_federation=Mock(),
diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py
index dbec81076f13..d48d40c8dd97 100644
--- a/tests/handlers/test_register.py
+++ b/tests/handlers/test_register.py
@@ -40,6 +40,7 @@ def setUp(self):
         self.mock_distributor.declare("registered_user")
         self.mock_captcha_client = Mock()
         self.hs = yield setup_test_homeserver(
+            self.addCleanup,
             handlers=None,
             http_client=None,
             expire_access_token=True,
diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py
index becfa77bfa9f..ad58073a140c 100644
--- a/tests/handlers/test_typing.py
+++ b/tests/handlers/test_typing.py
@@ -67,6 +67,7 @@ def setUp(self):
         self.state_handler = Mock()
 
         hs = yield setup_test_homeserver(
+            self.addCleanup,
             "test",
             auth=self.auth,
             clock=self.clock,
diff --git a/tests/replication/slave/storage/_base.py b/tests/replication/slave/storage/_base.py
index c23b6e2cfd7a..65df116efc75 100644
--- a/tests/replication/slave/storage/_base.py
+++ b/tests/replication/slave/storage/_base.py
@@ -54,6 +54,7 @@ class BaseSlavedStoreTestCase(unittest.TestCase):
     @defer.inlineCallbacks
     def setUp(self):
         self.hs = yield setup_test_homeserver(
+            self.addCleanup,
             "blue",
             http_client=None,
             federation_client=Mock(),
diff --git a/tests/rest/client/v1/test_admin.py b/tests/rest/client/v1/test_admin.py
index 67d9ab94e2c8..1a553fa3f928 100644
--- a/tests/rest/client/v1/test_admin.py
+++ b/tests/rest/client/v1/test_admin.py
@@ -51,7 +51,7 @@ def setUp(self):
         self.secrets = Mock()
 
         self.hs = setup_test_homeserver(
-            http_client=None, clock=self.hs_clock, reactor=self.clock
+            self.addCleanup, http_client=None, clock=self.hs_clock, reactor=self.clock
         )
 
         self.hs.config.registration_shared_secret = u"shared"
diff --git a/tests/rest/client/v1/test_events.py b/tests/rest/client/v1/test_events.py
index 0316b74fa111..956f7fc4c4d3 100644
--- a/tests/rest/client/v1/test_events.py
+++ b/tests/rest/client/v1/test_events.py
@@ -41,6 +41,7 @@ def setUp(self):
         self.mock_resource = MockHttpResource(prefix=PATH_PREFIX)
 
         hs = yield setup_test_homeserver(
+            self.addCleanup,
             http_client=None,
             federation_client=Mock(),
             ratelimiter=NonCallableMock(spec_set=["send_message"]),
diff --git a/tests/rest/client/v1/test_profile.py b/tests/rest/client/v1/test_profile.py
index 9ba0ffc19f8a..1eab9c3bdba2 100644
--- a/tests/rest/client/v1/test_profile.py
+++ b/tests/rest/client/v1/test_profile.py
@@ -46,6 +46,7 @@ def setUp(self):
         )
 
         hs = yield setup_test_homeserver(
+            self.addCleanup,
             "test",
             http_client=None,
             resource_for_client=self.mock_resource,
diff --git a/tests/rest/client/v1/test_register.py b/tests/rest/client/v1/test_register.py
index 6f15d69ecd54..4be88b8a39fe 100644
--- a/tests/rest/client/v1/test_register.py
+++ b/tests/rest/client/v1/test_register.py
@@ -49,7 +49,7 @@ def setUp(self):
         self.hs_clock = Clock(self.clock)
 
         self.hs = self.hs = setup_test_homeserver(
-            http_client=None, clock=self.hs_clock, reactor=self.clock
+            self.addCleanup, http_client=None, clock=self.hs_clock, reactor=self.clock
         )
         self.hs.get_datastore = Mock(return_value=self.datastore)
         self.hs.get_handlers = Mock(return_value=handlers)
diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py
index 00fc796787b0..9fe0760496de 100644
--- a/tests/rest/client/v1/test_rooms.py
+++ b/tests/rest/client/v1/test_rooms.py
@@ -50,6 +50,7 @@ def setUp(self):
         self.hs_clock = Clock(self.clock)
 
         self.hs = setup_test_homeserver(
+            self.addCleanup,
             "red",
             http_client=None,
             clock=self.hs_clock,
diff --git a/tests/rest/client/v1/test_typing.py b/tests/rest/client/v1/test_typing.py
index 7f1a435e7ba3..677265edf602 100644
--- a/tests/rest/client/v1/test_typing.py
+++ b/tests/rest/client/v1/test_typing.py
@@ -44,6 +44,7 @@ def setUp(self):
         self.auth_user_id = self.user_id
 
         hs = yield setup_test_homeserver(
+            self.addCleanup,
             "red",
             clock=self.clock,
             http_client=None,
diff --git a/tests/rest/client/v2_alpha/test_filter.py b/tests/rest/client/v2_alpha/test_filter.py
index de33b10a5f5e..8260c130f8e5 100644
--- a/tests/rest/client/v2_alpha/test_filter.py
+++ b/tests/rest/client/v2_alpha/test_filter.py
@@ -43,7 +43,7 @@ def setUp(self):
         self.hs_clock = Clock(self.clock)
 
         self.hs = setup_test_homeserver(
-            http_client=None, clock=self.hs_clock, reactor=self.clock
+            self.addCleanup, http_client=None, clock=self.hs_clock, reactor=self.clock
         )
 
         self.auth = self.hs.get_auth()
diff --git a/tests/rest/client/v2_alpha/test_register.py b/tests/rest/client/v2_alpha/test_register.py
index 9487babac343..b72bd0fb7fab 100644
--- a/tests/rest/client/v2_alpha/test_register.py
+++ b/tests/rest/client/v2_alpha/test_register.py
@@ -47,7 +47,7 @@ def setUp(self):
             login_handler=self.login_handler,
         )
         self.hs = setup_test_homeserver(
-            http_client=None, clock=self.hs_clock, reactor=self.clock
+            self.addCleanup, http_client=None, clock=self.hs_clock, reactor=self.clock
         )
         self.hs.get_auth = Mock(return_value=self.auth)
         self.hs.get_handlers = Mock(return_value=self.handlers)
diff --git a/tests/rest/client/v2_alpha/test_sync.py b/tests/rest/client/v2_alpha/test_sync.py
index bafc0d1df07d..2e1d06c50982 100644
--- a/tests/rest/client/v2_alpha/test_sync.py
+++ b/tests/rest/client/v2_alpha/test_sync.py
@@ -40,7 +40,7 @@ def setUp(self):
         self.hs_clock = Clock(self.clock)
 
         self.hs = setup_test_homeserver(
-            http_client=None, clock=self.hs_clock, reactor=self.clock
+            self.addCleanup, http_client=None, clock=self.hs_clock, reactor=self.clock
         )
 
         self.auth = self.hs.get_auth()
diff --git a/tests/server.py b/tests/server.py
index 05708be8b917..beb24cf0320a 100644
--- a/tests/server.py
+++ b/tests/server.py
@@ -147,12 +147,15 @@ def callFromThread(self, callback, *args, **kwargs):
         return d
 
 
-def setup_test_homeserver(*args, **kwargs):
+def setup_test_homeserver(cleanup_func, *args, **kwargs):
     """
     Set up a synchronous test server, driven by the reactor used by
     the homeserver.
     """
-    d = _sth(*args, **kwargs).result
+    d = _sth(cleanup_func, *args, **kwargs).result
+
+    if isinstance(d, Failure):
+        d.raiseException()
 
     # Make the thread pool synchronous.
     clock = d.get_clock()
@@ -189,6 +192,9 @@ class ThreadPool:
         def start(self):
             pass
 
+        def stop(self):
+            pass
+
         def callInThreadWithCallback(self, onResult, function, *args, **kwargs):
             def _(res):
                 if isinstance(res, Failure):
diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py
index fbb25a88441a..c893990454bb 100644
--- a/tests/storage/test_appservice.py
+++ b/tests/storage/test_appservice.py
@@ -43,7 +43,10 @@ def setUp(self):
             password_providers=[],
         )
         hs = yield setup_test_homeserver(
-            config=config, federation_sender=Mock(), federation_client=Mock()
+            self.addCleanup,
+            config=config,
+            federation_sender=Mock(),
+            federation_client=Mock(),
         )
 
         self.as_token = "token1"
@@ -108,7 +111,10 @@ def setUp(self):
             password_providers=[],
         )
         hs = yield setup_test_homeserver(
-            config=config, federation_sender=Mock(), federation_client=Mock()
+            self.addCleanup,
+            config=config,
+            federation_sender=Mock(),
+            federation_client=Mock(),
         )
         self.db_pool = hs.get_db_pool()
 
@@ -392,6 +398,7 @@ def test_unique_works(self):
             app_service_config_files=[f1, f2], event_cache_size=1, password_providers=[]
         )
         hs = yield setup_test_homeserver(
+            self.addCleanup,
             config=config,
             datastore=Mock(),
             federation_sender=Mock(),
@@ -409,6 +416,7 @@ def test_duplicate_ids(self):
             app_service_config_files=[f1, f2], event_cache_size=1, password_providers=[]
         )
         hs = yield setup_test_homeserver(
+            self.addCleanup,
             config=config,
             datastore=Mock(),
             federation_sender=Mock(),
@@ -432,6 +440,7 @@ def test_duplicate_as_tokens(self):
             app_service_config_files=[f1, f2], event_cache_size=1, password_providers=[]
         )
         hs = yield setup_test_homeserver(
+            self.addCleanup,
             config=config,
             datastore=Mock(),
             federation_sender=Mock(),
diff --git a/tests/storage/test_background_update.py b/tests/storage/test_background_update.py
index b4f6baf4418a..81403727c52b 100644
--- a/tests/storage/test_background_update.py
+++ b/tests/storage/test_background_update.py
@@ -9,7 +9,9 @@
 class BackgroundUpdateTestCase(unittest.TestCase):
     @defer.inlineCallbacks
     def setUp(self):
-        hs = yield setup_test_homeserver()  # type: synapse.server.HomeServer
+        hs = yield setup_test_homeserver(
+            self.addCleanup
+        )  # type: synapse.server.HomeServer
         self.store = hs.get_datastore()
         self.clock = hs.get_clock()
 
diff --git a/tests/storage/test_client_ips.py b/tests/storage/test_client_ips.py
index ea00bbe84cd4..fa60d949baf8 100644
--- a/tests/storage/test_client_ips.py
+++ b/tests/storage/test_client_ips.py
@@ -28,7 +28,7 @@ def __init__(self, *args, **kwargs):
 
     @defer.inlineCallbacks
     def setUp(self):
-        self.hs = yield tests.utils.setup_test_homeserver()
+        self.hs = yield tests.utils.setup_test_homeserver(self.addCleanup)
         self.store = self.hs.get_datastore()
         self.clock = self.hs.get_clock()
 
diff --git a/tests/storage/test_devices.py b/tests/storage/test_devices.py
index 63bc42d9e096..aef4dfaf57a0 100644
--- a/tests/storage/test_devices.py
+++ b/tests/storage/test_devices.py
@@ -28,7 +28,7 @@ def __init__(self, *args, **kwargs):
 
     @defer.inlineCallbacks
     def setUp(self):
-        hs = yield tests.utils.setup_test_homeserver()
+        hs = yield tests.utils.setup_test_homeserver(self.addCleanup)
 
         self.store = hs.get_datastore()
 
diff --git a/tests/storage/test_directory.py b/tests/storage/test_directory.py
index 9a8ba2fcfe29..b4510c1c8d2b 100644
--- a/tests/storage/test_directory.py
+++ b/tests/storage/test_directory.py
@@ -26,7 +26,7 @@
 class DirectoryStoreTestCase(unittest.TestCase):
     @defer.inlineCallbacks
     def setUp(self):
-        hs = yield setup_test_homeserver()
+        hs = yield setup_test_homeserver(self.addCleanup)
 
         self.store = DirectoryStore(None, hs)
 
diff --git a/tests/storage/test_end_to_end_keys.py b/tests/storage/test_end_to_end_keys.py
index d45c775c2d66..8f0aaece4061 100644
--- a/tests/storage/test_end_to_end_keys.py
+++ b/tests/storage/test_end_to_end_keys.py
@@ -26,8 +26,7 @@ def __init__(self, *args, **kwargs):
 
     @defer.inlineCallbacks
     def setUp(self):
-        hs = yield tests.utils.setup_test_homeserver()
-
+        hs = yield tests.utils.setup_test_homeserver(self.addCleanup)
         self.store = hs.get_datastore()
 
     @defer.inlineCallbacks
diff --git a/tests/storage/test_event_federation.py b/tests/storage/test_event_federation.py
index 66eb11958107..2fdf34fdf63a 100644
--- a/tests/storage/test_event_federation.py
+++ b/tests/storage/test_event_federation.py
@@ -22,7 +22,7 @@
 class EventFederationWorkerStoreTestCase(tests.unittest.TestCase):
     @defer.inlineCallbacks
     def setUp(self):
-        hs = yield tests.utils.setup_test_homeserver()
+        hs = yield tests.utils.setup_test_homeserver(self.addCleanup)
         self.store = hs.get_datastore()
 
     @defer.inlineCallbacks
diff --git a/tests/storage/test_event_push_actions.py b/tests/storage/test_event_push_actions.py
index 5e87b4530df8..b114c6fb1d4e 100644
--- a/tests/storage/test_event_push_actions.py
+++ b/tests/storage/test_event_push_actions.py
@@ -33,7 +33,7 @@
 class EventPushActionsStoreTestCase(tests.unittest.TestCase):
     @defer.inlineCallbacks
     def setUp(self):
-        hs = yield tests.utils.setup_test_homeserver()
+        hs = yield tests.utils.setup_test_homeserver(self.addCleanup)
         self.store = hs.get_datastore()
 
     @defer.inlineCallbacks
diff --git a/tests/storage/test_keys.py b/tests/storage/test_keys.py
index ad0a55b324b0..47f4a8ceac99 100644
--- a/tests/storage/test_keys.py
+++ b/tests/storage/test_keys.py
@@ -28,7 +28,7 @@ def __init__(self, *args, **kwargs):
 
     @defer.inlineCallbacks
     def setUp(self):
-        hs = yield tests.utils.setup_test_homeserver()
+        hs = yield tests.utils.setup_test_homeserver(self.addCleanup)
         self.store = hs.get_datastore()
 
     @defer.inlineCallbacks
diff --git a/tests/storage/test_monthly_active_users.py b/tests/storage/test_monthly_active_users.py
index 22b1072d9f57..0a2c859f26ca 100644
--- a/tests/storage/test_monthly_active_users.py
+++ b/tests/storage/test_monthly_active_users.py
@@ -28,7 +28,7 @@ def __init__(self, *args, **kwargs):
 
     @defer.inlineCallbacks
     def setUp(self):
-        self.hs = yield setup_test_homeserver()
+        self.hs = yield setup_test_homeserver(self.addCleanup)
         self.store = self.hs.get_datastore()
 
     @defer.inlineCallbacks
diff --git a/tests/storage/test_presence.py b/tests/storage/test_presence.py
index 12c540dfab40..b5b58ff660a3 100644
--- a/tests/storage/test_presence.py
+++ b/tests/storage/test_presence.py
@@ -26,7 +26,7 @@
 class PresenceStoreTestCase(unittest.TestCase):
     @defer.inlineCallbacks
     def setUp(self):
-        hs = yield setup_test_homeserver(clock=MockClock())
+        hs = yield setup_test_homeserver(self.addCleanup, clock=MockClock())
 
         self.store = PresenceStore(None, hs)
 
diff --git a/tests/storage/test_profile.py b/tests/storage/test_profile.py
index 5acbc8be0c57..a1f6618bf94b 100644
--- a/tests/storage/test_profile.py
+++ b/tests/storage/test_profile.py
@@ -26,7 +26,7 @@
 class ProfileStoreTestCase(unittest.TestCase):
     @defer.inlineCallbacks
     def setUp(self):
-        hs = yield setup_test_homeserver()
+        hs = yield setup_test_homeserver(self.addCleanup)
 
         self.store = ProfileStore(None, hs)
 
diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py
index 85ce61e841cf..c4e9fb72bfff 100644
--- a/tests/storage/test_redaction.py
+++ b/tests/storage/test_redaction.py
@@ -29,7 +29,7 @@ class RedactionTestCase(unittest.TestCase):
     @defer.inlineCallbacks
     def setUp(self):
         hs = yield setup_test_homeserver(
-            resource_for_federation=Mock(), http_client=None
+            self.addCleanup, resource_for_federation=Mock(), http_client=None
         )
 
         self.store = hs.get_datastore()
diff --git a/tests/storage/test_registration.py b/tests/storage/test_registration.py
index bd96896bb35e..4eda122edc39 100644
--- a/tests/storage/test_registration.py
+++ b/tests/storage/test_registration.py
@@ -23,7 +23,7 @@
 class RegistrationStoreTestCase(unittest.TestCase):
     @defer.inlineCallbacks
     def setUp(self):
-        hs = yield setup_test_homeserver()
+        hs = yield setup_test_homeserver(self.addCleanup)
         self.db_pool = hs.get_db_pool()
 
         self.store = hs.get_datastore()
diff --git a/tests/storage/test_room.py b/tests/storage/test_room.py
index 84d49b55c1cb..a1ea23b06896 100644
--- a/tests/storage/test_room.py
+++ b/tests/storage/test_room.py
@@ -26,7 +26,7 @@
 class RoomStoreTestCase(unittest.TestCase):
     @defer.inlineCallbacks
     def setUp(self):
-        hs = yield setup_test_homeserver()
+        hs = yield setup_test_homeserver(self.addCleanup)
 
         # We can't test RoomStore on its own without the DirectoryStore, for
         # management of the 'room_aliases' table
@@ -57,7 +57,7 @@ def test_get_room(self):
 class RoomEventsStoreTestCase(unittest.TestCase):
     @defer.inlineCallbacks
     def setUp(self):
-        hs = setup_test_homeserver()
+        hs = setup_test_homeserver(self.addCleanup)
 
         # Room events need the full datastore, for persist_event() and
         # get_room_state()
diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py
index 0d9908926a94..c83ef600625f 100644
--- a/tests/storage/test_roommember.py
+++ b/tests/storage/test_roommember.py
@@ -29,7 +29,7 @@ class RoomMemberStoreTestCase(unittest.TestCase):
     @defer.inlineCallbacks
     def setUp(self):
         hs = yield setup_test_homeserver(
-            resource_for_federation=Mock(), http_client=None
+            self.addCleanup, resource_for_federation=Mock(), http_client=None
         )
         # We can't test the RoomMemberStore on its own without the other event
         # storage logic
diff --git a/tests/storage/test_state.py b/tests/storage/test_state.py
index ed5b41644a5c..6168c46248d0 100644
--- a/tests/storage/test_state.py
+++ b/tests/storage/test_state.py
@@ -33,7 +33,7 @@ def __init__(self, *args, **kwargs):
 
     @defer.inlineCallbacks
     def setUp(self):
-        hs = yield tests.utils.setup_test_homeserver()
+        hs = yield tests.utils.setup_test_homeserver(self.addCleanup)
 
         self.store = hs.get_datastore()
         self.event_builder_factory = hs.get_event_builder_factory()
diff --git a/tests/storage/test_user_directory.py b/tests/storage/test_user_directory.py
index 7a273eab4880..b46e0ea7e279 100644
--- a/tests/storage/test_user_directory.py
+++ b/tests/storage/test_user_directory.py
@@ -29,7 +29,7 @@
 class UserDirectoryStoreTestCase(unittest.TestCase):
     @defer.inlineCallbacks
     def setUp(self):
-        self.hs = yield setup_test_homeserver()
+        self.hs = yield setup_test_homeserver(self.addCleanup)
         self.store = UserDirectoryStore(None, self.hs)
 
         # alice and bob are both in !room_id. bobby is not but shares
diff --git a/tests/test_federation.py b/tests/test_federation.py
index f40ff29b52af..2540604fcc91 100644
--- a/tests/test_federation.py
+++ b/tests/test_federation.py
@@ -18,7 +18,10 @@ def setUp(self):
         self.reactor = ThreadedMemoryReactorClock()
         self.hs_clock = Clock(self.reactor)
         self.homeserver = setup_test_homeserver(
-            http_client=self.http_client, clock=self.hs_clock, reactor=self.reactor
+            self.addCleanup,
+            http_client=self.http_client,
+            clock=self.hs_clock,
+            reactor=self.reactor,
         )
 
         user_id = UserID("us", "test")
diff --git a/tests/test_server.py b/tests/test_server.py
index fc396226ea27..895e49040688 100644
--- a/tests/test_server.py
+++ b/tests/test_server.py
@@ -16,7 +16,7 @@ def setUp(self):
         self.reactor = MemoryReactorClock()
         self.hs_clock = Clock(self.reactor)
         self.homeserver = setup_test_homeserver(
-            http_client=None, clock=self.hs_clock, reactor=self.reactor
+            self.addCleanup, http_client=None, clock=self.hs_clock, reactor=self.reactor
         )
 
     def test_handler_for_request(self):
diff --git a/tests/test_visibility.py b/tests/test_visibility.py
index 8643d6312570..45a78338d6aa 100644
--- a/tests/test_visibility.py
+++ b/tests/test_visibility.py
@@ -31,7 +31,7 @@
 class FilterEventsForServerTestCase(tests.unittest.TestCase):
     @defer.inlineCallbacks
     def setUp(self):
-        self.hs = yield setup_test_homeserver()
+        self.hs = yield setup_test_homeserver(self.addCleanup)
         self.event_creation_handler = self.hs.get_event_creation_handler()
         self.event_builder_factory = self.hs.get_event_builder_factory()
         self.store = self.hs.get_datastore()
diff --git a/tests/utils.py b/tests/utils.py
index 8668b5478fd8..90378326f898 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -13,7 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import atexit
 import hashlib
+import os
+import uuid
 from inspect import getcallargs
 
 from mock import Mock, patch
@@ -27,23 +30,80 @@
 from synapse.server import HomeServer
 from synapse.storage import PostgresEngine
 from synapse.storage.engines import create_engine
-from synapse.storage.prepare_database import prepare_database
+from synapse.storage.prepare_database import (
+    _get_or_create_schema_state,
+    _setup_new_database,
+    prepare_database,
+)
 from synapse.util.logcontext import LoggingContext
 from synapse.util.ratelimitutils import FederationRateLimiter
 
 # set this to True to run the tests against postgres instead of sqlite.
-# It requires you to have a local postgres database called synapse_test, within
-# which ALL TABLES WILL BE DROPPED
-USE_POSTGRES_FOR_TESTS = False
+USE_POSTGRES_FOR_TESTS = os.environ.get("SYNAPSE_POSTGRES", False)
+POSTGRES_USER = os.environ.get("SYNAPSE_POSTGRES_USER", "postgres")
+POSTGRES_BASE_DB = "_synapse_unit_tests_base_%s" % (os.getpid(),)
+
+
+def setupdb():
+
+    # If we're using PostgreSQL, set up the db once
+    if USE_POSTGRES_FOR_TESTS:
+        pgconfig = {
+            "name": "psycopg2",
+            "args": {
+                "database": POSTGRES_BASE_DB,
+                "user": POSTGRES_USER,
+                "cp_min": 1,
+                "cp_max": 5,
+            },
+        }
+        config = Mock()
+        config.password_providers = []
+        config.database_config = pgconfig
+        db_engine = create_engine(pgconfig)
+        db_conn = db_engine.module.connect(user=POSTGRES_USER)
+        db_conn.autocommit = True
+        cur = db_conn.cursor()
+        cur.execute("DROP DATABASE IF EXISTS %s;" % (POSTGRES_BASE_DB,))
+        cur.execute("CREATE DATABASE %s;" % (POSTGRES_BASE_DB,))
+        cur.close()
+        db_conn.close()
+
+        # Set up in the db
+        db_conn = db_engine.module.connect(
+            database=POSTGRES_BASE_DB, user=POSTGRES_USER
+        )
+        cur = db_conn.cursor()
+        _get_or_create_schema_state(cur, db_engine)
+        _setup_new_database(cur, db_engine)
+        db_conn.commit()
+        cur.close()
+        db_conn.close()
+
+        def _cleanup():
+            db_conn = db_engine.module.connect(user=POSTGRES_USER)
+            db_conn.autocommit = True
+            cur = db_conn.cursor()
+            cur.execute("DROP DATABASE IF EXISTS %s;" % (POSTGRES_BASE_DB,))
+            cur.close()
+            db_conn.close()
+
+        atexit.register(_cleanup)
 
 
 @defer.inlineCallbacks
 def setup_test_homeserver(
-    name="test", datastore=None, config=None, reactor=None, **kargs
+    cleanup_func, name="test", datastore=None, config=None, reactor=None, **kargs
 ):
-    """Setup a homeserver suitable for running tests against. Keyword arguments
-    are passed to the Homeserver constructor. If no datastore is supplied a
-    datastore backed by an in-memory sqlite db will be given to the HS.
+    """
+    Setup a homeserver suitable for running tests against.  Keyword arguments
+    are passed to the Homeserver constructor.
+
+    If no datastore is supplied, one is created and given to the homeserver.
+
+    Args:
+        cleanup_func : The function used to register a cleanup routine for
+                       after the test.
     """
     if reactor is None:
         from twisted.internet import reactor
@@ -95,9 +155,11 @@ def setup_test_homeserver(
         kargs["clock"] = MockClock()
 
     if USE_POSTGRES_FOR_TESTS:
+        test_db = "synapse_test_%s" % uuid.uuid4().hex
+
         config.database_config = {
             "name": "psycopg2",
-            "args": {"database": "synapse_test", "cp_min": 1, "cp_max": 5},
+            "args": {"database": test_db, "cp_min": 1, "cp_max": 5},
         }
     else:
         config.database_config = {
@@ -107,6 +169,21 @@ def setup_test_homeserver(
 
     db_engine = create_engine(config.database_config)
 
+    # Create the database before we actually try and connect to it, based off
+    # the template database we generate in setupdb()
+    if datastore is None and isinstance(db_engine, PostgresEngine):
+        db_conn = db_engine.module.connect(
+            database=POSTGRES_BASE_DB, user=POSTGRES_USER
+        )
+        db_conn.autocommit = True
+        cur = db_conn.cursor()
+        cur.execute("DROP DATABASE IF EXISTS %s;" % (test_db,))
+        cur.execute(
+            "CREATE DATABASE %s WITH TEMPLATE %s;" % (test_db, POSTGRES_BASE_DB)
+        )
+        cur.close()
+        db_conn.close()
+
     # we need to configure the connection pool to run the on_new_connection
     # function, so that we can test code that uses custom sqlite functions
     # (like rank).
@@ -125,15 +202,35 @@ def setup_test_homeserver(
             reactor=reactor,
             **kargs
         )
-        db_conn = hs.get_db_conn()
-        # make sure that the database is empty
-        if isinstance(db_engine, PostgresEngine):
-            cur = db_conn.cursor()
-            cur.execute("SELECT tablename FROM pg_tables where schemaname='public'")
-            rows = cur.fetchall()
-            for r in rows:
-                cur.execute("DROP TABLE %s CASCADE" % r[0])
-        yield prepare_database(db_conn, db_engine, config)
+
+        # Prepare the DB on SQLite -- PostgreSQL is a copy of an already up to
+        # date db
+        if not isinstance(db_engine, PostgresEngine):
+            db_conn = hs.get_db_conn()
+            yield prepare_database(db_conn, db_engine, config)
+            db_conn.commit()
+            db_conn.close()
+
+        else:
+            # We need to do cleanup on PostgreSQL
+            def cleanup():
+                # Close all the db pools
+                hs.get_db_pool().close()
+
+                # Drop the test database
+                db_conn = db_engine.module.connect(
+                    database=POSTGRES_BASE_DB, user=POSTGRES_USER
+                )
+                db_conn.autocommit = True
+                cur = db_conn.cursor()
+                cur.execute("DROP DATABASE IF EXISTS %s;" % (test_db,))
+                db_conn.commit()
+                cur.close()
+                db_conn.close()
+
+            # Register the cleanup hook
+            cleanup_func(cleanup)
+
         hs.setup()
     else:
         hs = HomeServer(
diff --git a/tox.ini b/tox.ini
index ed26644bd922..085f43898908 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,7 +1,7 @@
 [tox]
 envlist = packaging, py27, py36, pep8, check_isort
 
-[testenv]
+[base]
 deps =
     coverage
     Twisted>=15.1
@@ -15,6 +15,15 @@ deps =
 setenv =
     PYTHONDONTWRITEBYTECODE = no_byte_code
 
+[testenv]
+deps =
+    {[base]deps}
+
+setenv =
+    {[base]setenv}
+
+passenv = *
+
 commands =
     /usr/bin/find "{toxinidir}" -name '*.pyc' -delete
     coverage run {env:COVERAGE_OPTS:} --source="{toxinidir}/synapse" \
@@ -46,6 +55,15 @@ commands =
 # )
 usedevelop=true
 
+[testenv:py27-postgres]
+usedevelop=true
+deps =
+    {[base]deps}
+     psycopg2
+setenv =
+    {[base]setenv}
+    SYNAPSE_POSTGRES = 1
+
 [testenv:py36]
 usedevelop=true
 commands =

From 0d43f991a19840a224d3dac78d79f13d78212ee6 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Mon, 13 Aug 2018 18:00:23 +0100
Subject: [PATCH 376/830] support admin_email config and pass through into
 blocking errors, return AuthError in all cases

---
 synapse/api/auth.py             |  8 ++++++--
 synapse/api/errors.py           | 13 +++++++++++--
 synapse/config/server.py        |  4 ++++
 synapse/handlers/register.py    | 27 ++++++++++++++-------------
 tests/api/test_auth.py          |  6 +++++-
 tests/handlers/test_register.py |  8 ++++----
 tests/utils.py                  |  1 +
 7 files changed, 45 insertions(+), 22 deletions(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 9c62ec43742d..4f028078fa27 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -781,11 +781,15 @@ def check_auth_blocking(self):
         """
         if self.hs.config.hs_disabled:
             raise AuthError(
-                403, self.hs.config.hs_disabled_message, errcode=Codes.HS_DISABLED
+                403, self.hs.config.hs_disabled_message,
+                errcode=Codes.HS_DISABLED,
+                admin_email=self.hs.config.admin_email,
             )
         if self.hs.config.limit_usage_by_mau is True:
             current_mau = yield self.store.get_monthly_active_count()
             if current_mau >= self.hs.config.max_mau_value:
                 raise AuthError(
-                    403, "MAU Limit Exceeded", errcode=Codes.MAU_LIMIT_EXCEEDED
+                    403, "MAU Limit Exceeded",
+                    admin_email=self.hs.config.admin_email,
+                    errcode=Codes.MAU_LIMIT_EXCEEDED
                 )
diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index dc3bed5fcbbd..d74848159e44 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -225,11 +225,20 @@ def __init__(self, msg="Not found", errcode=Codes.NOT_FOUND):
 
 class AuthError(SynapseError):
     """An error raised when there was a problem authorising an event."""
-
     def __init__(self, *args, **kwargs):
         if "errcode" not in kwargs:
             kwargs["errcode"] = Codes.FORBIDDEN
-        super(AuthError, self).__init__(*args, **kwargs)
+        self.admin_email = kwargs.get('admin_email')
+        self.msg = kwargs.get('msg')
+        self.errcode = kwargs.get('errcode')
+        super(AuthError, self).__init__(*args, errcode=kwargs["errcode"])
+
+    def error_dict(self):
+        return cs_error(
+            self.msg,
+            self.errcode,
+            admin_email=self.admin_email,
+        )
 
 
 class EventSizeError(SynapseError):
diff --git a/synapse/config/server.py b/synapse/config/server.py
index 3b078d72cabd..64a5121a452f 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -82,6 +82,10 @@ def read_config(self, config):
         self.hs_disabled = config.get("hs_disabled", False)
         self.hs_disabled_message = config.get("hs_disabled_message", "")
 
+        # Admin email to direct users at should their instance become blocked
+        # due to resource constraints
+        self.admin_email = config.get("admin_email", None)
+
         # FIXME: federation_domain_whitelist needs sytests
         self.federation_domain_whitelist = None
         federation_domain_whitelist = config.get(
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index 3526b20d5a57..ef7222d7b860 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -144,7 +144,8 @@ def register(
         Raises:
             RegistrationError if there was a problem registering.
         """
-        yield self._check_mau_limits()
+
+        yield self.auth.check_auth_blocking()
         password_hash = None
         if password:
             password_hash = yield self.auth_handler().hash(password)
@@ -289,7 +290,7 @@ def register_saml2(self, localpart):
                 400,
                 "User ID can only contain characters a-z, 0-9, or '=_-./'",
             )
-        yield self._check_mau_limits()
+        yield self.auth.check_auth_blocking()
         user = UserID(localpart, self.hs.hostname)
         user_id = user.to_string()
 
@@ -439,7 +440,7 @@ def get_or_create_user(self, requester, localpart, displayname,
         """
         if localpart is None:
             raise SynapseError(400, "Request must include user id")
-        yield self._check_mau_limits()
+        yield self.auth.check_auth_blocking()
         need_register = True
 
         try:
@@ -534,13 +535,13 @@ def _join_user_to_room(self, requester, room_identifier):
             action="join",
         )
 
-    @defer.inlineCallbacks
-    def _check_mau_limits(self):
-        """
-        Do not accept registrations if monthly active user limits exceeded
-         and limiting is enabled
-        """
-        try:
-            yield self.auth.check_auth_blocking()
-        except AuthError as e:
-            raise RegistrationError(e.code, str(e), e.errcode)
+    # @defer.inlineCallbacks
+    # def _s(self):
+    #     """
+    #     Do not accept registrations if monthly active user limits exceeded
+    #      and limiting is enabled
+    #     """
+    #     try:
+    #         yield self.auth.check_auth_blocking()
+    #     except AuthError as e:
+    #         raise RegistrationError(e.code, str(e), e.errcode)
diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py
index a65689ba896e..e8a1894e651c 100644
--- a/tests/api/test_auth.py
+++ b/tests/api/test_auth.py
@@ -455,8 +455,11 @@ def test_blocking_mau(self):
             return_value=defer.succeed(lots_of_users)
         )
 
-        with self.assertRaises(AuthError):
+        with self.assertRaises(AuthError) as e:
             yield self.auth.check_auth_blocking()
+        self.assertEquals(e.exception.admin_email, self.hs.config.admin_email)
+        self.assertEquals(e.exception.errcode, Codes.MAU_LIMIT_EXCEEDED)
+        self.assertEquals(e.exception.code, 403)
 
         # Ensure does not throw an error
         self.store.get_monthly_active_count = Mock(
@@ -470,5 +473,6 @@ def test_hs_disabled(self):
         self.hs.config.hs_disabled_message = "Reason for being disabled"
         with self.assertRaises(AuthError) as e:
             yield self.auth.check_auth_blocking()
+        self.assertEquals(e.exception.admin_email, self.hs.config.admin_email)
         self.assertEquals(e.exception.errcode, Codes.HS_DISABLED)
         self.assertEquals(e.exception.code, 403)
diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py
index d48d40c8dd97..35d1bcab3e9f 100644
--- a/tests/handlers/test_register.py
+++ b/tests/handlers/test_register.py
@@ -17,7 +17,7 @@
 
 from twisted.internet import defer
 
-from synapse.api.errors import RegistrationError
+from synapse.api.errors import AuthError
 from synapse.handlers.register import RegistrationHandler
 from synapse.types import UserID, create_requester
 
@@ -109,7 +109,7 @@ def test_get_or_create_user_mau_blocked(self):
         self.store.get_monthly_active_count = Mock(
             return_value=defer.succeed(self.lots_of_users)
         )
-        with self.assertRaises(RegistrationError):
+        with self.assertRaises(AuthError):
             yield self.handler.get_or_create_user("requester", 'b', "display_name")
 
     @defer.inlineCallbacks
@@ -118,7 +118,7 @@ def test_register_mau_blocked(self):
         self.store.get_monthly_active_count = Mock(
             return_value=defer.succeed(self.lots_of_users)
         )
-        with self.assertRaises(RegistrationError):
+        with self.assertRaises(AuthError):
             yield self.handler.register(localpart="local_part")
 
     @defer.inlineCallbacks
@@ -127,5 +127,5 @@ def test_register_saml2_mau_blocked(self):
         self.store.get_monthly_active_count = Mock(
             return_value=defer.succeed(self.lots_of_users)
         )
-        with self.assertRaises(RegistrationError):
+        with self.assertRaises(AuthError):
             yield self.handler.register_saml2(localpart="local_part")
diff --git a/tests/utils.py b/tests/utils.py
index 90378326f898..4af81624eb39 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -139,6 +139,7 @@ def setup_test_homeserver(
         config.hs_disabled_message = ""
         config.max_mau_value = 50
         config.mau_limits_reserved_threepids = []
+        config.admin_email = None
 
         # we need a sane default_room_version, otherwise attempts to create rooms will
         # fail.

From ce7de9ae6b74e8e5e89ff442bc29f8cd73328042 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Mon, 13 Aug 2018 18:06:18 +0100
Subject: [PATCH 377/830] Revert "support admin_email config and pass through
 into blocking errors, return AuthError in all cases"

This reverts commit 0d43f991a19840a224d3dac78d79f13d78212ee6.
---
 synapse/api/auth.py             |  8 ++------
 synapse/api/errors.py           | 13 ++-----------
 synapse/config/server.py        |  4 ----
 synapse/handlers/register.py    | 27 +++++++++++++--------------
 tests/api/test_auth.py          |  6 +-----
 tests/handlers/test_register.py |  8 ++++----
 tests/utils.py                  |  1 -
 7 files changed, 22 insertions(+), 45 deletions(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 4f028078fa27..9c62ec43742d 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -781,15 +781,11 @@ def check_auth_blocking(self):
         """
         if self.hs.config.hs_disabled:
             raise AuthError(
-                403, self.hs.config.hs_disabled_message,
-                errcode=Codes.HS_DISABLED,
-                admin_email=self.hs.config.admin_email,
+                403, self.hs.config.hs_disabled_message, errcode=Codes.HS_DISABLED
             )
         if self.hs.config.limit_usage_by_mau is True:
             current_mau = yield self.store.get_monthly_active_count()
             if current_mau >= self.hs.config.max_mau_value:
                 raise AuthError(
-                    403, "MAU Limit Exceeded",
-                    admin_email=self.hs.config.admin_email,
-                    errcode=Codes.MAU_LIMIT_EXCEEDED
+                    403, "MAU Limit Exceeded", errcode=Codes.MAU_LIMIT_EXCEEDED
                 )
diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index d74848159e44..dc3bed5fcbbd 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -225,20 +225,11 @@ def __init__(self, msg="Not found", errcode=Codes.NOT_FOUND):
 
 class AuthError(SynapseError):
     """An error raised when there was a problem authorising an event."""
+
     def __init__(self, *args, **kwargs):
         if "errcode" not in kwargs:
             kwargs["errcode"] = Codes.FORBIDDEN
-        self.admin_email = kwargs.get('admin_email')
-        self.msg = kwargs.get('msg')
-        self.errcode = kwargs.get('errcode')
-        super(AuthError, self).__init__(*args, errcode=kwargs["errcode"])
-
-    def error_dict(self):
-        return cs_error(
-            self.msg,
-            self.errcode,
-            admin_email=self.admin_email,
-        )
+        super(AuthError, self).__init__(*args, **kwargs)
 
 
 class EventSizeError(SynapseError):
diff --git a/synapse/config/server.py b/synapse/config/server.py
index 64a5121a452f..3b078d72cabd 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -82,10 +82,6 @@ def read_config(self, config):
         self.hs_disabled = config.get("hs_disabled", False)
         self.hs_disabled_message = config.get("hs_disabled_message", "")
 
-        # Admin email to direct users at should their instance become blocked
-        # due to resource constraints
-        self.admin_email = config.get("admin_email", None)
-
         # FIXME: federation_domain_whitelist needs sytests
         self.federation_domain_whitelist = None
         federation_domain_whitelist = config.get(
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index ef7222d7b860..3526b20d5a57 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -144,8 +144,7 @@ def register(
         Raises:
             RegistrationError if there was a problem registering.
         """
-
-        yield self.auth.check_auth_blocking()
+        yield self._check_mau_limits()
         password_hash = None
         if password:
             password_hash = yield self.auth_handler().hash(password)
@@ -290,7 +289,7 @@ def register_saml2(self, localpart):
                 400,
                 "User ID can only contain characters a-z, 0-9, or '=_-./'",
             )
-        yield self.auth.check_auth_blocking()
+        yield self._check_mau_limits()
         user = UserID(localpart, self.hs.hostname)
         user_id = user.to_string()
 
@@ -440,7 +439,7 @@ def get_or_create_user(self, requester, localpart, displayname,
         """
         if localpart is None:
             raise SynapseError(400, "Request must include user id")
-        yield self.auth.check_auth_blocking()
+        yield self._check_mau_limits()
         need_register = True
 
         try:
@@ -535,13 +534,13 @@ def _join_user_to_room(self, requester, room_identifier):
             action="join",
         )
 
-    # @defer.inlineCallbacks
-    # def _s(self):
-    #     """
-    #     Do not accept registrations if monthly active user limits exceeded
-    #      and limiting is enabled
-    #     """
-    #     try:
-    #         yield self.auth.check_auth_blocking()
-    #     except AuthError as e:
-    #         raise RegistrationError(e.code, str(e), e.errcode)
+    @defer.inlineCallbacks
+    def _check_mau_limits(self):
+        """
+        Do not accept registrations if monthly active user limits exceeded
+         and limiting is enabled
+        """
+        try:
+            yield self.auth.check_auth_blocking()
+        except AuthError as e:
+            raise RegistrationError(e.code, str(e), e.errcode)
diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py
index e8a1894e651c..a65689ba896e 100644
--- a/tests/api/test_auth.py
+++ b/tests/api/test_auth.py
@@ -455,11 +455,8 @@ def test_blocking_mau(self):
             return_value=defer.succeed(lots_of_users)
         )
 
-        with self.assertRaises(AuthError) as e:
+        with self.assertRaises(AuthError):
             yield self.auth.check_auth_blocking()
-        self.assertEquals(e.exception.admin_email, self.hs.config.admin_email)
-        self.assertEquals(e.exception.errcode, Codes.MAU_LIMIT_EXCEEDED)
-        self.assertEquals(e.exception.code, 403)
 
         # Ensure does not throw an error
         self.store.get_monthly_active_count = Mock(
@@ -473,6 +470,5 @@ def test_hs_disabled(self):
         self.hs.config.hs_disabled_message = "Reason for being disabled"
         with self.assertRaises(AuthError) as e:
             yield self.auth.check_auth_blocking()
-        self.assertEquals(e.exception.admin_email, self.hs.config.admin_email)
         self.assertEquals(e.exception.errcode, Codes.HS_DISABLED)
         self.assertEquals(e.exception.code, 403)
diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py
index 35d1bcab3e9f..d48d40c8dd97 100644
--- a/tests/handlers/test_register.py
+++ b/tests/handlers/test_register.py
@@ -17,7 +17,7 @@
 
 from twisted.internet import defer
 
-from synapse.api.errors import AuthError
+from synapse.api.errors import RegistrationError
 from synapse.handlers.register import RegistrationHandler
 from synapse.types import UserID, create_requester
 
@@ -109,7 +109,7 @@ def test_get_or_create_user_mau_blocked(self):
         self.store.get_monthly_active_count = Mock(
             return_value=defer.succeed(self.lots_of_users)
         )
-        with self.assertRaises(AuthError):
+        with self.assertRaises(RegistrationError):
             yield self.handler.get_or_create_user("requester", 'b', "display_name")
 
     @defer.inlineCallbacks
@@ -118,7 +118,7 @@ def test_register_mau_blocked(self):
         self.store.get_monthly_active_count = Mock(
             return_value=defer.succeed(self.lots_of_users)
         )
-        with self.assertRaises(AuthError):
+        with self.assertRaises(RegistrationError):
             yield self.handler.register(localpart="local_part")
 
     @defer.inlineCallbacks
@@ -127,5 +127,5 @@ def test_register_saml2_mau_blocked(self):
         self.store.get_monthly_active_count = Mock(
             return_value=defer.succeed(self.lots_of_users)
         )
-        with self.assertRaises(AuthError):
+        with self.assertRaises(RegistrationError):
             yield self.handler.register_saml2(localpart="local_part")
diff --git a/tests/utils.py b/tests/utils.py
index 4af81624eb39..90378326f898 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -139,7 +139,6 @@ def setup_test_homeserver(
         config.hs_disabled_message = ""
         config.max_mau_value = 50
         config.mau_limits_reserved_threepids = []
-        config.admin_email = None
 
         # we need a sane default_room_version, otherwise attempts to create rooms will
         # fail.

From 34f51babc57b2eb8eec9a591e751a1a5474f4c38 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Mon, 13 Aug 2018 18:48:03 +0100
Subject: [PATCH 378/830] Revert "Use recaptcha_ajax.js directly from Google"

---
 changelog.d/1932.bugfix                       |   1 -
 synapse/static/client/register/index.html     |   2 +-
 .../client/register/js/recaptcha_ajax.js      | 195 ++++++++++++++++++
 3 files changed, 196 insertions(+), 2 deletions(-)
 delete mode 100644 changelog.d/1932.bugfix
 create mode 100644 synapse/static/client/register/js/recaptcha_ajax.js

diff --git a/changelog.d/1932.bugfix b/changelog.d/1932.bugfix
deleted file mode 100644
index caa551627beb..000000000000
--- a/changelog.d/1932.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Don’t ship recaptcha_ajax.js, use it directly from Google
diff --git a/synapse/static/client/register/index.html b/synapse/static/client/register/index.html
index 886f2edd1fad..600b3ee41eab 100644
--- a/synapse/static/client/register/index.html
+++ b/synapse/static/client/register/index.html
@@ -4,7 +4,7 @@
 <meta name='viewport' content='width=device-width, initial-scale=1, user-scalable=no, minimum-scale=1.0, maximum-scale=1.0'> 
 <link rel="stylesheet" href="style.css">
 <script src="js/jquery-2.1.3.min.js"></script>
-<script src="https://www.google.com/recaptcha/api/js/recaptcha_ajax.js"></script>
+<script src="js/recaptcha_ajax.js"></script>
 <script src="register_config.js"></script>
 <script src="js/register.js"></script>
 </head>
diff --git a/synapse/static/client/register/js/recaptcha_ajax.js b/synapse/static/client/register/js/recaptcha_ajax.js
new file mode 100644
index 000000000000..d0e71e5b8824
--- /dev/null
+++ b/synapse/static/client/register/js/recaptcha_ajax.js
@@ -0,0 +1,195 @@
+(function(){var h,k=this,l=function(a){return void 0!==a},ba=function(){},n=function(a){var b=typeof a;if("object"==b)if(a){if(a instanceof Array)return"array";if(a instanceof Object)return b;var c=Object.prototype.toString.call(a);if("[object Window]"==c)return"object";if("[object Array]"==c||"number"==typeof a.length&&"undefined"!=typeof a.splice&&"undefined"!=typeof a.propertyIsEnumerable&&!a.propertyIsEnumerable("splice"))return"array";if("[object Function]"==c||"undefined"!=typeof a.call&&"undefined"!=typeof a.propertyIsEnumerable&&
+!a.propertyIsEnumerable("call"))return"function"}else return"null";else if("function"==b&&"undefined"==typeof a.call)return"object";return b},p=function(a){return"array"==n(a)},ca=function(a){var b=n(a);return"array"==b||"object"==b&&"number"==typeof a.length},q=function(a){return"string"==typeof a},r=function(a){return"function"==n(a)},da=function(a){var b=typeof a;return"object"==b&&null!=a||"function"==b},ea=function(a,b,c){return a.call.apply(a.bind,arguments)},fa=function(a,b,c){if(!a)throw Error();
+if(2<arguments.length){var d=Array.prototype.slice.call(arguments,2);return function(){var c=Array.prototype.slice.call(arguments);Array.prototype.unshift.apply(c,d);return a.apply(b,c)}}return function(){return a.apply(b,arguments)}},s=function(a,b,c){s=Function.prototype.bind&&-1!=Function.prototype.bind.toString().indexOf("native code")?ea:fa;return s.apply(null,arguments)},ga=function(a,b){var c=Array.prototype.slice.call(arguments,1);return function(){var b=c.slice();b.push.apply(b,arguments);
+return a.apply(this,b)}},ha=Date.now||function(){return+new Date},ia=null,t=function(a,b){var c=a.split("."),d=k;c[0]in d||!d.execScript||d.execScript("var "+c[0]);for(var e;c.length&&(e=c.shift());)!c.length&&l(b)?d[e]=b:d=d[e]?d[e]:d[e]={}},u=function(a,b){function c(){}c.prototype=b.prototype;a.superClass_=b.prototype;a.prototype=new c;a.base=function(a,c,g){return b.prototype[c].apply(a,Array.prototype.slice.call(arguments,2))}};
+Function.prototype.bind=Function.prototype.bind||function(a,b){if(1<arguments.length){var c=Array.prototype.slice.call(arguments,1);c.unshift(this,a);return s.apply(null,c)}return s(this,a)};var v={};t("RecaptchaTemplates",v);v.VertHtml='<table id="recaptcha_table" class="recaptchatable" > <tr> <td colspan="6" class=\'recaptcha_r1_c1\'></td> </tr> <tr> <td class=\'recaptcha_r2_c1\'></td> <td colspan="4" class=\'recaptcha_image_cell\'><center><div id="recaptcha_image"></div></center></td> <td class=\'recaptcha_r2_c2\'></td> </tr> <tr> <td rowspan="6" class=\'recaptcha_r3_c1\'></td> <td colspan="4" class=\'recaptcha_r3_c2\'></td> <td rowspan="6" class=\'recaptcha_r3_c3\'></td> </tr> <tr> <td rowspan="3" class=\'recaptcha_r4_c1\' height="49"> <div class="recaptcha_input_area"> <input name="recaptcha_response_field" id="recaptcha_response_field" type="text" autocorrect="off" autocapitalize="off" placeholder="" /> <span id="recaptcha_privacy" class="recaptcha_only_if_privacy"></span> </div> </td> <td rowspan="4" class=\'recaptcha_r4_c2\'></td> <td><a id=\'recaptcha_reload_btn\'><img id=\'recaptcha_reload\' width="25" height="17" /></a></td> <td rowspan="4" class=\'recaptcha_r4_c4\'></td> </tr> <tr> <td><a id=\'recaptcha_switch_audio_btn\' class="recaptcha_only_if_image"><img id=\'recaptcha_switch_audio\' width="25" height="16" alt="" /></a><a id=\'recaptcha_switch_img_btn\' class="recaptcha_only_if_audio"><img id=\'recaptcha_switch_img\' width="25" height="16" alt=""/></a></td> </tr> <tr> <td><a id=\'recaptcha_whatsthis_btn\'><img id=\'recaptcha_whatsthis\' width="25" height="16" /></a></td> </tr> <tr> <td class=\'recaptcha_r7_c1\'></td> <td class=\'recaptcha_r8_c1\'></td> </tr> </table> ';v.CleanCss=".recaptchatable td img{display:block}.recaptchatable .recaptcha_image_cell center img{height:57px}.recaptchatable .recaptcha_image_cell center{height:57px}.recaptchatable .recaptcha_image_cell{background-color:white;height:57px;padding:7px!important}.recaptchatable,#recaptcha_area tr,#recaptcha_area td,#recaptcha_area th{margin:0!important;border:0!important;border-collapse:collapse!important;vertical-align:middle!important}.recaptchatable *{margin:0;padding:0;border:0;color:black;position:static;top:auto;left:auto;right:auto;bottom:auto}.recaptchatable #recaptcha_image{position:relative;margin:auto;border:1px solid #dfdfdf!important}.recaptchatable #recaptcha_image #recaptcha_challenge_image{display:block}.recaptchatable #recaptcha_image #recaptcha_ad_image{display:block;position:absolute;top:0}.recaptchatable a img{border:0}.recaptchatable a,.recaptchatable a:hover{cursor:pointer;outline:none;border:0!important;padding:0!important;text-decoration:none;color:blue;background:none!important;font-weight:normal}.recaptcha_input_area{position:relative!important;background:none!important}.recaptchatable label.recaptcha_input_area_text{border:1px solid #dfdfdf!important;margin:0!important;padding:0!important;position:static!important;top:auto!important;left:auto!important;right:auto!important;bottom:auto!important}.recaptcha_theme_red label.recaptcha_input_area_text,.recaptcha_theme_white label.recaptcha_input_area_text{color:black!important}.recaptcha_theme_blackglass label.recaptcha_input_area_text{color:white!important}.recaptchatable #recaptcha_response_field{font-size:11pt}.recaptcha_theme_blackglass #recaptcha_response_field,.recaptcha_theme_white #recaptcha_response_field{border:1px solid gray}.recaptcha_theme_red #recaptcha_response_field{border:1px solid #cca940}.recaptcha_audio_cant_hear_link{font-size:7pt;color:black}.recaptchatable{line-height:1em;border:1px solid #dfdfdf!important}.recaptcha_error_text{color:red}.recaptcha_only_if_privacy{float:right;text-align:right;margin-right:7px}#recaptcha-ad-choices{position:absolute;height:15px;top:0;right:0}#recaptcha-ad-choices img{height:15px}.recaptcha-ad-choices-collapsed{width:30px;height:15px;display:block}.recaptcha-ad-choices-expanded{width:75px;height:15px;display:none}#recaptcha-ad-choices:hover .recaptcha-ad-choices-collapsed{display:none}#recaptcha-ad-choices:hover .recaptcha-ad-choices-expanded{display:block}";v.CleanHtml='<table id="recaptcha_table" class="recaptchatable"> <tr height="73"> <td class=\'recaptcha_image_cell\' width="302"><center><div id="recaptcha_image"></div></center></td> <td style="padding: 10px 7px 7px 7px;"> <a id=\'recaptcha_reload_btn\'><img id=\'recaptcha_reload\' width="25" height="18" alt="" /></a> <a id=\'recaptcha_switch_audio_btn\' class="recaptcha_only_if_image"><img id=\'recaptcha_switch_audio\' width="25" height="15" alt="" /></a><a id=\'recaptcha_switch_img_btn\' class="recaptcha_only_if_audio"><img id=\'recaptcha_switch_img\' width="25" height="15" alt=""/></a> <a id=\'recaptcha_whatsthis_btn\'><img id=\'recaptcha_whatsthis\' width="25" height="16" /></a> </td> <td style="padding: 18px 7px 18px 7px;"> <img id=\'recaptcha_logo\' alt="" width="71" height="36" /> </td> </tr> <tr> <td style="padding-left: 7px;"> <div class="recaptcha_input_area" style="padding-top: 2px; padding-bottom: 7px;"> <input style="border: 1px solid #3c3c3c; width: 302px;" name="recaptcha_response_field" id="recaptcha_response_field" type="text" /> </div> </td> <td colspan=2><span id="recaptcha_privacy" class="recaptcha_only_if_privacy"></span></td> </tr> </table> ';v.VertCss=".recaptchatable td img{display:block}.recaptchatable .recaptcha_r1_c1{background:url('IMGROOT/sprite.png') 0 -63px no-repeat;width:318px;height:9px}.recaptchatable .recaptcha_r2_c1{background:url('IMGROOT/sprite.png') -18px 0 no-repeat;width:9px;height:57px}.recaptchatable .recaptcha_r2_c2{background:url('IMGROOT/sprite.png') -27px 0 no-repeat;width:9px;height:57px}.recaptchatable .recaptcha_r3_c1{background:url('IMGROOT/sprite.png') 0 0 no-repeat;width:9px;height:63px}.recaptchatable .recaptcha_r3_c2{background:url('IMGROOT/sprite.png') -18px -57px no-repeat;width:300px;height:6px}.recaptchatable .recaptcha_r3_c3{background:url('IMGROOT/sprite.png') -9px 0 no-repeat;width:9px;height:63px}.recaptchatable .recaptcha_r4_c1{background:url('IMGROOT/sprite.png') -43px 0 no-repeat;width:171px;height:49px}.recaptchatable .recaptcha_r4_c2{background:url('IMGROOT/sprite.png') -36px 0 no-repeat;width:7px;height:57px}.recaptchatable .recaptcha_r4_c4{background:url('IMGROOT/sprite.png') -214px 0 no-repeat;width:97px;height:57px}.recaptchatable .recaptcha_r7_c1{background:url('IMGROOT/sprite.png') -43px -49px no-repeat;width:171px;height:8px}.recaptchatable .recaptcha_r8_c1{background:url('IMGROOT/sprite.png') -43px -49px no-repeat;width:25px;height:8px}.recaptchatable .recaptcha_image_cell center img{height:57px}.recaptchatable .recaptcha_image_cell center{height:57px}.recaptchatable .recaptcha_image_cell{background-color:white;height:57px}#recaptcha_area,#recaptcha_table{width:318px!important}.recaptchatable,#recaptcha_area tr,#recaptcha_area td,#recaptcha_area th{margin:0!important;border:0!important;padding:0!important;border-collapse:collapse!important;vertical-align:middle!important}.recaptchatable *{margin:0;padding:0;border:0;font-family:helvetica,sans-serif;font-size:8pt;color:black;position:static;top:auto;left:auto;right:auto;bottom:auto}.recaptchatable #recaptcha_image{position:relative;margin:auto}.recaptchatable #recaptcha_image #recaptcha_challenge_image{display:block}.recaptchatable #recaptcha_image #recaptcha_ad_image{display:block;position:absolute;top:0}.recaptchatable img{border:0!important;margin:0!important;padding:0!important}.recaptchatable a,.recaptchatable a:hover{cursor:pointer;outline:none;border:0!important;padding:0!important;text-decoration:none;color:blue;background:none!important;font-weight:normal}.recaptcha_input_area{position:relative!important;width:153px!important;height:45px!important;margin-left:7px!important;margin-right:7px!important;background:none!important}.recaptchatable label.recaptcha_input_area_text{margin:0!important;padding:0!important;position:static!important;top:auto!important;left:auto!important;right:auto!important;bottom:auto!important;background:none!important;height:auto!important;width:auto!important}.recaptcha_theme_red label.recaptcha_input_area_text,.recaptcha_theme_white label.recaptcha_input_area_text{color:black!important}.recaptcha_theme_blackglass label.recaptcha_input_area_text{color:white!important}.recaptchatable #recaptcha_response_field{width:153px!important;position:relative!important;bottom:7px!important;padding:0!important;margin:15px 0 0 0!important;font-size:10pt}.recaptcha_theme_blackglass #recaptcha_response_field,.recaptcha_theme_white #recaptcha_response_field{border:1px solid gray}.recaptcha_theme_red #recaptcha_response_field{border:1px solid #cca940}.recaptcha_audio_cant_hear_link{font-size:7pt;color:black}.recaptchatable{line-height:1!important}#recaptcha_instructions_error{color:red!important}.recaptcha_only_if_privacy{float:right;text-align:right}#recaptcha-ad-choices{position:absolute;height:15px;top:0;right:0}#recaptcha-ad-choices img{height:15px}.recaptcha-ad-choices-collapsed{width:30px;height:15px;display:block}.recaptcha-ad-choices-expanded{width:75px;height:15px;display:none}#recaptcha-ad-choices:hover .recaptcha-ad-choices-collapsed{display:none}#recaptcha-ad-choices:hover .recaptcha-ad-choices-expanded{display:block}";var w={visual_challenge:"Get a visual challenge",audio_challenge:"Get an audio challenge",refresh_btn:"Get a new challenge",instructions_visual:"Type the text:",instructions_audio:"Type what you hear:",help_btn:"Help",play_again:"Play sound again",cant_hear_this:"Download sound as MP3",incorrect_try_again:"Incorrect. Try again.",image_alt_text:"reCAPTCHA challenge image",privacy_and_terms:"Privacy & Terms"},ja={visual_challenge:"\u0627\u0644\u062d\u0635\u0648\u0644 \u0639\u0644\u0649 \u062a\u062d\u062f\u064d \u0645\u0631\u0626\u064a",
+audio_challenge:"\u0627\u0644\u062d\u0635\u0648\u0644 \u0639\u0644\u0649 \u062a\u062d\u062f\u064d \u0635\u0648\u062a\u064a",refresh_btn:"\u0627\u0644\u062d\u0635\u0648\u0644 \u0639\u0644\u0649 \u062a\u062d\u062f\u064d \u062c\u062f\u064a\u062f",instructions_visual:"\u064a\u0631\u062c\u0649 \u0643\u062a\u0627\u0628\u0629 \u0627\u0644\u0646\u0635:",instructions_audio:"\u0627\u0643\u062a\u0628 \u0645\u0627 \u062a\u0633\u0645\u0639\u0647:",help_btn:"\u0645\u0633\u0627\u0639\u062f\u0629",play_again:"\u062a\u0634\u063a\u064a\u0644 \u0627\u0644\u0635\u0648\u062a \u0645\u0631\u0629 \u0623\u062e\u0631\u0649",
+cant_hear_this:"\u062a\u0646\u0632\u064a\u0644 \u0627\u0644\u0635\u0648\u062a \u0628\u062a\u0646\u0633\u064a\u0642 MP3",incorrect_try_again:"\u063a\u064a\u0631 \u0635\u062d\u064a\u062d. \u0623\u0639\u062f \u0627\u0644\u0645\u062d\u0627\u0648\u0644\u0629.",image_alt_text:"\u0635\u0648\u0631\u0629 \u0627\u0644\u062a\u062d\u062f\u064a \u0645\u0646 reCAPTCHA",privacy_and_terms:"\u0627\u0644\u062e\u0635\u0648\u0635\u064a\u0629 \u0648\u0627\u0644\u0628\u0646\u0648\u062f"},ka={visual_challenge:"Obtener una pista visual",
+audio_challenge:"Obtener una pista sonora",refresh_btn:"Obtener una pista nueva",instructions_visual:"Introduzca el texto:",instructions_audio:"Escribe lo que oigas:",help_btn:"Ayuda",play_again:"Volver a reproducir el sonido",cant_hear_this:"Descargar el sonido en MP3",incorrect_try_again:"Incorrecto. Vu\u00e9lvelo a intentar.",image_alt_text:"Pista de imagen reCAPTCHA",privacy_and_terms:"Privacidad y condiciones"},la={visual_challenge:"Kumuha ng pagsubok na visual",audio_challenge:"Kumuha ng pagsubok na audio",
+refresh_btn:"Kumuha ng bagong pagsubok",instructions_visual:"I-type ang teksto:",instructions_audio:"I-type ang iyong narinig",help_btn:"Tulong",play_again:"I-play muli ang tunog",cant_hear_this:"I-download ang tunog bilang MP3",incorrect_try_again:"Hindi wasto. Muling subukan.",image_alt_text:"larawang panghamon ng reCAPTCHA",privacy_and_terms:"Privacy at Mga Tuntunin"},ma={visual_challenge:"Test visuel",audio_challenge:"Test audio",refresh_btn:"Nouveau test",instructions_visual:"Saisissez le texte\u00a0:",
+instructions_audio:"Qu'entendez-vous ?",help_btn:"Aide",play_again:"R\u00e9\u00e9couter",cant_hear_this:"T\u00e9l\u00e9charger l'audio au format MP3",incorrect_try_again:"Incorrect. Veuillez r\u00e9essayer.",image_alt_text:"Image reCAPTCHA",privacy_and_terms:"Confidentialit\u00e9 et conditions d'utilisation"},na={visual_challenge:"Dapatkan kata pengujian berbentuk visual",audio_challenge:"Dapatkan kata pengujian berbentuk audio",refresh_btn:"Dapatkan kata pengujian baru",instructions_visual:"Ketik teks:",
+instructions_audio:"Ketik yang Anda dengar:",help_btn:"Bantuan",play_again:"Putar suara sekali lagi",cant_hear_this:"Unduh suara sebagai MP3",incorrect_try_again:"Salah. Coba lagi.",image_alt_text:"Gambar tantangan reCAPTCHA",privacy_and_terms:"Privasi & Persyaratan"},oa={visual_challenge:"\u05e7\u05d1\u05dc \u05d0\u05ea\u05d2\u05e8 \u05d7\u05d6\u05d5\u05ea\u05d9",audio_challenge:"\u05e7\u05d1\u05dc \u05d0\u05ea\u05d2\u05e8 \u05e9\u05de\u05e2",refresh_btn:"\u05e7\u05d1\u05dc \u05d0\u05ea\u05d2\u05e8 \u05d7\u05d3\u05e9",
+instructions_visual:"\u05d4\u05e7\u05dc\u05d3 \u05d0\u05ea \u05d4\u05d8\u05e7\u05e1\u05d8:",instructions_audio:"\u05d4\u05e7\u05dc\u05d3 \u05d0\u05ea \u05de\u05d4 \u05e9\u05d0\u05ea\u05d4 \u05e9\u05d5\u05de\u05e2:",help_btn:"\u05e2\u05d6\u05e8\u05d4",play_again:"\u05d4\u05e4\u05e2\u05dc \u05e9\u05d5\u05d1 \u05d0\u05ea \u05d4\u05e9\u05de\u05e2",cant_hear_this:"\u05d4\u05d5\u05e8\u05d3 \u05e9\u05de\u05e2 \u05db-3MP",incorrect_try_again:"\u05e9\u05d2\u05d5\u05d9. \u05e0\u05e1\u05d4 \u05e9\u05d5\u05d1.",
+image_alt_text:"\u05ea\u05de\u05d5\u05e0\u05ea \u05d0\u05ea\u05d2\u05e8 \u05e9\u05dc reCAPTCHA",privacy_and_terms:"\u05e4\u05e8\u05d8\u05d9\u05d5\u05ea \u05d5\u05ea\u05e0\u05d0\u05d9\u05dd"},pa={visual_challenge:"Obter um desafio visual",audio_challenge:"Obter um desafio de \u00e1udio",refresh_btn:"Obter um novo desafio",instructions_visual:"Digite o texto:",instructions_audio:"Digite o que voc\u00ea ouve:",help_btn:"Ajuda",play_again:"Reproduzir som novamente",cant_hear_this:"Fazer download do som no formato MP3",
+incorrect_try_again:"Incorreto. Tente novamente.",image_alt_text:"Imagem de desafio reCAPTCHA",privacy_and_terms:"Privacidade e Termos"},qa={visual_challenge:"Ob\u0163ine\u0163i un cod captcha vizual",audio_challenge:"Ob\u0163ine\u0163i un cod captcha audio",refresh_btn:"Ob\u0163ine\u0163i un nou cod captcha",instructions_visual:"Introduce\u021bi textul:",instructions_audio:"Introduce\u0163i ceea ce auzi\u0163i:",help_btn:"Ajutor",play_again:"Reda\u0163i sunetul din nou",cant_hear_this:"Desc\u0103rca\u0163i fi\u015fierul audio ca MP3",
+incorrect_try_again:"Incorect. \u00cencerca\u0163i din nou.",image_alt_text:"Imagine de verificare reCAPTCHA",privacy_and_terms:"Confiden\u0163ialitate \u015fi termeni"},ra={visual_challenge:"\u6536\u5230\u4e00\u4e2a\u89c6\u9891\u9080\u8bf7",audio_challenge:"\u6362\u4e00\u7ec4\u97f3\u9891\u9a8c\u8bc1\u7801",refresh_btn:"\u6362\u4e00\u7ec4\u9a8c\u8bc1\u7801",instructions_visual:"\u8f93\u5165\u6587\u5b57\uff1a",instructions_audio:"\u8bf7\u952e\u5165\u60a8\u542c\u5230\u7684\u5185\u5bb9\uff1a",help_btn:"\u5e2e\u52a9",
+play_again:"\u91cd\u65b0\u64ad\u653e",cant_hear_this:"\u4ee5 MP3 \u683c\u5f0f\u4e0b\u8f7d\u58f0\u97f3",incorrect_try_again:"\u4e0d\u6b63\u786e\uff0c\u8bf7\u91cd\u8bd5\u3002",image_alt_text:"reCAPTCHA \u9a8c\u8bc1\u56fe\u7247",privacy_and_terms:"\u9690\u79c1\u6743\u548c\u4f7f\u7528\u6761\u6b3e"},sa={en:w,af:{visual_challenge:"Kry 'n visuele verifi\u00ebring",audio_challenge:"Kry 'n klankverifi\u00ebring",refresh_btn:"Kry 'n nuwe verifi\u00ebring",instructions_visual:"",instructions_audio:"Tik wat jy hoor:",
+help_btn:"Hulp",play_again:"Speel geluid weer",cant_hear_this:"Laai die klank af as MP3",incorrect_try_again:"Verkeerd. Probeer weer.",image_alt_text:"reCAPTCHA-uitdagingprent",privacy_and_terms:"Privaatheid en bepalings"},am:{visual_challenge:"\u12e8\u12a5\u12ed\u1273 \u1270\u130b\u1323\u121a \u12a0\u130d\u129d",audio_challenge:"\u120c\u120b \u12a0\u12f2\u1235 \u12e8\u12f5\u121d\u133d \u1325\u12eb\u1244 \u12ed\u1245\u1228\u1265",refresh_btn:"\u120c\u120b \u12a0\u12f2\u1235 \u1325\u12eb\u1244 \u12ed\u1245\u1228\u1265",
+instructions_visual:"",instructions_audio:"\u12e8\u121d\u1275\u1230\u121b\u12cd\u1295 \u1270\u12ed\u1265\u1361-",help_btn:"\u12a5\u1308\u12db",play_again:"\u12f5\u121d\u1339\u1295 \u12a5\u1295\u12f0\u1308\u1293 \u12a0\u132b\u12cd\u1275",cant_hear_this:"\u12f5\u121d\u1339\u1295 \u1260MP3 \u1245\u122d\u133d \u12a0\u12cd\u122d\u12f5",incorrect_try_again:"\u1275\u12ad\u12ad\u120d \u12a0\u12ed\u12f0\u1208\u121d\u1362 \u12a5\u1295\u12f0\u1308\u1293 \u121e\u12ad\u122d\u1362",image_alt_text:"reCAPTCHA \u121d\u1235\u120d \u130d\u1320\u121d",
+privacy_and_terms:"\u130d\u120b\u12ca\u1290\u1275 \u12a5\u1293 \u12cd\u120d"},ar:ja,"ar-EG":ja,bg:{visual_challenge:"\u041f\u043e\u043b\u0443\u0447\u0430\u0432\u0430\u043d\u0435 \u043d\u0430 \u0432\u0438\u0437\u0443\u0430\u043b\u043d\u0430 \u043f\u0440\u043e\u0432\u0435\u0440\u043a\u0430",audio_challenge:"\u0417\u0430\u0440\u0435\u0436\u0434\u0430\u043d\u0435 \u043d\u0430 \u0430\u0443\u0434\u0438\u043e\u0442\u0435\u0441\u0442",refresh_btn:"\u0417\u0430\u0440\u0435\u0436\u0434\u0430\u043d\u0435 \u043d\u0430 \u043d\u043e\u0432 \u0442\u0435\u0441\u0442",
+instructions_visual:"\u0412\u044a\u0432\u0435\u0434\u0435\u0442\u0435 \u0442\u0435\u043a\u0441\u0442\u0430:",instructions_audio:"\u0412\u044a\u0432\u0435\u0434\u0435\u0442\u0435 \u0447\u0443\u0442\u043e\u0442\u043e:",help_btn:"\u041f\u043e\u043c\u043e\u0449",play_again:"\u041f\u043e\u0432\u0442\u043e\u0440\u043d\u043e \u043f\u0443\u0441\u043a\u0430\u043d\u0435 \u043d\u0430 \u0437\u0432\u0443\u043a\u0430",cant_hear_this:"\u0418\u0437\u0442\u0435\u0433\u043b\u044f\u043d\u0435 \u043d\u0430 \u0437\u0432\u0443\u043a\u0430 \u0432\u044a\u0432 \u0444\u043e\u0440\u043c\u0430\u0442 MP3",
+incorrect_try_again:"\u041d\u0435\u043f\u0440\u0430\u0432\u0438\u043b\u043d\u043e. \u041e\u043f\u0438\u0442\u0430\u0439\u0442\u0435 \u043e\u0442\u043d\u043e\u0432\u043e.",image_alt_text:"\u0418\u0437\u043e\u0431\u0440\u0430\u0436\u0435\u043d\u0438\u0435 \u043d\u0430 \u043f\u0440\u043e\u0432\u0435\u0440\u043a\u0430\u0442\u0430 \u0441 reCAPTCHA",privacy_and_terms:"\u041f\u043e\u0432\u0435\u0440\u0438\u0442\u0435\u043b\u043d\u043e\u0441\u0442 \u0438 \u041e\u0431\u0449\u0438 \u0443\u0441\u043b\u043e\u0432\u0438\u044f"},
+bn:{visual_challenge:"\u098f\u0995\u099f\u09bf \u09a6\u09c3\u09b6\u09cd\u09af\u09ae\u09be\u09a8 \u09aa\u09cd\u09b0\u09a4\u09bf\u09a6\u09cd\u09ac\u09a8\u09cd\u09a6\u09cd\u09ac\u09bf\u09a4\u09be \u09aa\u09be\u09a8",audio_challenge:"\u098f\u0995\u099f\u09bf \u0985\u09a1\u09bf\u0993 \u09aa\u09cd\u09b0\u09a4\u09bf\u09a6\u09cd\u09ac\u09a8\u09cd\u09a6\u09cd\u09ac\u09bf\u09a4\u09be  \u09aa\u09be\u09a8",refresh_btn:"\u098f\u0995\u099f\u09bf \u09a8\u09a4\u09c1\u09a8 \u09aa\u09cd\u09b0\u09a4\u09bf\u09a6\u09cd\u09ac\u09a8\u09cd\u09a6\u09cd\u09ac\u09bf\u09a4\u09be  \u09aa\u09be\u09a8",
+instructions_visual:"",instructions_audio:"\u0986\u09aa\u09a8\u09bf \u09af\u09be \u09b6\u09c1\u09a8\u099b\u09c7\u09a8 \u09a4\u09be \u09b2\u09bf\u0996\u09c1\u09a8:",help_btn:"\u09b8\u09b9\u09be\u09df\u09a4\u09be",play_again:"\u0986\u09ac\u09be\u09b0 \u09b8\u09be\u0989\u09a8\u09cd\u09a1 \u09aa\u09cd\u09b2\u09c7 \u0995\u09b0\u09c1\u09a8",cant_hear_this:"MP3 \u09b0\u09c2\u09aa\u09c7 \u09b6\u09ac\u09cd\u09a6 \u09a1\u09be\u0989\u09a8\u09b2\u09cb\u09a1 \u0995\u09b0\u09c1\u09a8",incorrect_try_again:"\u09ac\u09c7\u09a0\u09bf\u0995\u09f7 \u0986\u09ac\u09be\u09b0 \u099a\u09c7\u09b7\u09cd\u099f\u09be \u0995\u09b0\u09c1\u09a8\u09f7",
+image_alt_text:"reCAPTCHA \u099a\u09cd\u09af\u09be\u09b2\u09c7\u099e\u09cd\u099c \u099a\u09bf\u09a4\u09cd\u09b0",privacy_and_terms:"\u0997\u09cb\u09aa\u09a8\u09c0\u09af\u09bc\u09a4\u09be \u0993 \u09b6\u09b0\u09cd\u09a4\u09be\u09ac\u09b2\u09c0"},ca:{visual_challenge:"Obt\u00e9n un repte visual",audio_challenge:"Obteniu una pista sonora",refresh_btn:"Obteniu una pista nova",instructions_visual:"Escriviu el text:",instructions_audio:"Escriviu el que escolteu:",help_btn:"Ajuda",play_again:"Torna a reproduir el so",
+cant_hear_this:"Baixa el so com a MP3",incorrect_try_again:"No \u00e9s correcte. Torna-ho a provar.",image_alt_text:"Imatge del repte de reCAPTCHA",privacy_and_terms:"Privadesa i condicions"},cs:{visual_challenge:"Zobrazit vizu\u00e1ln\u00ed podobu v\u00fdrazu",audio_challenge:"P\u0159ehr\u00e1t zvukovou podobu v\u00fdrazu",refresh_btn:"Zobrazit nov\u00fd v\u00fdraz",instructions_visual:"Zadejte text:",instructions_audio:"Napi\u0161te, co jste sly\u0161eli:",help_btn:"N\u00e1pov\u011bda",play_again:"Znovu p\u0159ehr\u00e1t zvuk",
+cant_hear_this:"St\u00e1hnout zvuk ve form\u00e1tu MP3",incorrect_try_again:"\u0160patn\u011b. Zkuste to znovu.",image_alt_text:"Obr\u00e1zek reCAPTCHA",privacy_and_terms:"Ochrana soukrom\u00ed a smluvn\u00ed podm\u00ednky"},da:{visual_challenge:"Hent en visuel udfordring",audio_challenge:"Hent en lydudfordring",refresh_btn:"Hent en ny udfordring",instructions_visual:"Indtast teksten:",instructions_audio:"Indtast det, du h\u00f8rer:",help_btn:"Hj\u00e6lp",play_again:"Afspil lyden igen",cant_hear_this:"Download lyd som MP3",
+incorrect_try_again:"Forkert. Pr\u00f8v igen.",image_alt_text:"reCAPTCHA-udfordringsbillede",privacy_and_terms:"Privatliv og vilk\u00e5r"},de:{visual_challenge:"Captcha abrufen",audio_challenge:"Audio-Captcha abrufen",refresh_btn:"Neues Captcha abrufen",instructions_visual:"Geben Sie den angezeigten Text ein:",instructions_audio:"Geben Sie das Geh\u00f6rte ein:",help_btn:"Hilfe",play_again:"Wort erneut abspielen",cant_hear_this:"Wort als MP3 herunterladen",incorrect_try_again:"Falsch. Bitte versuchen Sie es erneut.",
+image_alt_text:"reCAPTCHA-Bild",privacy_and_terms:"Datenschutzerkl\u00e4rung & Nutzungsbedingungen"},el:{visual_challenge:"\u039f\u03c0\u03c4\u03b9\u03ba\u03ae \u03c0\u03c1\u03cc\u03ba\u03bb\u03b7\u03c3\u03b7",audio_challenge:"\u0397\u03c7\u03b7\u03c4\u03b9\u03ba\u03ae \u03c0\u03c1\u03cc\u03ba\u03bb\u03b7\u03c3\u03b7",refresh_btn:"\u039d\u03ad\u03b1 \u03c0\u03c1\u03cc\u03ba\u03bb\u03b7\u03c3\u03b7",instructions_visual:"\u03a0\u03bb\u03b7\u03ba\u03c4\u03c1\u03bf\u03bb\u03bf\u03b3\u03ae\u03c3\u03c4\u03b5 \u03c4\u03bf \u03ba\u03b5\u03af\u03bc\u03b5\u03bd\u03bf:",
+instructions_audio:"\u03a0\u03bb\u03b7\u03ba\u03c4\u03c1\u03bf\u03bb\u03bf\u03b3\u03ae\u03c3\u03c4\u03b5 \u03cc\u03c4\u03b9 \u03b1\u03ba\u03bf\u03cd\u03c4\u03b5:",help_btn:"\u0392\u03bf\u03ae\u03b8\u03b5\u03b9\u03b1",play_again:"\u0391\u03bd\u03b1\u03c0\u03b1\u03c1\u03b1\u03b3\u03c9\u03b3\u03ae \u03ae\u03c7\u03bf\u03c5 \u03be\u03b1\u03bd\u03ac",cant_hear_this:"\u039b\u03ae\u03c8\u03b7 \u03ae\u03c7\u03bf\u03c5 \u03c9\u03c2 \u039c\u03a13",incorrect_try_again:"\u039b\u03ac\u03b8\u03bf\u03c2. \u0394\u03bf\u03ba\u03b9\u03bc\u03ac\u03c3\u03c4\u03b5 \u03be\u03b1\u03bd\u03ac.",
+image_alt_text:"\u0395\u03b9\u03ba\u03cc\u03bd\u03b1 \u03c0\u03c1\u03cc\u03ba\u03bb\u03b7\u03c3\u03b7\u03c2 reCAPTCHA",privacy_and_terms:"\u0391\u03c0\u03cc\u03c1\u03c1\u03b7\u03c4\u03bf \u03ba\u03b1\u03b9 \u03cc\u03c1\u03bf\u03b9"},"en-GB":w,"en-US":w,es:ka,"es-419":{visual_challenge:"Enfrentar un desaf\u00edo visual",audio_challenge:"Enfrentar un desaf\u00edo de audio",refresh_btn:"Enfrentar un nuevo desaf\u00edo",instructions_visual:"Escriba el texto:",instructions_audio:"Escribe lo que escuchas:",
+help_btn:"Ayuda",play_again:"Reproducir sonido de nuevo",cant_hear_this:"Descargar sonido en formato MP3",incorrect_try_again:"Incorrecto. Vuelve a intentarlo.",image_alt_text:"Imagen del desaf\u00edo de la reCAPTCHA",privacy_and_terms:"Privacidad y condiciones"},"es-ES":ka,et:{visual_challenge:"Kuva kuvap\u00f5hine robotil\u00f5ks",audio_challenge:"Kuva helip\u00f5hine robotil\u00f5ks",refresh_btn:"Kuva uus robotil\u00f5ks",instructions_visual:"Tippige tekst:",instructions_audio:"Tippige, mida kuulete.",
+help_btn:"Abi",play_again:"Esita heli uuesti",cant_hear_this:"Laadi heli alla MP3-vormingus",incorrect_try_again:"Vale. Proovige uuesti.",image_alt_text:"reCAPTCHA robotil\u00f5ksu kujutis",privacy_and_terms:"Privaatsus ja tingimused"},eu:{visual_challenge:"Eskuratu ikusizko erronka",audio_challenge:"Eskuratu audio-erronka",refresh_btn:"Eskuratu erronka berria",instructions_visual:"",instructions_audio:"Idatzi entzuten duzuna:",help_btn:"Laguntza",play_again:"Erreproduzitu soinua berriro",cant_hear_this:"Deskargatu soinua MP3 gisa",
+incorrect_try_again:"Ez da zuzena. Saiatu berriro.",image_alt_text:"reCAPTCHA erronkaren irudia",privacy_and_terms:"Pribatutasuna eta baldintzak"},fa:{visual_challenge:"\u062f\u0631\u06cc\u0627\u0641\u062a \u06cc\u06a9 \u0645\u0639\u0645\u0627\u06cc \u062f\u06cc\u062f\u0627\u0631\u06cc",audio_challenge:"\u062f\u0631\u06cc\u0627\u0641\u062a \u06cc\u06a9 \u0645\u0639\u0645\u0627\u06cc \u0635\u0648\u062a\u06cc",refresh_btn:"\u062f\u0631\u06cc\u0627\u0641\u062a \u06cc\u06a9 \u0645\u0639\u0645\u0627\u06cc \u062c\u062f\u06cc\u062f",
+instructions_visual:"",instructions_audio:"\u0622\u0646\u0686\u0647 \u0631\u0627 \u06a9\u0647 \u0645\u06cc\u200c\u0634\u0646\u0648\u06cc\u062f \u062a\u0627\u06cc\u067e \u06a9\u0646\u06cc\u062f:",help_btn:"\u0631\u0627\u0647\u0646\u0645\u0627\u06cc\u06cc",play_again:"\u067e\u062e\u0634 \u0645\u062c\u062f\u062f \u0635\u062f\u0627",cant_hear_this:"\u062f\u0627\u0646\u0644\u0648\u062f \u0635\u062f\u0627 \u0628\u0647 \u0635\u0648\u0631\u062a MP3",incorrect_try_again:"\u0646\u0627\u062f\u0631\u0633\u062a. \u062f\u0648\u0628\u0627\u0631\u0647 \u0627\u0645\u062a\u062d\u0627\u0646 \u06a9\u0646\u06cc\u062f.",
+image_alt_text:"\u062a\u0635\u0648\u06cc\u0631 \u0686\u0627\u0644\u0634\u06cc reCAPTCHA",privacy_and_terms:"\u062d\u0631\u06cc\u0645 \u062e\u0635\u0648\u0635\u06cc \u0648 \u0634\u0631\u0627\u06cc\u0637"},fi:{visual_challenge:"Kuvavahvistus",audio_challenge:"\u00c4\u00e4nivahvistus",refresh_btn:"Uusi kuva",instructions_visual:"Kirjoita teksti:",instructions_audio:"Kirjoita kuulemasi:",help_btn:"Ohje",play_again:"Toista \u00e4\u00e4ni uudelleen",cant_hear_this:"Lataa \u00e4\u00e4ni MP3-tiedostona",
+incorrect_try_again:"V\u00e4\u00e4rin. Yrit\u00e4 uudelleen.",image_alt_text:"reCAPTCHA-kuva",privacy_and_terms:"Tietosuoja ja k\u00e4ytt\u00f6ehdot"},fil:la,fr:ma,"fr-CA":{visual_challenge:"Obtenir un test visuel",audio_challenge:"Obtenir un test audio",refresh_btn:"Obtenir un nouveau test",instructions_visual:"Saisissez le texte\u00a0:",instructions_audio:"Tapez ce que vous entendez\u00a0:",help_btn:"Aide",play_again:"Jouer le son de nouveau",cant_hear_this:"T\u00e9l\u00e9charger le son en format MP3",
+incorrect_try_again:"Erreur, essayez \u00e0 nouveau",image_alt_text:"Image reCAPTCHA",privacy_and_terms:"Confidentialit\u00e9 et conditions d'utilisation"},"fr-FR":ma,gl:{visual_challenge:"Obter unha proba visual",audio_challenge:"Obter unha proba de audio",refresh_btn:"Obter unha proba nova",instructions_visual:"",instructions_audio:"Escribe o que escoitas:",help_btn:"Axuda",play_again:"Reproducir o son de novo",cant_hear_this:"Descargar son como MP3",incorrect_try_again:"Incorrecto. T\u00e9ntao de novo.",
+image_alt_text:"Imaxe de proba de reCAPTCHA",privacy_and_terms:"Privacidade e condici\u00f3ns"},gu:{visual_challenge:"\u0a8f\u0a95 \u0aa6\u0ac3\u0ab6\u0acd\u0aaf\u0abe\u0aa4\u0acd\u0aae\u0a95 \u0aaa\u0aa1\u0a95\u0abe\u0ab0 \u0aae\u0ac7\u0ab3\u0ab5\u0acb",audio_challenge:"\u0a8f\u0a95 \u0a91\u0aa1\u0abf\u0a93 \u0aaa\u0aa1\u0a95\u0abe\u0ab0 \u0aae\u0ac7\u0ab3\u0ab5\u0acb",refresh_btn:"\u0a8f\u0a95 \u0aa8\u0ab5\u0acb \u0aaa\u0aa1\u0a95\u0abe\u0ab0 \u0aae\u0ac7\u0ab3\u0ab5\u0acb",instructions_visual:"",
+instructions_audio:"\u0aa4\u0aae\u0ac7 \u0a9c\u0ac7 \u0ab8\u0abe\u0a82\u0aad\u0ab3\u0acb \u0a9b\u0acb \u0aa4\u0ac7 \u0ab2\u0a96\u0acb:",help_btn:"\u0ab8\u0ab9\u0abe\u0aaf",play_again:"\u0aa7\u0acd\u0ab5\u0aa8\u0abf \u0aab\u0ab0\u0ac0\u0aa5\u0ac0 \u0a9a\u0ab2\u0abe\u0ab5\u0acb",cant_hear_this:"MP3 \u0aa4\u0ab0\u0ac0\u0a95\u0ac7 \u0aa7\u0acd\u0ab5\u0aa8\u0abf\u0aa8\u0ac7 \u0aa1\u0abe\u0a89\u0aa8\u0ab2\u0acb\u0aa1 \u0a95\u0ab0\u0acb",incorrect_try_again:"\u0a96\u0acb\u0a9f\u0ac1\u0a82. \u0aab\u0ab0\u0ac0 \u0aaa\u0acd\u0ab0\u0aaf\u0abe\u0ab8 \u0a95\u0ab0\u0acb.",
+image_alt_text:"reCAPTCHA \u0aaa\u0aa1\u0a95\u0abe\u0ab0 \u0a9b\u0aac\u0ac0",privacy_and_terms:"\u0a97\u0acb\u0aaa\u0aa8\u0ac0\u0aaf\u0aa4\u0abe \u0a85\u0aa8\u0ac7 \u0ab6\u0ab0\u0aa4\u0acb"},hi:{visual_challenge:"\u0915\u094b\u0908 \u0935\u093f\u091c\u0941\u0905\u0932 \u091a\u0941\u0928\u094c\u0924\u0940 \u0932\u0947\u0902",audio_challenge:"\u0915\u094b\u0908 \u0911\u0921\u093f\u092f\u094b \u091a\u0941\u0928\u094c\u0924\u0940 \u0932\u0947\u0902",refresh_btn:"\u0915\u094b\u0908 \u0928\u0908 \u091a\u0941\u0928\u094c\u0924\u0940 \u0932\u0947\u0902",
+instructions_visual:"\u091f\u0947\u0915\u094d\u0938\u094d\u091f \u091f\u093e\u0907\u092a \u0915\u0930\u0947\u0902:",instructions_audio:"\u091c\u094b \u0906\u092a \u0938\u0941\u0928 \u0930\u0939\u0947 \u0939\u0948\u0902 \u0909\u0938\u0947 \u0932\u093f\u0916\u0947\u0902:",help_btn:"\u0938\u0939\u093e\u092f\u0924\u093e",play_again:"\u0927\u094d\u200d\u0935\u0928\u093f \u092a\u0941\u0928: \u091a\u0932\u093e\u090f\u0902",cant_hear_this:"\u0927\u094d\u200d\u0935\u0928\u093f \u0915\u094b MP3 \u0915\u0947 \u0930\u0942\u092a \u092e\u0947\u0902 \u0921\u093e\u0909\u0928\u0932\u094b\u0921 \u0915\u0930\u0947\u0902",
+incorrect_try_again:"\u0917\u0932\u0924. \u092a\u0941\u0928: \u092a\u094d\u0930\u092f\u093e\u0938 \u0915\u0930\u0947\u0902.",image_alt_text:"reCAPTCHA \u091a\u0941\u0928\u094c\u0924\u0940 \u091a\u093f\u0924\u094d\u0930",privacy_and_terms:"\u0917\u094b\u092a\u0928\u0940\u092f\u0924\u093e \u0914\u0930 \u0936\u0930\u094d\u0924\u0947\u0902"},hr:{visual_challenge:"Dohvati vizualni upit",audio_challenge:"Dohvati zvu\u010dni upit",refresh_btn:"Dohvati novi upit",instructions_visual:"Unesite tekst:",instructions_audio:"Upi\u0161ite \u0161to \u010dujete:",
+help_btn:"Pomo\u0107",play_again:"Ponovi zvuk",cant_hear_this:"Preuzmi zvuk u MP3 formatu",incorrect_try_again:"Nije to\u010dno. Poku\u0161ajte ponovno.",image_alt_text:"Slikovni izazov reCAPTCHA",privacy_and_terms:"Privatnost i odredbe"},hu:{visual_challenge:"Vizu\u00e1lis kih\u00edv\u00e1s k\u00e9r\u00e9se",audio_challenge:"Hangkih\u00edv\u00e1s k\u00e9r\u00e9se",refresh_btn:"\u00daj kih\u00edv\u00e1s k\u00e9r\u00e9se",instructions_visual:"\u00cdrja be a sz\u00f6veget:",instructions_audio:"\u00cdrja le, amit hall:",
+help_btn:"S\u00fag\u00f3",play_again:"Hang ism\u00e9telt lej\u00e1tsz\u00e1sa",cant_hear_this:"Hang let\u00f6lt\u00e9se MP3 form\u00e1tumban",incorrect_try_again:"Hib\u00e1s. Pr\u00f3b\u00e1lkozzon \u00fajra.",image_alt_text:"reCAPTCHA ellen\u0151rz\u0151 k\u00e9p",privacy_and_terms:"Adatv\u00e9delem \u00e9s Szerz\u0151d\u00e9si Felt\u00e9telek"},hy:{visual_challenge:"\u054d\u057f\u0561\u0576\u0561\u056c \u057f\u0565\u057d\u0578\u0572\u0561\u056f\u0561\u0576 \u056d\u0576\u0564\u056b\u0580",audio_challenge:"\u054d\u057f\u0561\u0576\u0561\u056c \u0571\u0561\u0575\u0576\u0561\u0575\u056b\u0576 \u056d\u0576\u0564\u056b\u0580",
+refresh_btn:"\u054d\u057f\u0561\u0576\u0561\u056c \u0576\u0578\u0580 \u056d\u0576\u0564\u056b\u0580",instructions_visual:"\u0544\u0578\u0582\u057f\u0584\u0561\u0563\u0580\u0565\u0584 \u057f\u0565\u0584\u057d\u057f\u0568\u055d",instructions_audio:"\u0544\u0578\u0582\u057f\u0584\u0561\u0563\u0580\u0565\u0584 \u0561\u0575\u0576, \u056b\u0576\u0579 \u056c\u057d\u0578\u0582\u0574 \u0565\u0584\u055d",help_btn:"\u0555\u0563\u0576\u0578\u0582\u0569\u0575\u0578\u0582\u0576",play_again:"\u0546\u057e\u0561\u0563\u0561\u0580\u056f\u0565\u056c \u0571\u0561\u0575\u0576\u0568 \u056f\u0580\u056f\u056b\u0576",
+cant_hear_this:"\u0532\u0565\u057c\u0576\u0565\u056c \u0571\u0561\u0575\u0576\u0568 \u0578\u0580\u057a\u0565\u057d MP3",incorrect_try_again:"\u054d\u056d\u0561\u056c \u0567: \u0553\u0578\u0580\u0571\u0565\u0584 \u056f\u0580\u056f\u056b\u0576:",image_alt_text:"reCAPTCHA \u057a\u0561\u057f\u056f\u0565\u0580\u0578\u057e \u056d\u0576\u0564\u056b\u0580",privacy_and_terms:"\u0533\u0561\u0572\u057f\u0576\u056b\u0578\u0582\u0569\u0575\u0561\u0576 & \u057a\u0561\u0575\u0574\u0561\u0576\u0576\u0565\u0580"},
+id:na,is:{visual_challenge:"F\u00e1 a\u00f0gangspr\u00f3f sem mynd",audio_challenge:"F\u00e1 a\u00f0gangspr\u00f3f sem hlj\u00f3\u00f0skr\u00e1",refresh_btn:"F\u00e1 n\u00fdtt a\u00f0gangspr\u00f3f",instructions_visual:"",instructions_audio:"Sl\u00e1\u00f0u inn \u00fea\u00f0 sem \u00fe\u00fa heyrir:",help_btn:"Hj\u00e1lp",play_again:"Spila hlj\u00f3\u00f0 aftur",cant_hear_this:"S\u00e6kja hlj\u00f3\u00f0 sem MP3",incorrect_try_again:"Rangt. Reyndu aftur.",image_alt_text:"mynd reCAPTCHA a\u00f0gangspr\u00f3fs",
+privacy_and_terms:"Pers\u00f3nuvernd og skilm\u00e1lar"},it:{visual_challenge:"Verifica visiva",audio_challenge:"Verifica audio",refresh_btn:"Nuova verifica",instructions_visual:"Digita il testo:",instructions_audio:"Digita ci\u00f2 che senti:",help_btn:"Guida",play_again:"Riproduci di nuovo audio",cant_hear_this:"Scarica audio in MP3",incorrect_try_again:"Sbagliato. Riprova.",image_alt_text:"Immagine di verifica reCAPTCHA",privacy_and_terms:"Privacy e Termini"},iw:oa,ja:{visual_challenge:"\u753b\u50cf\u3067\u78ba\u8a8d\u3057\u307e\u3059",
+audio_challenge:"\u97f3\u58f0\u3067\u78ba\u8a8d\u3057\u307e\u3059",refresh_btn:"\u5225\u306e\u5358\u8a9e\u3067\u3084\u308a\u76f4\u3057\u307e\u3059",instructions_visual:"\u30c6\u30ad\u30b9\u30c8\u3092\u5165\u529b:",instructions_audio:"\u805e\u3053\u3048\u305f\u5358\u8a9e\u3092\u5165\u529b\u3057\u307e\u3059:",help_btn:"\u30d8\u30eb\u30d7",play_again:"\u3082\u3046\u4e00\u5ea6\u805e\u304f",cant_hear_this:"MP3 \u3067\u97f3\u58f0\u3092\u30c0\u30a6\u30f3\u30ed\u30fc\u30c9",incorrect_try_again:"\u6b63\u3057\u304f\u3042\u308a\u307e\u305b\u3093\u3002\u3082\u3046\u4e00\u5ea6\u3084\u308a\u76f4\u3057\u3066\u304f\u3060\u3055\u3044\u3002",
+image_alt_text:"reCAPTCHA \u78ba\u8a8d\u7528\u753b\u50cf",privacy_and_terms:"\u30d7\u30e9\u30a4\u30d0\u30b7\u30fc\u3068\u5229\u7528\u898f\u7d04"},kn:{visual_challenge:"\u0ca6\u0cc3\u0cb6\u0ccd\u0caf \u0cb8\u0cb5\u0cbe\u0cb2\u0cca\u0c82\u0ca6\u0ca8\u0ccd\u0ca8\u0cc1 \u0cb8\u0ccd\u0cb5\u0cc0\u0c95\u0cb0\u0cbf\u0cb8\u0cbf",audio_challenge:"\u0c86\u0ca1\u0cbf\u0caf\u0ccb \u0cb8\u0cb5\u0cbe\u0cb2\u0cca\u0c82\u0ca6\u0ca8\u0ccd\u0ca8\u0cc1 \u0cb8\u0ccd\u0cb5\u0cc0\u0c95\u0cb0\u0cbf\u0cb8\u0cbf",refresh_btn:"\u0cb9\u0cca\u0cb8 \u0cb8\u0cb5\u0cbe\u0cb2\u0cca\u0c82\u0ca6\u0ca8\u0ccd\u0ca8\u0cc1 \u0caa\u0ca1\u0cc6\u0caf\u0cbf\u0cb0\u0cbf",
+instructions_visual:"",instructions_audio:"\u0ca8\u0cbf\u0cae\u0c97\u0cc6 \u0c95\u0cc7\u0cb3\u0cbf\u0cb8\u0cc1\u0cb5\u0cc1\u0ca6\u0ca8\u0ccd\u0ca8\u0cc1 \u0c9f\u0cc8\u0caa\u0ccd\u200c \u0cae\u0cbe\u0ca1\u0cbf:",help_btn:"\u0cb8\u0cb9\u0cbe\u0caf",play_again:"\u0ca7\u0ccd\u0cb5\u0ca8\u0cbf\u0caf\u0ca8\u0ccd\u0ca8\u0cc1 \u0cae\u0ca4\u0ccd\u0ca4\u0cc6 \u0caa\u0ccd\u0cb2\u0cc7 \u0cae\u0cbe\u0ca1\u0cbf",cant_hear_this:"\u0ca7\u0ccd\u0cb5\u0ca8\u0cbf\u0caf\u0ca8\u0ccd\u0ca8\u0cc1 MP3 \u0cb0\u0cc2\u0caa\u0ca6\u0cb2\u0ccd\u0cb2\u0cbf \u0ca1\u0ccc\u0ca8\u0ccd\u200c\u0cb2\u0ccb\u0ca1\u0ccd \u0cae\u0cbe\u0ca1\u0cbf",
+incorrect_try_again:"\u0ca4\u0caa\u0ccd\u0caa\u0cbe\u0c97\u0cbf\u0ca6\u0cc6. \u0cae\u0ca4\u0ccd\u0ca4\u0cca\u0cae\u0ccd\u0cae\u0cc6 \u0caa\u0ccd\u0cb0\u0caf\u0ca4\u0ccd\u0ca8\u0cbf\u0cb8\u0cbf.",image_alt_text:"reCAPTCHA \u0cb8\u0cb5\u0cbe\u0cb2\u0cc1 \u0c9a\u0cbf\u0ca4\u0ccd\u0cb0",privacy_and_terms:"\u0c97\u0ccc\u0caa\u0ccd\u0caf\u0ca4\u0cc6 \u0cae\u0ca4\u0ccd\u0ca4\u0cc1 \u0ca8\u0cbf\u0caf\u0cae\u0c97\u0cb3\u0cc1"},ko:{visual_challenge:"\uadf8\ub9bc\uc73c\ub85c \ubcf4\uc548\ubb38\uc790 \ubc1b\uae30",
+audio_challenge:"\uc74c\uc131\uc73c\ub85c \ubcf4\uc548\ubb38\uc790 \ubc1b\uae30",refresh_btn:"\ubcf4\uc548\ubb38\uc790 \uc0c8\ub85c \ubc1b\uae30",instructions_visual:"\ud14d\uc2a4\ud2b8 \uc785\ub825:",instructions_audio:"\uc74c\uc131 \ubcf4\uc548\ubb38\uc790 \uc785\ub825:",help_btn:"\ub3c4\uc6c0\ub9d0",play_again:"\uc74c\uc131 \ub2e4\uc2dc \ub4e3\uae30",cant_hear_this:"\uc74c\uc131\uc744 MP3\ub85c \ub2e4\uc6b4\ub85c\ub4dc",incorrect_try_again:"\ud2c0\ub838\uc2b5\ub2c8\ub2e4. \ub2e4\uc2dc \uc2dc\ub3c4\ud574 \uc8fc\uc138\uc694.",
+image_alt_text:"reCAPTCHA \ubcf4\uc548\ubb38\uc790 \uc774\ubbf8\uc9c0",privacy_and_terms:"\uac1c\uc778\uc815\ubcf4 \ubcf4\ud638 \ubc0f \uc57d\uad00"},ln:ma,lt:{visual_challenge:"Gauti vaizdin\u012f atpa\u017einimo test\u0105",audio_challenge:"Gauti garso atpa\u017einimo test\u0105",refresh_btn:"Gauti nauj\u0105 atpa\u017einimo test\u0105",instructions_visual:"\u012eveskite tekst\u0105:",instructions_audio:"\u012eveskite tai, k\u0105 girdite:",help_btn:"Pagalba",play_again:"Dar kart\u0105 paleisti gars\u0105",
+cant_hear_this:"Atsisi\u0173sti gars\u0105 kaip MP3",incorrect_try_again:"Neteisingai. Bandykite dar kart\u0105.",image_alt_text:"Testo \u201ereCAPTCHA\u201c vaizdas",privacy_and_terms:"Privatumas ir s\u0105lygos"},lv:{visual_challenge:"Sa\u0146emt vizu\u0101lu izaicin\u0101jumu",audio_challenge:"Sa\u0146emt audio izaicin\u0101jumu",refresh_btn:"Sa\u0146emt jaunu izaicin\u0101jumu",instructions_visual:"Ievadiet tekstu:",instructions_audio:"Ierakstiet dzirdamo:",help_btn:"Pal\u012bdz\u012bba",play_again:"V\u0113lreiz atska\u0146ot ska\u0146u",
+cant_hear_this:"Lejupiel\u0101d\u0113t ska\u0146u MP3\u00a0form\u0101t\u0101",incorrect_try_again:"Nepareizi. M\u0113\u0123iniet v\u0113lreiz.",image_alt_text:"reCAPTCHA izaicin\u0101juma att\u0113ls",privacy_and_terms:"Konfidencialit\u0101te un noteikumi"},ml:{visual_challenge:"\u0d12\u0d30\u0d41 \u0d26\u0d43\u0d36\u0d4d\u0d2f \u0d1a\u0d32\u0d1e\u0d4d\u0d1a\u0d4d \u0d28\u0d47\u0d1f\u0d41\u0d15",audio_challenge:"\u0d12\u0d30\u0d41 \u0d13\u0d21\u0d3f\u0d2f\u0d4b \u0d1a\u0d32\u0d1e\u0d4d\u0d1a\u0d4d \u0d28\u0d47\u0d1f\u0d41\u0d15",
+refresh_btn:"\u0d12\u0d30\u0d41 \u0d2a\u0d41\u0d24\u0d3f\u0d2f \u0d1a\u0d32\u0d1e\u0d4d\u0d1a\u0d4d \u0d28\u0d47\u0d1f\u0d41\u0d15",instructions_visual:"",instructions_audio:"\u0d15\u0d47\u0d7e\u0d15\u0d4d\u0d15\u0d41\u0d28\u0d4d\u0d28\u0d24\u0d4d \u0d1f\u0d48\u0d2a\u0d4d\u0d2a\u0d4d \u0d1a\u0d46\u0d2f\u0d4d\u0d2f\u0d42:",help_btn:"\u0d38\u0d39\u0d3e\u0d2f\u0d02",play_again:"\u0d36\u0d2c\u0d4d\u200c\u0d26\u0d02 \u0d35\u0d40\u0d23\u0d4d\u0d1f\u0d41\u0d02 \u0d2a\u0d4d\u0d32\u0d47 \u0d1a\u0d46\u0d2f\u0d4d\u0d2f\u0d41\u0d15",
+cant_hear_this:"\u0d36\u0d2c\u0d4d\u200c\u0d26\u0d02 MP3 \u0d06\u0d2f\u0d3f \u0d21\u0d57\u0d7a\u0d32\u0d4b\u0d21\u0d4d \u0d1a\u0d46\u0d2f\u0d4d\u0d2f\u0d41\u0d15",incorrect_try_again:"\u0d24\u0d46\u0d31\u0d4d\u0d31\u0d3e\u0d23\u0d4d. \u0d35\u0d40\u0d23\u0d4d\u0d1f\u0d41\u0d02 \u0d36\u0d4d\u0d30\u0d2e\u0d3f\u0d15\u0d4d\u0d15\u0d41\u0d15.",image_alt_text:"reCAPTCHA \u0d1a\u0d32\u0d1e\u0d4d\u0d1a\u0d4d \u0d07\u0d2e\u0d47\u0d1c\u0d4d",privacy_and_terms:"\u0d38\u0d4d\u0d35\u0d15\u0d3e\u0d30\u0d4d\u0d2f\u0d24\u0d2f\u0d41\u0d02 \u0d28\u0d3f\u0d2c\u0d28\u0d4d\u0d27\u0d28\u0d15\u0d33\u0d41\u0d02"},
+mr:{visual_challenge:"\u0926\u0943\u0936\u094d\u200d\u092f\u092e\u093e\u0928 \u0906\u0935\u094d\u0939\u093e\u0928 \u092a\u094d\u0930\u093e\u092a\u094d\u0924 \u0915\u0930\u093e",audio_challenge:"\u0911\u0921\u0940\u0913 \u0906\u0935\u094d\u0939\u093e\u0928 \u092a\u094d\u0930\u093e\u092a\u094d\u0924 \u0915\u0930\u093e",refresh_btn:"\u090f\u0915 \u0928\u0935\u0940\u0928 \u0906\u0935\u094d\u0939\u093e\u0928 \u092a\u094d\u0930\u093e\u092a\u094d\u0924 \u0915\u0930\u093e",instructions_visual:"",instructions_audio:"\u0906\u092a\u0932\u094d\u092f\u093e\u0932\u093e \u091c\u0947 \u0910\u0915\u0942 \u092f\u0947\u0908\u0932 \u0924\u0947 \u091f\u093e\u0907\u092a \u0915\u0930\u093e:",
+help_btn:"\u092e\u0926\u0924",play_again:"\u0927\u094d\u200d\u0935\u0928\u0940 \u092a\u0941\u0928\u094d\u0939\u093e \u092a\u094d\u200d\u0932\u0947 \u0915\u0930\u093e",cant_hear_this:"MP3 \u0930\u0941\u092a\u093e\u0924 \u0927\u094d\u200d\u0935\u0928\u0940 \u0921\u093e\u0909\u0928\u0932\u094b\u0921 \u0915\u0930\u093e",incorrect_try_again:"\u0905\u092f\u094b\u0917\u094d\u200d\u092f. \u092a\u0941\u0928\u094d\u200d\u0939\u093e \u092a\u094d\u0930\u092f\u0924\u094d\u200d\u0928 \u0915\u0930\u093e.",image_alt_text:"reCAPTCHA \u0906\u0935\u094d\u200d\u0939\u093e\u0928 \u092a\u094d\u0930\u0924\u093f\u092e\u093e",
+privacy_and_terms:"\u0917\u094b\u092a\u0928\u0940\u092f\u0924\u093e \u0906\u0923\u093f \u0905\u091f\u0940"},ms:{visual_challenge:"Dapatkan cabaran visual",audio_challenge:"Dapatkan cabaran audio",refresh_btn:"Dapatkan cabaran baru",instructions_visual:"Taipkan teksnya:",instructions_audio:"Taip apa yang didengari:",help_btn:"Bantuan",play_again:"Mainkan bunyi sekali lagi",cant_hear_this:"Muat turun bunyi sebagai MP3",incorrect_try_again:"Tidak betul. Cuba lagi.",image_alt_text:"Imej cabaran reCAPTCHA",
+privacy_and_terms:"Privasi & Syarat"},nl:{visual_challenge:"Een visuele uitdaging proberen",audio_challenge:"Een audio-uitdaging proberen",refresh_btn:"Een nieuwe uitdaging proberen",instructions_visual:"Typ de tekst:",instructions_audio:"Typ wat u hoort:",help_btn:"Help",play_again:"Geluid opnieuw afspelen",cant_hear_this:"Geluid downloaden als MP3",incorrect_try_again:"Onjuist. Probeer het opnieuw.",image_alt_text:"reCAPTCHA-uitdagingsafbeelding",privacy_and_terms:"Privacy en voorwaarden"},no:{visual_challenge:"F\u00e5 en bildeutfordring",
+audio_challenge:"F\u00e5 en lydutfordring",refresh_btn:"F\u00e5 en ny utfordring",instructions_visual:"Skriv inn teksten:",instructions_audio:"Skriv inn det du h\u00f8rer:",help_btn:"Hjelp",play_again:"Spill av lyd p\u00e5 nytt",cant_hear_this:"Last ned lyd som MP3",incorrect_try_again:"Feil. Pr\u00f8v p\u00e5 nytt.",image_alt_text:"reCAPTCHA-utfordringsbilde",privacy_and_terms:"Personvern og vilk\u00e5r"},pl:{visual_challenge:"Poka\u017c podpowied\u017a wizualn\u0105",audio_challenge:"Odtw\u00f3rz podpowied\u017a d\u017awi\u0119kow\u0105",
+refresh_btn:"Nowa podpowied\u017a",instructions_visual:"Przepisz tekst:",instructions_audio:"Wpisz us\u0142yszane s\u0142owa:",help_btn:"Pomoc",play_again:"Odtw\u00f3rz d\u017awi\u0119k ponownie",cant_hear_this:"Pobierz d\u017awi\u0119k jako plik MP3",incorrect_try_again:"Nieprawid\u0142owo. Spr\u00f3buj ponownie.",image_alt_text:"Zadanie obrazkowe reCAPTCHA",privacy_and_terms:"Prywatno\u015b\u0107 i warunki"},pt:pa,"pt-BR":pa,"pt-PT":{visual_challenge:"Obter um desafio visual",audio_challenge:"Obter um desafio de \u00e1udio",
+refresh_btn:"Obter um novo desafio",instructions_visual:"Introduza o texto:",instructions_audio:"Escreva o que ouvir:",help_btn:"Ajuda",play_again:"Reproduzir som novamente",cant_hear_this:"Transferir som como MP3",incorrect_try_again:"Incorreto. Tente novamente.",image_alt_text:"Imagem de teste reCAPTCHA",privacy_and_terms:"Privacidade e Termos de Utiliza\u00e7\u00e3o"},ro:qa,ru:{visual_challenge:"\u0412\u0438\u0437\u0443\u0430\u043b\u044c\u043d\u0430\u044f \u043f\u0440\u043e\u0432\u0435\u0440\u043a\u0430",
+audio_challenge:"\u0417\u0432\u0443\u043a\u043e\u0432\u0430\u044f \u043f\u0440\u043e\u0432\u0435\u0440\u043a\u0430",refresh_btn:"\u041e\u0431\u043d\u043e\u0432\u0438\u0442\u044c",instructions_visual:"\u0412\u0432\u0435\u0434\u0438\u0442\u0435 \u0442\u0435\u043a\u0441\u0442:",instructions_audio:"\u0412\u0432\u0435\u0434\u0438\u0442\u0435 \u0442\u043e, \u0447\u0442\u043e \u0441\u043b\u044b\u0448\u0438\u0442\u0435:",help_btn:"\u0421\u043f\u0440\u0430\u0432\u043a\u0430",play_again:"\u041f\u0440\u043e\u0441\u043b\u0443\u0448\u0430\u0442\u044c \u0435\u0449\u0435 \u0440\u0430\u0437",
+cant_hear_this:"\u0417\u0430\u0433\u0440\u0443\u0437\u0438\u0442\u044c MP3-\u0444\u0430\u0439\u043b",incorrect_try_again:"\u041d\u0435\u043f\u0440\u0430\u0432\u0438\u043b\u044c\u043d\u043e. \u041f\u043e\u0432\u0442\u043e\u0440\u0438\u0442\u0435 \u043f\u043e\u043f\u044b\u0442\u043a\u0443.",image_alt_text:"\u041f\u0440\u043e\u0432\u0435\u0440\u043a\u0430 \u043f\u043e \u0441\u043b\u043e\u0432\u0443 reCAPTCHA",privacy_and_terms:"\u041f\u0440\u0430\u0432\u0438\u043b\u0430 \u0438 \u043f\u0440\u0438\u043d\u0446\u0438\u043f\u044b"},
+sk:{visual_challenge:"Zobrazi\u0165 vizu\u00e1lnu podobu",audio_challenge:"Prehra\u0165 zvukov\u00fa podobu",refresh_btn:"Zobrazi\u0165 nov\u00fd v\u00fdraz",instructions_visual:"Zadajte text:",instructions_audio:"Zadajte, \u010do po\u010dujete:",help_btn:"Pomocn\u00edk",play_again:"Znova prehra\u0165 zvuk",cant_hear_this:"Prevzia\u0165 zvuk v podobe s\u00faboru MP3",incorrect_try_again:"Nespr\u00e1vne. Sk\u00faste to znova.",image_alt_text:"Obr\u00e1zok zadania reCAPTCHA",privacy_and_terms:"Ochrana osobn\u00fdch \u00fadajov a Zmluvn\u00e9 podmienky"},
+sl:{visual_challenge:"Vizualni preskus",audio_challenge:"Zvo\u010dni preskus",refresh_btn:"Nov preskus",instructions_visual:"Vnesite besedilo:",instructions_audio:"Natipkajte, kaj sli\u0161ite:",help_btn:"Pomo\u010d",play_again:"Znova predvajaj zvok",cant_hear_this:"Prenesi zvok kot MP3",incorrect_try_again:"Napa\u010dno. Poskusite znova.",image_alt_text:"Slika izziva reCAPTCHA",privacy_and_terms:"Zasebnost in pogoji"},sr:{visual_challenge:"\u041f\u0440\u0438\u043c\u0438\u0442\u0435 \u0432\u0438\u0437\u0443\u0435\u043b\u043d\u0438 \u0443\u043f\u0438\u0442",
+audio_challenge:"\u041f\u0440\u0438\u043c\u0438\u0442\u0435 \u0430\u0443\u0434\u0438\u043e \u0443\u043f\u0438\u0442",refresh_btn:"\u041f\u0440\u0438\u043c\u0438\u0442\u0435 \u043d\u043e\u0432\u0438 \u0443\u043f\u0438\u0442",instructions_visual:"\u0423\u043d\u0435\u0441\u0438\u0442\u0435 \u0442\u0435\u043a\u0441\u0442:",instructions_audio:"\u041e\u0442\u043a\u0443\u0446\u0430\u0458\u0442\u0435 \u043e\u043d\u043e \u0448\u0442\u043e \u0447\u0443\u0458\u0435\u0442\u0435:",help_btn:"\u041f\u043e\u043c\u043e\u045b",
+play_again:"\u041f\u043e\u043d\u043e\u0432\u043e \u043f\u0443\u0441\u0442\u0438 \u0437\u0432\u0443\u043a",cant_hear_this:"\u041f\u0440\u0435\u0443\u0437\u043c\u0438 \u0437\u0432\u0443\u043a \u043a\u0430\u043e MP3 \u0441\u043d\u0438\u043c\u0430\u043a",incorrect_try_again:"\u041d\u0435\u0442\u0430\u0447\u043d\u043e. \u041f\u043e\u043a\u0443\u0448\u0430\u0458\u0442\u0435 \u043f\u043e\u043d\u043e\u0432\u043e.",image_alt_text:"\u0421\u043b\u0438\u043a\u0430 reCAPTCHA \u043f\u0440\u043e\u0432\u0435\u0440\u0435",
+privacy_and_terms:"\u041f\u0440\u0438\u0432\u0430\u0442\u043d\u043e\u0441\u0442 \u0438 \u0443\u0441\u043b\u043e\u0432\u0438"},sv:{visual_challenge:"H\u00e4mta captcha i bildformat",audio_challenge:"H\u00e4mta captcha i ljudformat",refresh_btn:"H\u00e4mta ny captcha",instructions_visual:"Skriv texten:",instructions_audio:"Skriv det du h\u00f6r:",help_btn:"Hj\u00e4lp",play_again:"Spela upp ljudet igen",cant_hear_this:"H\u00e4mta ljud som MP3",incorrect_try_again:"Fel. F\u00f6rs\u00f6k igen.",image_alt_text:"reCAPTCHA-bild",
+privacy_and_terms:"Sekretess och villkor"},sw:{visual_challenge:"Pata herufi za kusoma",audio_challenge:"Pata herufi za kusikiliza",refresh_btn:"Pata herufi mpya",instructions_visual:"",instructions_audio:"Charaza unachosikia:",help_btn:"Usaidizi",play_again:"Cheza sauti tena",cant_hear_this:"Pakua sauti kama MP3",incorrect_try_again:"Sio sahihi. Jaribu tena.",image_alt_text:"picha ya changamoto ya reCAPTCHA",privacy_and_terms:"Faragha & Masharti"},ta:{visual_challenge:"\u0baa\u0bbe\u0bb0\u0bcd\u0bb5\u0bc8 \u0b9a\u0bc7\u0bb2\u0b9e\u0bcd\u0b9a\u0bc8\u0baa\u0bcd \u0baa\u0bc6\u0bb1\u0bc1\u0b95",
+audio_challenge:"\u0b86\u0b9f\u0bbf\u0baf\u0bcb \u0b9a\u0bc7\u0bb2\u0b9e\u0bcd\u0b9a\u0bc8\u0baa\u0bcd \u0baa\u0bc6\u0bb1\u0bc1\u0b95",refresh_btn:"\u0baa\u0bc1\u0ba4\u0bbf\u0baf \u0b9a\u0bc7\u0bb2\u0b9e\u0bcd\u0b9a\u0bc8\u0baa\u0bcd \u0baa\u0bc6\u0bb1\u0bc1\u0b95",instructions_visual:"",instructions_audio:"\u0b95\u0bc7\u0b9f\u0bcd\u0baa\u0ba4\u0bc8 \u0b9f\u0bc8\u0baa\u0bcd \u0b9a\u0bc6\u0baf\u0bcd\u0b95:",help_btn:"\u0b89\u0ba4\u0bb5\u0bbf",play_again:"\u0b92\u0bb2\u0bbf\u0baf\u0bc8 \u0bae\u0bc0\u0ba3\u0bcd\u0b9f\u0bc1\u0bae\u0bcd \u0b87\u0baf\u0b95\u0bcd\u0b95\u0bc1",
+cant_hear_this:"\u0b92\u0bb2\u0bbf\u0baf\u0bc8 MP3 \u0b86\u0b95 \u0baa\u0ba4\u0bbf\u0bb5\u0bbf\u0bb1\u0b95\u0bcd\u0b95\u0bc1\u0b95",incorrect_try_again:"\u0ba4\u0bb5\u0bb1\u0bbe\u0ba9\u0ba4\u0bc1. \u0bae\u0bc0\u0ba3\u0bcd\u0b9f\u0bc1\u0bae\u0bcd \u0bae\u0bc1\u0baf\u0bb2\u0bb5\u0bc1\u0bae\u0bcd.",image_alt_text:"reCAPTCHA \u0b9a\u0bc7\u0bb2\u0b9e\u0bcd\u0b9a\u0bcd \u0baa\u0b9f\u0bae\u0bcd",privacy_and_terms:"\u0ba4\u0ba9\u0bbf\u0baf\u0bc1\u0bb0\u0bbf\u0bae\u0bc8 & \u0bb5\u0bbf\u0ba4\u0bbf\u0bae\u0bc1\u0bb1\u0bc8\u0b95\u0bb3\u0bcd"},
+te:{visual_challenge:"\u0c12\u0c15 \u0c26\u0c43\u0c36\u0c4d\u0c2f\u0c2e\u0c3e\u0c28 \u0c38\u0c35\u0c3e\u0c32\u0c41\u0c28\u0c41 \u0c38\u0c4d\u0c35\u0c40\u0c15\u0c30\u0c3f\u0c02\u0c1a\u0c02\u0c21\u0c3f",audio_challenge:"\u0c12\u0c15 \u0c06\u0c21\u0c3f\u0c2f\u0c4b \u0c38\u0c35\u0c3e\u0c32\u0c41\u0c28\u0c41 \u0c38\u0c4d\u0c35\u0c40\u0c15\u0c30\u0c3f\u0c02\u0c1a\u0c02\u0c21\u0c3f",refresh_btn:"\u0c15\u0c4d\u0c30\u0c4a\u0c24\u0c4d\u0c24 \u0c38\u0c35\u0c3e\u0c32\u0c41\u0c28\u0c41 \u0c38\u0c4d\u0c35\u0c40\u0c15\u0c30\u0c3f\u0c02\u0c1a\u0c02\u0c21\u0c3f",
+instructions_visual:"",instructions_audio:"\u0c2e\u0c40\u0c30\u0c41 \u0c35\u0c3f\u0c28\u0c4d\u0c28\u0c26\u0c3f \u0c1f\u0c48\u0c2a\u0c4d \u0c1a\u0c47\u0c2f\u0c02\u0c21\u0c3f:",help_btn:"\u0c38\u0c39\u0c3e\u0c2f\u0c02",play_again:"\u0c27\u0c4d\u0c35\u0c28\u0c3f\u0c28\u0c3f \u0c2e\u0c33\u0c4d\u0c32\u0c40 \u0c2a\u0c4d\u0c32\u0c47 \u0c1a\u0c47\u0c2f\u0c3f",cant_hear_this:"\u0c27\u0c4d\u0c35\u0c28\u0c3f\u0c28\u0c3f MP3 \u0c35\u0c32\u0c46 \u0c21\u0c4c\u0c28\u0c4d\u200c\u0c32\u0c4b\u0c21\u0c4d \u0c1a\u0c47\u0c2f\u0c3f",
+incorrect_try_again:"\u0c24\u0c2a\u0c4d\u0c2a\u0c41. \u0c2e\u0c33\u0c4d\u0c32\u0c40 \u0c2a\u0c4d\u0c30\u0c2f\u0c24\u0c4d\u0c28\u0c3f\u0c02\u0c1a\u0c02\u0c21\u0c3f.",image_alt_text:"reCAPTCHA \u0c38\u0c35\u0c3e\u0c32\u0c41 \u0c1a\u0c3f\u0c24\u0c4d\u0c30\u0c02",privacy_and_terms:"\u0c17\u0c4b\u0c2a\u0c4d\u0c2f\u0c24 & \u0c28\u0c3f\u0c2c\u0c02\u0c27\u0c28\u0c32\u0c41"},th:{visual_challenge:"\u0e23\u0e31\u0e1a\u0e04\u0e27\u0e32\u0e21\u0e17\u0e49\u0e32\u0e17\u0e32\u0e22\u0e14\u0e49\u0e32\u0e19\u0e20\u0e32\u0e1e",
+audio_challenge:"\u0e23\u0e31\u0e1a\u0e04\u0e27\u0e32\u0e21\u0e17\u0e49\u0e32\u0e17\u0e32\u0e22\u0e14\u0e49\u0e32\u0e19\u0e40\u0e2a\u0e35\u0e22\u0e07",refresh_btn:"\u0e23\u0e31\u0e1a\u0e04\u0e27\u0e32\u0e21\u0e17\u0e49\u0e32\u0e17\u0e32\u0e22\u0e43\u0e2b\u0e21\u0e48",instructions_visual:"\u0e1e\u0e34\u0e21\u0e1e\u0e4c\u0e02\u0e49\u0e2d\u0e04\u0e27\u0e32\u0e21\u0e19\u0e35\u0e49:",instructions_audio:"\u0e1e\u0e34\u0e21\u0e1e\u0e4c\u0e2a\u0e34\u0e48\u0e07\u0e17\u0e35\u0e48\u0e04\u0e38\u0e13\u0e44\u0e14\u0e49\u0e22\u0e34\u0e19:",
+help_btn:"\u0e04\u0e27\u0e32\u0e21\u0e0a\u0e48\u0e27\u0e22\u0e40\u0e2b\u0e25\u0e37\u0e2d",play_again:"\u0e40\u0e25\u0e48\u0e19\u0e40\u0e2a\u0e35\u0e22\u0e07\u0e2d\u0e35\u0e01\u0e04\u0e23\u0e31\u0e49\u0e07",cant_hear_this:"\u0e14\u0e32\u0e27\u0e42\u0e2b\u0e25\u0e14\u0e40\u0e2a\u0e35\u0e22\u0e07\u0e40\u0e1b\u0e47\u0e19 MP3",incorrect_try_again:"\u0e44\u0e21\u0e48\u0e16\u0e39\u0e01\u0e15\u0e49\u0e2d\u0e07 \u0e25\u0e2d\u0e07\u0e2d\u0e35\u0e01\u0e04\u0e23\u0e31\u0e49\u0e07",image_alt_text:"\u0e23\u0e2b\u0e31\u0e2a\u0e20\u0e32\u0e1e reCAPTCHA",
+privacy_and_terms:"\u0e19\u0e42\u0e22\u0e1a\u0e32\u0e22\u0e2a\u0e48\u0e27\u0e19\u0e1a\u0e38\u0e04\u0e04\u0e25\u0e41\u0e25\u0e30\u0e02\u0e49\u0e2d\u0e01\u0e33\u0e2b\u0e19\u0e14"},tr:{visual_challenge:"G\u00f6rsel sorgu al",audio_challenge:"Sesli sorgu al",refresh_btn:"Yeniden y\u00fckle",instructions_visual:"Metni yaz\u0131n:",instructions_audio:"Duydu\u011funuzu yaz\u0131n:",help_btn:"Yard\u0131m",play_again:"Sesi tekrar \u00e7al",cant_hear_this:"Sesi MP3 olarak indir",incorrect_try_again:"Yanl\u0131\u015f. Tekrar deneyin.",
+image_alt_text:"reCAPTCHA sorusu resmi",privacy_and_terms:"Gizlilik ve \u015eartlar"},uk:{visual_challenge:"\u041e\u0442\u0440\u0438\u043c\u0430\u0442\u0438 \u0432\u0456\u0437\u0443\u0430\u043b\u044c\u043d\u0438\u0439 \u0442\u0435\u043a\u0441\u0442",audio_challenge:"\u041e\u0442\u0440\u0438\u043c\u0430\u0442\u0438 \u0430\u0443\u0434\u0456\u043e\u0437\u0430\u043f\u0438\u0441",refresh_btn:"\u041e\u043d\u043e\u0432\u0438\u0442\u0438 \u0442\u0435\u043a\u0441\u0442",instructions_visual:"\u0412\u0432\u0435\u0434\u0456\u0442\u044c \u0442\u0435\u043a\u0441\u0442:",
+instructions_audio:"\u0412\u0432\u0435\u0434\u0456\u0442\u044c \u043f\u043e\u0447\u0443\u0442\u0435:",help_btn:"\u0414\u043e\u0432\u0456\u0434\u043a\u0430",play_again:"\u0412\u0456\u0434\u0442\u0432\u043e\u0440\u0438\u0442\u0438 \u0437\u0430\u043f\u0438\u0441 \u0449\u0435 \u0440\u0430\u0437",cant_hear_this:"\u0417\u0430\u0432\u0430\u043d\u0442\u0430\u0436\u0438\u0442\u0438 \u0437\u0430\u043f\u0438\u0441 \u044f\u043a MP3",incorrect_try_again:"\u041d\u0435\u043f\u0440\u0430\u0432\u0438\u043b\u044c\u043d\u043e. \u0421\u043f\u0440\u043e\u0431\u0443\u0439\u0442\u0435 \u0449\u0435 \u0440\u0430\u0437.",
+image_alt_text:"\u0417\u043e\u0431\u0440\u0430\u0436\u0435\u043d\u043d\u044f \u0437\u0430\u0432\u0434\u0430\u043d\u043d\u044f reCAPTCHA",privacy_and_terms:"\u041a\u043e\u043d\u0444\u0456\u0434\u0435\u043d\u0446\u0456\u0439\u043d\u0456\u0441\u0442\u044c \u0456 \u0443\u043c\u043e\u0432\u0438"},ur:{visual_challenge:"\u0627\u06cc\u06a9 \u0645\u0631\u0626\u06cc \u0686\u06cc\u0644\u0646\u062c \u062d\u0627\u0635\u0644 \u06a9\u0631\u06cc\u06ba",audio_challenge:"\u0627\u06cc\u06a9 \u0622\u0688\u06cc\u0648 \u0686\u06cc\u0644\u0646\u062c \u062d\u0627\u0635\u0644 \u06a9\u0631\u06cc\u06ba",
+refresh_btn:"\u0627\u06cc\u06a9 \u0646\u06cc\u0627 \u0686\u06cc\u0644\u0646\u062c \u062d\u0627\u0635\u0644 \u06a9\u0631\u06cc\u06ba",instructions_visual:"",instructions_audio:"\u062c\u0648 \u0633\u0646\u0627\u0626\u06cc \u062f\u06cc\u062a\u0627 \u06c1\u06d2 \u0648\u06c1 \u0679\u0627\u0626\u067e \u06a9\u0631\u06cc\u06ba:",help_btn:"\u0645\u062f\u062f",play_again:"\u0622\u0648\u0627\u0632 \u062f\u0648\u0628\u0627\u0631\u06c1 \u0686\u0644\u0627\u0626\u06cc\u06ba",cant_hear_this:"\u0622\u0648\u0627\u0632 \u06a9\u0648 MP3 \u06a9\u06d2 \u0628\u0637\u0648\u0631 \u0688\u0627\u0624\u0646 \u0644\u0648\u0688 \u06a9\u0631\u06cc\u06ba",
+incorrect_try_again:"\u063a\u0644\u0637\u06d4 \u062f\u0648\u0628\u0627\u0631\u06c1 \u06a9\u0648\u0634\u0634 \u06a9\u0631\u06cc\u06ba\u06d4",image_alt_text:"reCAPTCHA \u0686\u06cc\u0644\u0646\u062c \u0648\u0627\u0644\u06cc \u0634\u0628\u06cc\u06c1",privacy_and_terms:"\u0631\u0627\u0632\u062f\u0627\u0631\u06cc \u0648 \u0634\u0631\u0627\u0626\u0637"},vi:{visual_challenge:"Nh\u1eadn th\u1eed th\u00e1ch h\u00ecnh \u1ea3nh",audio_challenge:"Nh\u1eadn th\u1eed th\u00e1ch \u00e2m thanh",refresh_btn:"Nh\u1eadn th\u1eed th\u00e1ch m\u1edbi",
+instructions_visual:"Nh\u1eadp v\u0103n b\u1ea3n:",instructions_audio:"Nh\u1eadp n\u1ed9i dung b\u1ea1n nghe th\u1ea5y:",help_btn:"Tr\u1ee3 gi\u00fap",play_again:"Ph\u00e1t l\u1ea1i \u00e2m thanh",cant_hear_this:"T\u1ea3i \u00e2m thanh xu\u1ed1ng d\u01b0\u1edbi d\u1ea1ng MP3",incorrect_try_again:"Kh\u00f4ng ch\u00ednh x\u00e1c. H\u00e3y th\u1eed l\u1ea1i.",image_alt_text:"H\u00ecnh x\u00e1c th\u1ef1c reCAPTCHA",privacy_and_terms:"B\u1ea3o m\u1eadt v\u00e0 \u0111i\u1ec1u kho\u1ea3n"},"zh-CN":ra,"zh-HK":{visual_challenge:"\u56de\u7b54\u5716\u50cf\u9a57\u8b49\u554f\u984c",
+audio_challenge:"\u53d6\u5f97\u8a9e\u97f3\u9a57\u8b49\u554f\u984c",refresh_btn:"\u63db\u4e00\u500b\u9a57\u8b49\u554f\u984c",instructions_visual:"\u8f38\u5165\u6587\u5b57\uff1a",instructions_audio:"\u9375\u5165\u60a8\u6240\u807d\u5230\u7684\uff1a",help_btn:"\u8aaa\u660e",play_again:"\u518d\u6b21\u64ad\u653e\u8072\u97f3",cant_hear_this:"\u5c07\u8072\u97f3\u4e0b\u8f09\u70ba MP3",incorrect_try_again:"\u4e0d\u6b63\u78ba\uff0c\u518d\u8a66\u4e00\u6b21\u3002",image_alt_text:"reCAPTCHA \u9a57\u8b49\u6587\u5b57\u5716\u7247",
+privacy_and_terms:"\u79c1\u96b1\u6b0a\u8207\u689d\u6b3e"},"zh-TW":{visual_challenge:"\u53d6\u5f97\u5716\u7247\u9a57\u8b49\u554f\u984c",audio_challenge:"\u53d6\u5f97\u8a9e\u97f3\u9a57\u8b49\u554f\u984c",refresh_btn:"\u53d6\u5f97\u65b0\u7684\u9a57\u8b49\u554f\u984c",instructions_visual:"\u8acb\u8f38\u5165\u5716\u7247\u4e2d\u7684\u6587\u5b57\uff1a",instructions_audio:"\u8acb\u8f38\u5165\u8a9e\u97f3\u5167\u5bb9\uff1a",help_btn:"\u8aaa\u660e",play_again:"\u518d\u6b21\u64ad\u653e",cant_hear_this:"\u4ee5 MP3 \u683c\u5f0f\u4e0b\u8f09\u8072\u97f3",
+incorrect_try_again:"\u9a57\u8b49\u78bc\u6709\u8aa4\uff0c\u8acb\u518d\u8a66\u4e00\u6b21\u3002",image_alt_text:"reCAPTCHA \u9a57\u8b49\u6587\u5b57\u5716\u7247",privacy_and_terms:"\u96b1\u79c1\u6b0a\u8207\u689d\u6b3e"},zu:{visual_challenge:"Thola inselelo ebonakalayo",audio_challenge:"Thola inselelo yokulalelwayo",refresh_btn:"Thola inselelo entsha",instructions_visual:"",instructions_audio:"Bhala okuzwayo:",help_btn:"Usizo",play_again:"Phinda udlale okulalelwayo futhi",cant_hear_this:"Layisha umsindo njenge-MP3",
+incorrect_try_again:"Akulungile. Zama futhi.",image_alt_text:"umfanekiso oyinselelo we-reCAPTCHA",privacy_and_terms:"Okwangasese kanye nemigomo"},tl:la,he:oa,"in":na,mo:qa,zh:ra};var x=function(a){if(Error.captureStackTrace)Error.captureStackTrace(this,x);else{var b=Error().stack;b&&(this.stack=b)}a&&(this.message=String(a))};u(x,Error);x.prototype.name="CustomError";var ta;var ua=function(a,b){for(var c=a.split("%s"),d="",e=Array.prototype.slice.call(arguments,1);e.length&&1<c.length;)d+=c.shift()+e.shift();return d+c.join("%s")},va=String.prototype.trim?function(a){return a.trim()}:function(a){return a.replace(/^[\s\xa0]+|[\s\xa0]+$/g,"")},Da=function(a){if(!wa.test(a))return a;-1!=a.indexOf("&")&&(a=a.replace(xa,"&amp;"));-1!=a.indexOf("<")&&(a=a.replace(ya,"&lt;"));-1!=a.indexOf(">")&&(a=a.replace(za,"&gt;"));-1!=a.indexOf('"')&&(a=a.replace(Aa,"&quot;"));-1!=a.indexOf("'")&&
+(a=a.replace(Ba,"&#39;"));-1!=a.indexOf("\x00")&&(a=a.replace(Ca,"&#0;"));return a},xa=/&/g,ya=/</g,za=/>/g,Aa=/"/g,Ba=/'/g,Ca=/\x00/g,wa=/[\x00&<>"']/,Ea=function(a,b){return a<b?-1:a>b?1:0},Fa=function(a){return String(a).replace(/\-([a-z])/g,function(a,c){return c.toUpperCase()})},Ga=function(a){var b=q(void 0)?"undefined".replace(/([-()\[\]{}+?*.$\^|,:#<!\\])/g,"\\$1").replace(/\x08/g,"\\x08"):"\\s";return a.replace(new RegExp("(^"+(b?"|["+b+"]+":"")+")([a-z])","g"),function(a,b,e){return b+e.toUpperCase()})};var Ha=function(a,b){b.unshift(a);x.call(this,ua.apply(null,b));b.shift()};u(Ha,x);Ha.prototype.name="AssertionError";
+var Ia=function(a,b,c,d){var e="Assertion failed";if(c)var e=e+(": "+c),g=d;else a&&(e+=": "+a,g=b);throw new Ha(""+e,g||[]);},y=function(a,b,c){a||Ia("",null,b,Array.prototype.slice.call(arguments,2))},Ja=function(a,b){throw new Ha("Failure"+(a?": "+a:""),Array.prototype.slice.call(arguments,1));},Ka=function(a,b,c){q(a)||Ia("Expected string but got %s: %s.",[n(a),a],b,Array.prototype.slice.call(arguments,2));return a},La=function(a,b,c){r(a)||Ia("Expected function but got %s: %s.",[n(a),a],b,Array.prototype.slice.call(arguments,
+2))};var z=Array.prototype,Ma=z.indexOf?function(a,b,c){y(null!=a.length);return z.indexOf.call(a,b,c)}:function(a,b,c){c=null==c?0:0>c?Math.max(0,a.length+c):c;if(q(a))return q(b)&&1==b.length?a.indexOf(b,c):-1;for(;c<a.length;c++)if(c in a&&a[c]===b)return c;return-1},Na=z.forEach?function(a,b,c){y(null!=a.length);z.forEach.call(a,b,c)}:function(a,b,c){for(var d=a.length,e=q(a)?a.split(""):a,g=0;g<d;g++)g in e&&b.call(c,e[g],g,a)},Oa=z.map?function(a,b,c){y(null!=a.length);return z.map.call(a,b,c)}:
+function(a,b,c){for(var d=a.length,e=Array(d),g=q(a)?a.split(""):a,f=0;f<d;f++)f in g&&(e[f]=b.call(c,g[f],f,a));return e},Pa=z.some?function(a,b,c){y(null!=a.length);return z.some.call(a,b,c)}:function(a,b,c){for(var d=a.length,e=q(a)?a.split(""):a,g=0;g<d;g++)if(g in e&&b.call(c,e[g],g,a))return!0;return!1},Qa=function(a,b){var c=Ma(a,b),d;if(d=0<=c)y(null!=a.length),z.splice.call(a,c,1);return d},Ra=function(a){var b=a.length;if(0<b){for(var c=Array(b),d=0;d<b;d++)c[d]=a[d];return c}return[]},
+Sa=function(a,b,c){y(null!=a.length);return 2>=arguments.length?z.slice.call(a,b):z.slice.call(a,b,c)};var Ta=function(a,b){for(var c in a)b.call(void 0,a[c],c,a)},Ua=function(a){var b=[],c=0,d;for(d in a)b[c++]=d;return b},Va=function(a){for(var b in a)return!1;return!0},Xa=function(){var a=Wa()?k.google_ad:null,b={},c;for(c in a)b[c]=a[c];return b},Ya="constructor hasOwnProperty isPrototypeOf propertyIsEnumerable toLocaleString toString valueOf".split(" "),Za=function(a,b){for(var c,d,e=1;e<arguments.length;e++){d=arguments[e];for(c in d)a[c]=d[c];for(var g=0;g<Ya.length;g++)c=Ya[g],Object.prototype.hasOwnProperty.call(d,
+c)&&(a[c]=d[c])}},$a=function(a){var b=arguments.length;if(1==b&&p(arguments[0]))return $a.apply(null,arguments[0]);for(var c={},d=0;d<b;d++)c[arguments[d]]=!0;return c};var A;t:{var ab=k.navigator;if(ab){var bb=ab.userAgent;if(bb){A=bb;break t}}A=""}var B=function(a){return-1!=A.indexOf(a)};var cb=B("Opera")||B("OPR"),C=B("Trident")||B("MSIE"),D=B("Gecko")&&-1==A.toLowerCase().indexOf("webkit")&&!(B("Trident")||B("MSIE")),E=-1!=A.toLowerCase().indexOf("webkit"),db=function(){var a=k.document;return a?a.documentMode:void 0},eb=function(){var a="",b;if(cb&&k.opera)return a=k.opera.version,r(a)?a():a;D?b=/rv\:([^\);]+)(\)|;)/:C?b=/\b(?:MSIE|rv)[: ]([^\);]+)(\)|;)/:E&&(b=/WebKit\/(\S+)/);b&&(a=(a=b.exec(A))?a[1]:"");return C&&(b=db(),b>parseFloat(a))?String(b):a}(),fb={},F=function(a){var b;
+if(!(b=fb[a])){b=0;for(var c=va(String(eb)).split("."),d=va(String(a)).split("."),e=Math.max(c.length,d.length),g=0;0==b&&g<e;g++){var f=c[g]||"",m=d[g]||"",$=RegExp("(\\d*)(\\D*)","g"),K=RegExp("(\\d*)(\\D*)","g");do{var G=$.exec(f)||["","",""],aa=K.exec(m)||["","",""];if(0==G[0].length&&0==aa[0].length)break;b=Ea(0==G[1].length?0:parseInt(G[1],10),0==aa[1].length?0:parseInt(aa[1],10))||Ea(0==G[2].length,0==aa[2].length)||Ea(G[2],aa[2])}while(0==b)}b=fb[a]=0<=b}return b},gb=k.document,hb=gb&&C?db()||
+("CSS1Compat"==gb.compatMode?parseInt(eb,10):5):void 0;var ib=function(a){if(8192>a.length)return String.fromCharCode.apply(null,a);for(var b="",c=0;c<a.length;c+=8192)var d=Sa(a,c,c+8192),b=b+String.fromCharCode.apply(null,d);return b},jb=function(a){return Oa(a,function(a){a=a.toString(16);return 1<a.length?a:"0"+a}).join("")};var kb=null,lb=null,mb=function(a){if(!kb){kb={};lb={};for(var b=0;65>b;b++)kb[b]="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=".charAt(b),lb[kb[b]]=b,62<=b&&(lb["ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_.".charAt(b)]=b)}for(var b=lb,c=[],d=0;d<a.length;){var e=b[a.charAt(d++)],g=d<a.length?b[a.charAt(d)]:0;++d;var f=d<a.length?b[a.charAt(d)]:64;++d;var m=d<a.length?b[a.charAt(d)]:64;++d;if(null==e||null==g||null==f||null==m)throw Error();c.push(e<<2|g>>
+4);64!=f&&(c.push(g<<4&240|f>>2),64!=m&&c.push(f<<6&192|m))}return c};var H=function(){this.disposed_=this.disposed_;this.onDisposeCallbacks_=this.onDisposeCallbacks_};H.prototype.disposed_=!1;H.prototype.dispose=function(){this.disposed_||(this.disposed_=!0,this.disposeInternal())};var nb=function(a,b){a.onDisposeCallbacks_||(a.onDisposeCallbacks_=[]);a.onDisposeCallbacks_.push(l(void 0)?s(b,void 0):b)};H.prototype.disposeInternal=function(){if(this.onDisposeCallbacks_)for(;this.onDisposeCallbacks_.length;)this.onDisposeCallbacks_.shift()()};
+var ob=function(a){a&&"function"==typeof a.dispose&&a.dispose()};var pb=!C||C&&9<=hb;!D&&!C||C&&C&&9<=hb||D&&F("1.9.1");C&&F("9");var sb=function(a){return a?new qb(rb(a)):ta||(ta=new qb)},tb=function(a,b){return q(b)?a.getElementById(b):b},vb=function(a,b){Ta(b,function(b,d){"style"==d?a.style.cssText=b:"class"==d?a.className=b:"for"==d?a.htmlFor=b:d in ub?a.setAttribute(ub[d],b):0==d.lastIndexOf("aria-",0)||0==d.lastIndexOf("data-",0)?a.setAttribute(d,b):a[d]=b})},ub={cellpadding:"cellPadding",cellspacing:"cellSpacing",colspan:"colSpan",frameborder:"frameBorder",height:"height",maxlength:"maxLength",role:"role",rowspan:"rowSpan",
+type:"type",usemap:"useMap",valign:"vAlign",width:"width"},xb=function(a,b,c){return wb(document,arguments)},wb=function(a,b){var c=b[0],d=b[1];if(!pb&&d&&(d.name||d.type)){c=["<",c];d.name&&c.push(' name="',Da(d.name),'"');if(d.type){c.push(' type="',Da(d.type),'"');var e={};Za(e,d);delete e.type;d=e}c.push(">");c=c.join("")}c=a.createElement(c);d&&(q(d)?c.className=d:p(d)?c.className=d.join(" "):vb(c,d));2<b.length&&yb(a,c,b);return c},yb=function(a,b,c){function d(c){c&&b.appendChild(q(c)?a.createTextNode(c):
+c)}for(var e=2;e<c.length;e++){var g=c[e];!ca(g)||da(g)&&0<g.nodeType?d(g):Na(zb(g)?Ra(g):g,d)}},Ab=function(a){for(var b;b=a.firstChild;)a.removeChild(b)},Bb=function(a){a&&a.parentNode&&a.parentNode.removeChild(a)},rb=function(a){y(a,"Node cannot be null or undefined.");return 9==a.nodeType?a:a.ownerDocument||a.document},zb=function(a){if(a&&"number"==typeof a.length){if(da(a))return"function"==typeof a.item||"string"==typeof a.item;if(r(a))return"function"==typeof a.item}return!1},qb=function(a){this.document_=
+a||k.document||document};h=qb.prototype;h.getDomHelper=sb;h.getElement=function(a){return tb(this.document_,a)};h.$=qb.prototype.getElement;h.createDom=function(a,b,c){return wb(this.document_,arguments)};h.createElement=function(a){return this.document_.createElement(a)};h.createTextNode=function(a){return this.document_.createTextNode(String(a))};h.appendChild=function(a,b){a.appendChild(b)};var Cb=function(a){k.setTimeout(function(){throw a;},0)},Db,Eb=function(){var a=k.MessageChannel;"undefined"===typeof a&&"undefined"!==typeof window&&window.postMessage&&window.addEventListener&&(a=function(){var a=document.createElement("iframe");a.style.display="none";a.src="";document.documentElement.appendChild(a);var b=a.contentWindow,a=b.document;a.open();a.write("");a.close();var c="callImmediate"+Math.random(),d="file:"==b.location.protocol?"*":b.location.protocol+"//"+b.location.host,a=s(function(a){if(("*"==
+d||a.origin==d)&&a.data==c)this.port1.onmessage()},this);b.addEventListener("message",a,!1);this.port1={};this.port2={postMessage:function(){b.postMessage(c,d)}}});if("undefined"!==typeof a&&!B("Trident")&&!B("MSIE")){var b=new a,c={},d=c;b.port1.onmessage=function(){if(l(c.next)){c=c.next;var a=c.cb;c.cb=null;a()}};return function(a){d.next={cb:a};d=d.next;b.port2.postMessage(0)}}return"undefined"!==typeof document&&"onreadystatechange"in document.createElement("script")?function(a){var b=document.createElement("script");
+b.onreadystatechange=function(){b.onreadystatechange=null;b.parentNode.removeChild(b);b=null;a();a=null};document.documentElement.appendChild(b)}:function(a){k.setTimeout(a,0)}};var Kb=function(a,b){Fb||Gb();Hb||(Fb(),Hb=!0);Ib.push(new Jb(a,b))},Fb,Gb=function(){if(k.Promise&&k.Promise.resolve){var a=k.Promise.resolve();Fb=function(){a.then(Lb)}}else Fb=function(){var a=Lb;!r(k.setImmediate)||k.Window&&k.Window.prototype.setImmediate==k.setImmediate?(Db||(Db=Eb()),Db(a)):k.setImmediate(a)}},Hb=!1,Ib=[],Lb=function(){for(;Ib.length;){var a=Ib;Ib=[];for(var b=0;b<a.length;b++){var c=a[b];try{c.fn.call(c.scope)}catch(d){Cb(d)}}}Hb=!1},Jb=function(a,b){this.fn=a;this.scope=
+b};var Mb=function(a){a.prototype.then=a.prototype.then;a.prototype.$goog_Thenable=!0},Nb=function(a){if(!a)return!1;try{return!!a.$goog_Thenable}catch(b){return!1}};var L=function(a,b){this.state_=0;this.result_=void 0;this.callbackEntries_=this.parent_=null;this.hadUnhandledRejection_=this.executing_=!1;try{var c=this;a.call(b,function(a){I(c,2,a)},function(a){if(!(a instanceof J))try{if(a instanceof Error)throw a;throw Error("Promise rejected.");}catch(b){}I(c,3,a)})}catch(d){I(this,3,d)}};
+L.prototype.then=function(a,b,c){null!=a&&La(a,"opt_onFulfilled should be a function.");null!=b&&La(b,"opt_onRejected should be a function. Did you pass opt_context as the second argument instead of the third?");return Ob(this,r(a)?a:null,r(b)?b:null,c)};Mb(L);L.prototype.cancel=function(a){0==this.state_&&Kb(function(){var b=new J(a);Pb(this,b)},this)};
+var Pb=function(a,b){if(0==a.state_)if(a.parent_){var c=a.parent_;if(c.callbackEntries_){for(var d=0,e=-1,g=0,f;f=c.callbackEntries_[g];g++)if(f=f.child)if(d++,f==a&&(e=g),0<=e&&1<d)break;0<=e&&(0==c.state_&&1==d?Pb(c,b):(d=c.callbackEntries_.splice(e,1)[0],d.child&&Qb(c),d.onRejected(b)))}}else I(a,3,b)},Sb=function(a,b){a.callbackEntries_&&a.callbackEntries_.length||2!=a.state_&&3!=a.state_||Rb(a);a.callbackEntries_||(a.callbackEntries_=[]);a.callbackEntries_.push(b)},Ob=function(a,b,c,d){var e=
+{child:null,onFulfilled:null,onRejected:null};e.child=new L(function(a,f){e.onFulfilled=b?function(c){try{var e=b.call(d,c);a(e)}catch(K){f(K)}}:a;e.onRejected=c?function(b){try{var e=c.call(d,b);!l(e)&&b instanceof J?f(b):a(e)}catch(K){f(K)}}:f});e.child.parent_=a;Sb(a,e);return e.child};L.prototype.unblockAndFulfill_=function(a){y(1==this.state_);this.state_=0;I(this,2,a)};L.prototype.unblockAndReject_=function(a){y(1==this.state_);this.state_=0;I(this,3,a)};
+var I=function(a,b,c){if(0==a.state_){if(a==c)b=3,c=new TypeError("Promise cannot resolve to itself");else{if(Nb(c)){a.state_=1;c.then(a.unblockAndFulfill_,a.unblockAndReject_,a);return}if(da(c))try{var d=c.then;if(r(d)){Tb(a,c,d);return}}catch(e){b=3,c=e}}a.result_=c;a.state_=b;Rb(a);3!=b||c instanceof J||Ub(a,c)}},Tb=function(a,b,c){a.state_=1;var d=!1,e=function(b){d||(d=!0,a.unblockAndFulfill_(b))},g=function(b){d||(d=!0,a.unblockAndReject_(b))};try{c.call(b,e,g)}catch(f){g(f)}},Rb=function(a){a.executing_||
+(a.executing_=!0,Kb(a.executeCallbacks_,a))};L.prototype.executeCallbacks_=function(){for(;this.callbackEntries_&&this.callbackEntries_.length;){var a=this.callbackEntries_;this.callbackEntries_=[];for(var b=0;b<a.length;b++){var c=a[b],d=this.result_;if(2==this.state_)c.onFulfilled(d);else c.child&&Qb(this),c.onRejected(d)}}this.executing_=!1};
+var Qb=function(a){for(;a&&a.hadUnhandledRejection_;a=a.parent_)a.hadUnhandledRejection_=!1},Ub=function(a,b){a.hadUnhandledRejection_=!0;Kb(function(){a.hadUnhandledRejection_&&Vb.call(null,b)})},Vb=Cb,J=function(a){x.call(this,a)};u(J,x);J.prototype.name="cancel";/*
+ Portions of this code are from MochiKit, received by
+ The Closure Authors under the MIT license. All other code is Copyright
+ 2005-2009 The Closure Authors. All Rights Reserved.
+*/
+var M=function(a,b){this.sequence_=[];this.onCancelFunction_=a;this.defaultScope_=b||null;this.hadError_=this.fired_=!1;this.result_=void 0;this.silentlyCanceled_=this.blocking_=this.blocked_=!1;this.unhandledErrorId_=0;this.parent_=null;this.branches_=0};
+M.prototype.cancel=function(a){if(this.fired_)this.result_ instanceof M&&this.result_.cancel();else{if(this.parent_){var b=this.parent_;delete this.parent_;a?b.cancel(a):(b.branches_--,0>=b.branches_&&b.cancel())}this.onCancelFunction_?this.onCancelFunction_.call(this.defaultScope_,this):this.silentlyCanceled_=!0;this.fired_||Wb(this,new Xb)}};M.prototype.continue_=function(a,b){this.blocked_=!1;Yb(this,a,b)};
+var Yb=function(a,b,c){a.fired_=!0;a.result_=c;a.hadError_=!b;Zb(a)},ac=function(a){if(a.fired_){if(!a.silentlyCanceled_)throw new $b;a.silentlyCanceled_=!1}};M.prototype.callback=function(a){ac(this);bc(a);Yb(this,!0,a)};var Wb=function(a,b){ac(a);bc(b);Yb(a,!1,b)},bc=function(a){y(!(a instanceof M),"An execution sequence may not be initiated with a blocking Deferred.")},cc=function(a,b,c,d){y(!a.blocking_,"Blocking Deferreds can not be re-used");a.sequence_.push([b,c,d]);a.fired_&&Zb(a)};
+M.prototype.then=function(a,b,c){var d,e,g=new L(function(a,b){d=a;e=b});cc(this,d,function(a){a instanceof Xb?g.cancel():e(a)});return g.then(a,b,c)};Mb(M);
+var dc=function(a){return Pa(a.sequence_,function(a){return r(a[1])})},Zb=function(a){if(a.unhandledErrorId_&&a.fired_&&dc(a)){var b=a.unhandledErrorId_,c=ec[b];c&&(k.clearTimeout(c.id_),delete ec[b]);a.unhandledErrorId_=0}a.parent_&&(a.parent_.branches_--,delete a.parent_);for(var b=a.result_,d=c=!1;a.sequence_.length&&!a.blocked_;){var e=a.sequence_.shift(),g=e[0],f=e[1],e=e[2];if(g=a.hadError_?f:g)try{var m=g.call(e||a.defaultScope_,b);l(m)&&(a.hadError_=a.hadError_&&(m==b||m instanceof Error),
+a.result_=b=m);Nb(b)&&(d=!0,a.blocked_=!0)}catch($){b=$,a.hadError_=!0,dc(a)||(c=!0)}}a.result_=b;d&&(m=s(a.continue_,a,!0),d=s(a.continue_,a,!1),b instanceof M?(cc(b,m,d),b.blocking_=!0):b.then(m,d));c&&(b=new fc(b),ec[b.id_]=b,a.unhandledErrorId_=b.id_)},$b=function(){x.call(this)};u($b,x);$b.prototype.message="Deferred has already fired";$b.prototype.name="AlreadyCalledError";var Xb=function(){x.call(this)};u(Xb,x);Xb.prototype.message="Deferred was canceled";Xb.prototype.name="CanceledError";
+var fc=function(a){this.id_=k.setTimeout(s(this.throwError,this),0);this.error_=a};fc.prototype.throwError=function(){y(ec[this.id_],"Cannot throw an error that is not scheduled.");delete ec[this.id_];throw this.error_;};var ec={};var kc=function(a){var b={},c=b.document||document,d=document.createElement("SCRIPT"),e={script_:d,timeout_:void 0},g=new M(gc,e),f=null,m=null!=b.timeout?b.timeout:5E3;0<m&&(f=window.setTimeout(function(){hc(d,!0);Wb(g,new ic(1,"Timeout reached for loading script "+a))},m),e.timeout_=f);d.onload=d.onreadystatechange=function(){d.readyState&&"loaded"!=d.readyState&&"complete"!=d.readyState||(hc(d,b.cleanupWhenDone||!1,f),g.callback(null))};d.onerror=function(){hc(d,!0,f);Wb(g,new ic(0,"Error while loading script "+
+a))};vb(d,{type:"text/javascript",charset:"UTF-8",src:a});jc(c).appendChild(d);return g},jc=function(a){var b=a.getElementsByTagName("HEAD");return b&&0!=b.length?b[0]:a.documentElement},gc=function(){if(this&&this.script_){var a=this.script_;a&&"SCRIPT"==a.tagName&&hc(a,!0,this.timeout_)}},hc=function(a,b,c){null!=c&&k.clearTimeout(c);a.onload=ba;a.onerror=ba;a.onreadystatechange=ba;b&&window.setTimeout(function(){Bb(a)},0)},ic=function(a,b){var c="Jsloader error (code #"+a+")";b&&(c+=": "+b);x.call(this,
+c);this.code=a};u(ic,x);var lc=function(a){lc[" "](a);return a};lc[" "]=ba;var mc=!C||C&&9<=hb,nc=C&&!F("9");!E||F("528");D&&F("1.9b")||C&&F("8")||cb&&F("9.5")||E&&F("528");D&&!F("8")||C&&F("9");var N=function(a,b){this.type=a;this.currentTarget=this.target=b;this.defaultPrevented=this.propagationStopped_=!1;this.returnValue_=!0};N.prototype.disposeInternal=function(){};N.prototype.dispose=function(){};N.prototype.preventDefault=function(){this.defaultPrevented=!0;this.returnValue_=!1};var O=function(a,b){N.call(this,a?a.type:"");this.relatedTarget=this.currentTarget=this.target=null;this.charCode=this.keyCode=this.button=this.screenY=this.screenX=this.clientY=this.clientX=this.offsetY=this.offsetX=0;this.metaKey=this.shiftKey=this.altKey=this.ctrlKey=!1;this.event_=this.state=null;if(a){var c=this.type=a.type;this.target=a.target||a.srcElement;this.currentTarget=b;var d=a.relatedTarget;if(d){if(D){var e;t:{try{lc(d.nodeName);e=!0;break t}catch(g){}e=!1}e||(d=null)}}else"mouseover"==
+c?d=a.fromElement:"mouseout"==c&&(d=a.toElement);this.relatedTarget=d;this.offsetX=E||void 0!==a.offsetX?a.offsetX:a.layerX;this.offsetY=E||void 0!==a.offsetY?a.offsetY:a.layerY;this.clientX=void 0!==a.clientX?a.clientX:a.pageX;this.clientY=void 0!==a.clientY?a.clientY:a.pageY;this.screenX=a.screenX||0;this.screenY=a.screenY||0;this.button=a.button;this.keyCode=a.keyCode||0;this.charCode=a.charCode||("keypress"==c?a.keyCode:0);this.ctrlKey=a.ctrlKey;this.altKey=a.altKey;this.shiftKey=a.shiftKey;this.metaKey=
+a.metaKey;this.state=a.state;this.event_=a;a.defaultPrevented&&this.preventDefault()}};u(O,N);O.prototype.preventDefault=function(){O.superClass_.preventDefault.call(this);var a=this.event_;if(a.preventDefault)a.preventDefault();else if(a.returnValue=!1,nc)try{if(a.ctrlKey||112<=a.keyCode&&123>=a.keyCode)a.keyCode=-1}catch(b){}};O.prototype.disposeInternal=function(){};var oc="closure_listenable_"+(1E6*Math.random()|0),pc=0;var qc=function(a,b,c,d,e){this.listener=a;this.proxy=null;this.src=b;this.type=c;this.capture=!!d;this.handler=e;this.key=++pc;this.removed=this.callOnce=!1},rc=function(a){a.removed=!0;a.listener=null;a.proxy=null;a.src=null;a.handler=null};var P=function(a){this.src=a;this.listeners={};this.typeCount_=0};P.prototype.add=function(a,b,c,d,e){var g=a.toString();a=this.listeners[g];a||(a=this.listeners[g]=[],this.typeCount_++);var f=sc(a,b,d,e);-1<f?(b=a[f],c||(b.callOnce=!1)):(b=new qc(b,this.src,g,!!d,e),b.callOnce=c,a.push(b));return b};
+P.prototype.remove=function(a,b,c,d){a=a.toString();if(!(a in this.listeners))return!1;var e=this.listeners[a];b=sc(e,b,c,d);return-1<b?(rc(e[b]),y(null!=e.length),z.splice.call(e,b,1),0==e.length&&(delete this.listeners[a],this.typeCount_--),!0):!1};var tc=function(a,b){var c=b.type;if(!(c in a.listeners))return!1;var d=Qa(a.listeners[c],b);d&&(rc(b),0==a.listeners[c].length&&(delete a.listeners[c],a.typeCount_--));return d};
+P.prototype.removeAll=function(a){a=a&&a.toString();var b=0,c;for(c in this.listeners)if(!a||c==a){for(var d=this.listeners[c],e=0;e<d.length;e++)++b,rc(d[e]);delete this.listeners[c];this.typeCount_--}return b};P.prototype.getListener=function(a,b,c,d){a=this.listeners[a.toString()];var e=-1;a&&(e=sc(a,b,c,d));return-1<e?a[e]:null};var sc=function(a,b,c,d){for(var e=0;e<a.length;++e){var g=a[e];if(!g.removed&&g.listener==b&&g.capture==!!c&&g.handler==d)return e}return-1};var uc="closure_lm_"+(1E6*Math.random()|0),vc={},wc=0,xc=function(a,b,c,d,e){if(p(b)){for(var g=0;g<b.length;g++)xc(a,b[g],c,d,e);return null}c=yc(c);if(a&&a[oc])a=a.listen(b,c,d,e);else{if(!b)throw Error("Invalid event type");var g=!!d,f=zc(a);f||(a[uc]=f=new P(a));c=f.add(b,c,!1,d,e);c.proxy||(d=Ac(),c.proxy=d,d.src=a,d.listener=c,a.addEventListener?a.addEventListener(b.toString(),d,g):a.attachEvent(Bc(b.toString()),d),wc++);a=c}return a},Ac=function(){var a=Cc,b=mc?function(c){return a.call(b.src,
+b.listener,c)}:function(c){c=a.call(b.src,b.listener,c);if(!c)return c};return b},Dc=function(a,b,c,d,e){if(p(b))for(var g=0;g<b.length;g++)Dc(a,b[g],c,d,e);else c=yc(c),a&&a[oc]?a.unlisten(b,c,d,e):a&&(a=zc(a))&&(b=a.getListener(b,c,!!d,e))&&Ec(b)},Ec=function(a){if("number"==typeof a||!a||a.removed)return!1;var b=a.src;if(b&&b[oc])return tc(b.eventTargetListeners_,a);var c=a.type,d=a.proxy;b.removeEventListener?b.removeEventListener(c,d,a.capture):b.detachEvent&&b.detachEvent(Bc(c),d);wc--;(c=zc(b))?
+(tc(c,a),0==c.typeCount_&&(c.src=null,b[uc]=null)):rc(a);return!0},Bc=function(a){return a in vc?vc[a]:vc[a]="on"+a},Gc=function(a,b,c,d){var e=1;if(a=zc(a))if(b=a.listeners[b.toString()])for(b=b.concat(),a=0;a<b.length;a++){var g=b[a];g&&g.capture==c&&!g.removed&&(e&=!1!==Fc(g,d))}return Boolean(e)},Fc=function(a,b){var c=a.listener,d=a.handler||a.src;a.callOnce&&Ec(a);return c.call(d,b)},Cc=function(a,b){if(a.removed)return!0;if(!mc){var c;if(!(c=b))t:{c=["window","event"];for(var d=k,e;e=c.shift();)if(null!=
+d[e])d=d[e];else{c=null;break t}c=d}e=c;c=new O(e,this);d=!0;if(!(0>e.keyCode||void 0!=e.returnValue)){t:{var g=!1;if(0==e.keyCode)try{e.keyCode=-1;break t}catch(f){g=!0}if(g||void 0==e.returnValue)e.returnValue=!0}e=[];for(g=c.currentTarget;g;g=g.parentNode)e.push(g);for(var g=a.type,m=e.length-1;!c.propagationStopped_&&0<=m;m--)c.currentTarget=e[m],d&=Gc(e[m],g,!0,c);for(m=0;!c.propagationStopped_&&m<e.length;m++)c.currentTarget=e[m],d&=Gc(e[m],g,!1,c)}return d}return Fc(a,new O(b,this))},zc=function(a){a=
+a[uc];return a instanceof P?a:null},Hc="__closure_events_fn_"+(1E9*Math.random()>>>0),yc=function(a){y(a,"Listener can not be null.");if(r(a))return a;y(a.handleEvent,"An object listener must have handleEvent method.");a[Hc]||(a[Hc]=function(b){return a.handleEvent(b)});return a[Hc]};var Q=function(a){H.call(this);this.handler_=a;this.keys_={}};u(Q,H);var Ic=[];h=Q.prototype;h.listen=function(a,b,c,d){p(b)||(b&&(Ic[0]=b.toString()),b=Ic);for(var e=0;e<b.length;e++){var g=xc(a,b[e],c||this.handleEvent,d||!1,this.handler_||this);if(!g)break;this.keys_[g.key]=g}return this};
+h.unlisten=function(a,b,c,d,e){if(p(b))for(var g=0;g<b.length;g++)this.unlisten(a,b[g],c,d,e);else c=c||this.handleEvent,e=e||this.handler_||this,c=yc(c),d=!!d,b=a&&a[oc]?a.getListener(b,c,d,e):a?(a=zc(a))?a.getListener(b,c,d,e):null:null,b&&(Ec(b),delete this.keys_[b.key]);return this};h.removeAll=function(){Ta(this.keys_,Ec);this.keys_={}};h.disposeInternal=function(){Q.superClass_.disposeInternal.call(this);this.removeAll()};
+h.handleEvent=function(){throw Error("EventHandler.handleEvent not implemented");};var R=function(){H.call(this);this.eventTargetListeners_=new P(this);this.actualEventTarget_=this;this.parentEventTarget_=null};u(R,H);R.prototype[oc]=!0;h=R.prototype;h.setParentEventTarget=function(a){this.parentEventTarget_=a};h.addEventListener=function(a,b,c,d){xc(this,a,b,c,d)};h.removeEventListener=function(a,b,c,d){Dc(this,a,b,c,d)};
+h.dispatchEvent=function(a){Jc(this);var b,c=this.parentEventTarget_;if(c){b=[];for(var d=1;c;c=c.parentEventTarget_)b.push(c),y(1E3>++d,"infinite loop")}c=this.actualEventTarget_;d=a.type||a;if(q(a))a=new N(a,c);else if(a instanceof N)a.target=a.target||c;else{var e=a;a=new N(d,c);Za(a,e)}var e=!0,g;if(b)for(var f=b.length-1;!a.propagationStopped_&&0<=f;f--)g=a.currentTarget=b[f],e=Kc(g,d,!0,a)&&e;a.propagationStopped_||(g=a.currentTarget=c,e=Kc(g,d,!0,a)&&e,a.propagationStopped_||(e=Kc(g,d,!1,a)&&
+e));if(b)for(f=0;!a.propagationStopped_&&f<b.length;f++)g=a.currentTarget=b[f],e=Kc(g,d,!1,a)&&e;return e};h.disposeInternal=function(){R.superClass_.disposeInternal.call(this);this.eventTargetListeners_&&this.eventTargetListeners_.removeAll(void 0);this.parentEventTarget_=null};h.listen=function(a,b,c,d){Jc(this);return this.eventTargetListeners_.add(String(a),b,!1,c,d)};h.unlisten=function(a,b,c,d){return this.eventTargetListeners_.remove(String(a),b,c,d)};
+var Kc=function(a,b,c,d){b=a.eventTargetListeners_.listeners[String(b)];if(!b)return!0;b=b.concat();for(var e=!0,g=0;g<b.length;++g){var f=b[g];if(f&&!f.removed&&f.capture==c){var m=f.listener,$=f.handler||f.src;f.callOnce&&tc(a.eventTargetListeners_,f);e=!1!==m.call($,d)&&e}}return e&&0!=d.returnValue_};R.prototype.getListener=function(a,b,c,d){return this.eventTargetListeners_.getListener(String(a),b,c,d)};var Jc=function(a){y(a.eventTargetListeners_,"Event target is not initialized. Did you call the superclass (goog.events.EventTarget) constructor?")};var S=function(a){R.call(this);this.imageIdToRequestMap_={};this.imageIdToImageMap_={};this.handler_=new Q(this);this.parent_=a};u(S,R);var Lc=[C&&!F("11")?"readystatechange":"load","abort","error"],Mc=function(a,b,c){(c=q(c)?c:c.src)&&(a.imageIdToRequestMap_[b]={src:c,corsRequestType:l(void 0)?void 0:null})};
+S.prototype.start=function(){var a=this.imageIdToRequestMap_;Na(Ua(a),function(b){var c=a[b];if(c&&(delete a[b],!this.disposed_)){var d;d=this.parent_?sb(this.parent_).createDom("img"):new Image;c.corsRequestType&&(d.crossOrigin=c.corsRequestType);this.handler_.listen(d,Lc,this.onNetworkEvent_);this.imageIdToImageMap_[b]=d;d.id=b;d.src=c.src}},this)};
+S.prototype.onNetworkEvent_=function(a){var b=a.currentTarget;if(b){if("readystatechange"==a.type)if("complete"==b.readyState)a.type="load";else return;"undefined"==typeof b.naturalWidth&&("load"==a.type?(b.naturalWidth=b.width,b.naturalHeight=b.height):(b.naturalWidth=0,b.naturalHeight=0));this.dispatchEvent({type:a.type,target:b});!this.disposed_&&(a=b.id,delete this.imageIdToRequestMap_[a],b=this.imageIdToImageMap_[a])&&(delete this.imageIdToImageMap_[a],this.handler_.unlisten(b,Lc,this.onNetworkEvent_),
+Va(this.imageIdToImageMap_)&&Va(this.imageIdToRequestMap_)&&this.dispatchEvent("complete"))}};S.prototype.disposeInternal=function(){delete this.imageIdToRequestMap_;delete this.imageIdToImageMap_;ob(this.handler_);S.superClass_.disposeInternal.call(this)};var T=function(){};T.getInstance=function(){return T.instance_?T.instance_:T.instance_=new T};T.prototype.nextId_=0;var U=function(a){R.call(this);this.dom_=a||sb();this.id_=null;this.inDocument_=!1;this.element_=null;this.googUiComponentHandler_=void 0;this.childIndex_=this.children_=this.parent_=null;this.wasDecorated_=!1};u(U,R);h=U.prototype;h.idGenerator_=T.getInstance();h.getElement=function(){return this.element_};h.setParentEventTarget=function(a){if(this.parent_&&this.parent_!=a)throw Error("Method not supported");U.superClass_.setParentEventTarget.call(this,a)};h.getDomHelper=function(){return this.dom_};
+h.createDom=function(){this.element_=this.dom_.createElement("div")};
+var Oc=function(a,b){if(a.inDocument_)throw Error("Component already rendered");a.element_||a.createDom();b?b.insertBefore(a.element_,null):a.dom_.document_.body.appendChild(a.element_);a.parent_&&!a.parent_.inDocument_||Nc(a)},Nc=function(a){a.inDocument_=!0;Pc(a,function(a){!a.inDocument_&&a.getElement()&&Nc(a)})},Qc=function(a){Pc(a,function(a){a.inDocument_&&Qc(a)});a.googUiComponentHandler_&&a.googUiComponentHandler_.removeAll();a.inDocument_=!1};
+U.prototype.disposeInternal=function(){this.inDocument_&&Qc(this);this.googUiComponentHandler_&&(this.googUiComponentHandler_.dispose(),delete this.googUiComponentHandler_);Pc(this,function(a){a.dispose()});!this.wasDecorated_&&this.element_&&Bb(this.element_);this.parent_=this.element_=this.childIndex_=this.children_=null;U.superClass_.disposeInternal.call(this)};var Pc=function(a,b){a.children_&&Na(a.children_,b,void 0)};
+U.prototype.removeChild=function(a,b){if(a){var c=q(a)?a:a.id_||(a.id_=":"+(a.idGenerator_.nextId_++).toString(36)),d;this.childIndex_&&c?(d=this.childIndex_,d=(c in d?d[c]:void 0)||null):d=null;a=d;if(c&&a){d=this.childIndex_;c in d&&delete d[c];Qa(this.children_,a);b&&(Qc(a),a.element_&&Bb(a.element_));c=a;if(null==c)throw Error("Unable to set parent component");c.parent_=null;U.superClass_.setParentEventTarget.call(c,null)}}if(!a)throw Error("Child is not in parent component");return a};var V=function(a,b,c){U.call(this,c);this.captchaImage_=a;this.adImage_=b&&300==b.naturalWidth&&57==b.naturalHeight?b:null};u(V,U);V.prototype.createDom=function(){V.superClass_.createDom.call(this);var a=this.getElement();this.captchaImage_.alt=W.image_alt_text;this.getDomHelper().appendChild(a,this.captchaImage_);this.adImage_&&(this.adImage_.alt=W.image_alt_text,this.getDomHelper().appendChild(a,this.adImage_),this.adImage_&&Rc(this.adImage_)&&(a.innerHTML+='<div id="recaptcha-ad-choices"><div class="recaptcha-ad-choices-collapsed"><img height="15" width="30" alt="AdChoices" border="0" src="//www.gstatic.com/recaptcha/api/img/adicon.png"/></div><div class="recaptcha-ad-choices-expanded"><a href="https://support.google.com/adsense/troubleshooter/1631343" target="_blank"><img height="15" width="75" alt="AdChoices" border="0" src="//www.gstatic.com/recaptcha/api/img/adchoices.png"/></a></div></div>'))};
+var Rc=function(a){var b=Sc(a,"visibility");a=Sc(a,"display");return"hidden"!=b&&"none"!=a},Sc=function(a,b){var c;t:{c=rb(a);if(c.defaultView&&c.defaultView.getComputedStyle&&(c=c.defaultView.getComputedStyle(a,null))){c=c[b]||c.getPropertyValue(b)||"";break t}c=""}if(!(c=c||(a.currentStyle?a.currentStyle[b]:null))&&(c=a.style[Fa(b)],"undefined"===typeof c)){c=a.style;var d;t:if(d=Fa(b),void 0===a.style[d]){var e=(E?"Webkit":D?"Moz":C?"ms":cb?"O":null)+Ga(d);if(void 0!==a.style[e]){d=e;break t}}c=
+c[d]||""}return c};V.prototype.disposeInternal=function(){delete this.captchaImage_;delete this.adImage_;V.superClass_.disposeInternal.call(this)};var Tc=function(a,b,c){H.call(this);this.listener_=a;this.interval_=b||0;this.handler_=c;this.callback_=s(this.doAction_,this)};u(Tc,H);h=Tc.prototype;h.id_=0;h.disposeInternal=function(){Tc.superClass_.disposeInternal.call(this);this.stop();delete this.listener_;delete this.handler_};
+h.start=function(a){this.stop();var b=this.callback_;a=l(a)?a:this.interval_;if(!r(b))if(b&&"function"==typeof b.handleEvent)b=s(b.handleEvent,b);else throw Error("Invalid listener argument");this.id_=2147483647<a?-1:k.setTimeout(b,a||0)};h.stop=function(){this.isActive()&&k.clearTimeout(this.id_);this.id_=0};h.isActive=function(){return 0!=this.id_};h.doAction_=function(){this.id_=0;this.listener_&&this.listener_.call(this.handler_)};var Uc=function(a,b){H.call(this);this.listener_=a;this.handler_=b;this.delay_=new Tc(s(this.onTick_,this),0,this)};u(Uc,H);h=Uc.prototype;h.interval_=0;h.runUntil_=0;h.disposeInternal=function(){this.delay_.dispose();delete this.listener_;delete this.handler_;Uc.superClass_.disposeInternal.call(this)};h.start=function(a,b){this.stop();var c=b||0;this.interval_=Math.max(a||0,0);this.runUntil_=0>c?-1:ha()+c;this.delay_.start(0>c?this.interval_:Math.min(this.interval_,c))};h.stop=function(){this.delay_.stop()};
+h.isActive=function(){return this.delay_.isActive()};h.onSuccess=function(){};h.onFailure=function(){};h.onTick_=function(){if(this.listener_.call(this.handler_))this.onSuccess();else if(0>this.runUntil_)this.delay_.start(this.interval_);else{var a=this.runUntil_-ha();if(0>=a)this.onFailure();else this.delay_.start(Math.min(this.interval_,a))}};$a("area base br col command embed hr img input keygen link meta param source track wbr".split(" "));$a("action","cite","data","formaction","href","manifest","poster","src");$a("link","script","style");var Vc={sanitizedContentKindHtml:!0},Wc={sanitizedContentKindText:!0},Xc=function(){throw Error("Do not instantiate directly");};Xc.prototype.contentDir=null;Xc.prototype.toString=function(){return this.content};var bd=function(a){var b=Yc;y(b,"Soy template may not be null.");var c=sb().createElement("DIV");a=Zc(b(a||$c,void 0,void 0));b=a.match(ad);y(!b,"This template starts with a %s, which cannot be a child of a <div>, as required by soy internals. Consider using goog.soy.renderElement instead.\nTemplate output: %s",b&&b[0],a);c.innerHTML=a;return 1==c.childNodes.length&&(a=c.firstChild,1==a.nodeType)?a:c},Zc=function(a){if(!da(a))return String(a);if(a instanceof Xc){if(a.contentKind===Vc)return Ka(a.content);
+if(a.contentKind===Wc)return Da(a.content)}Ja("Soy template output is unsafe for use as HTML: "+a);return"zSoyz"},ad=/^<(body|caption|col|colgroup|head|html|tr|td|tbody|thead|tfoot)>/i,$c={};C&&F(8);var cd=function(){Xc.call(this)};u(cd,Xc);cd.prototype.contentKind=Vc;var dd=function(a){function b(a){this.content=a}b.prototype=a.prototype;return function(a,d){var e=new b(String(a));void 0!==d&&(e.contentDir=d);return e}}(cd);(function(a){function b(a){this.content=a}b.prototype=a.prototype;return function(a,d){var e=String(a);if(!e)return"";e=new b(e);void 0!==d&&(e.contentDir=d);return e}})(cd);
+var ed={"\x00":"\\x00","\b":"\\x08","\t":"\\t","\n":"\\n","\x0B":"\\x0b","\f":"\\f","\r":"\\r",'"':"\\x22",$:"\\x24","&":"\\x26","'":"\\x27","(":"\\x28",")":"\\x29","*":"\\x2a","+":"\\x2b",",":"\\x2c","-":"\\x2d",".":"\\x2e","/":"\\/",":":"\\x3a","<":"\\x3c","=":"\\x3d",">":"\\x3e","?":"\\x3f","[":"\\x5b","\\":"\\\\","]":"\\x5d","^":"\\x5e","{":"\\x7b","|":"\\x7c","}":"\\x7d","\u0085":"\\x85","\u2028":"\\u2028","\u2029":"\\u2029"},fd=function(a){return ed[a]},gd=/[\x00\x08-\x0d\x22\x26\x27\/\x3c-\x3e\\\x85\u2028\u2029]/g;var Yc=function(a){return dd('<script type="text/javascript">var challenge = \''+String(a.challenge).replace(gd,fd)+"'; var publisherId = '"+String(a.publisherId).replace(gd,fd)+"';"+("ca-mongoogle"==a.publisherId?'google_page_url = "3pcerttesting.com/dab/recaptcha.html";':"")+"\n    google_ad_client = publisherId;\n    google_ad_type = 'html';\n    google_ad_output = 'js';\n    google_image_size = '300x57';\n    google_captcha_token = challenge;\n    google_ad_request_done = function(ad) {\n      window.parent.recaptcha.ads.adutils.googleAdRequestDone(ad);\n    };\n    \x3c/script><script type=\"text/javascript\" src=\"//pagead2.googlesyndication.com/pagead/show_ads.js\">\x3c/script>")};
+Yc.soyTemplateName="recaptcha.soy.ads.iframeAdsLoader.main";var Wa=function(){var a=k.google_ad;return!!(a&&a.token&&a.imageAdUrl&&a.hashedAnswer&&a.salt&&a.delayedImpressionUrl&&a.engagementUrl)},hd=function(){k.google_ad&&(k.google_ad=null)},id=function(a){a=a||document.body;var b=k.google_ad;b&&b.searchUpliftUrl&&(b=xb("iframe",{src:'data:text/html;charset=utf-8,<body><img src="https://'+b.searchUpliftUrl+'"></img></body>',style:"display:none"}),a.appendChild(b))},jd=0,kd=function(a){var b=new S;Mc(b,"recaptcha-url-"+jd++,a);b.start()},ld=function(a,b){var c=
+RecaptchaState.publisher_id;hd();var d=xb("iframe",{id:"recaptcha-loader-"+jd++,style:"display: none"});document.body.appendChild(d);var e=d.contentWindow?d.contentWindow.document:d.contentDocument;e.open("text/html","replace");e.write(bd({challenge:a,publisherId:c}).innerHTML);e.close();c=new Uc(function(){return!!k.google_ad});c.onSuccess=function(){Bb(d);b()};c.onFailure=function(){Bb(d);b()};c.start(50,2E3)};t("recaptcha.ads.adutils.googleAdRequestDone",function(a){k.google_ad=a});var md=function(){this.blockSize=-1};var nd=function(){this.blockSize=-1;this.blockSize=64;this.chain_=Array(4);this.block_=Array(this.blockSize);this.totalLength_=this.blockLength_=0;this.reset()};u(nd,md);nd.prototype.reset=function(){this.chain_[0]=1732584193;this.chain_[1]=4023233417;this.chain_[2]=2562383102;this.chain_[3]=271733878;this.totalLength_=this.blockLength_=0};
+var od=function(a,b,c){c||(c=0);var d=Array(16);if(q(b))for(var e=0;16>e;++e)d[e]=b.charCodeAt(c++)|b.charCodeAt(c++)<<8|b.charCodeAt(c++)<<16|b.charCodeAt(c++)<<24;else for(e=0;16>e;++e)d[e]=b[c++]|b[c++]<<8|b[c++]<<16|b[c++]<<24;b=a.chain_[0];c=a.chain_[1];var e=a.chain_[2],g=a.chain_[3],f=0,f=b+(g^c&(e^g))+d[0]+3614090360&4294967295;b=c+(f<<7&4294967295|f>>>25);f=g+(e^b&(c^e))+d[1]+3905402710&4294967295;g=b+(f<<12&4294967295|f>>>20);f=e+(c^g&(b^c))+d[2]+606105819&4294967295;e=g+(f<<17&4294967295|
+f>>>15);f=c+(b^e&(g^b))+d[3]+3250441966&4294967295;c=e+(f<<22&4294967295|f>>>10);f=b+(g^c&(e^g))+d[4]+4118548399&4294967295;b=c+(f<<7&4294967295|f>>>25);f=g+(e^b&(c^e))+d[5]+1200080426&4294967295;g=b+(f<<12&4294967295|f>>>20);f=e+(c^g&(b^c))+d[6]+2821735955&4294967295;e=g+(f<<17&4294967295|f>>>15);f=c+(b^e&(g^b))+d[7]+4249261313&4294967295;c=e+(f<<22&4294967295|f>>>10);f=b+(g^c&(e^g))+d[8]+1770035416&4294967295;b=c+(f<<7&4294967295|f>>>25);f=g+(e^b&(c^e))+d[9]+2336552879&4294967295;g=b+(f<<12&4294967295|
+f>>>20);f=e+(c^g&(b^c))+d[10]+4294925233&4294967295;e=g+(f<<17&4294967295|f>>>15);f=c+(b^e&(g^b))+d[11]+2304563134&4294967295;c=e+(f<<22&4294967295|f>>>10);f=b+(g^c&(e^g))+d[12]+1804603682&4294967295;b=c+(f<<7&4294967295|f>>>25);f=g+(e^b&(c^e))+d[13]+4254626195&4294967295;g=b+(f<<12&4294967295|f>>>20);f=e+(c^g&(b^c))+d[14]+2792965006&4294967295;e=g+(f<<17&4294967295|f>>>15);f=c+(b^e&(g^b))+d[15]+1236535329&4294967295;c=e+(f<<22&4294967295|f>>>10);f=b+(e^g&(c^e))+d[1]+4129170786&4294967295;b=c+(f<<
+5&4294967295|f>>>27);f=g+(c^e&(b^c))+d[6]+3225465664&4294967295;g=b+(f<<9&4294967295|f>>>23);f=e+(b^c&(g^b))+d[11]+643717713&4294967295;e=g+(f<<14&4294967295|f>>>18);f=c+(g^b&(e^g))+d[0]+3921069994&4294967295;c=e+(f<<20&4294967295|f>>>12);f=b+(e^g&(c^e))+d[5]+3593408605&4294967295;b=c+(f<<5&4294967295|f>>>27);f=g+(c^e&(b^c))+d[10]+38016083&4294967295;g=b+(f<<9&4294967295|f>>>23);f=e+(b^c&(g^b))+d[15]+3634488961&4294967295;e=g+(f<<14&4294967295|f>>>18);f=c+(g^b&(e^g))+d[4]+3889429448&4294967295;c=
+e+(f<<20&4294967295|f>>>12);f=b+(e^g&(c^e))+d[9]+568446438&4294967295;b=c+(f<<5&4294967295|f>>>27);f=g+(c^e&(b^c))+d[14]+3275163606&4294967295;g=b+(f<<9&4294967295|f>>>23);f=e+(b^c&(g^b))+d[3]+4107603335&4294967295;e=g+(f<<14&4294967295|f>>>18);f=c+(g^b&(e^g))+d[8]+1163531501&4294967295;c=e+(f<<20&4294967295|f>>>12);f=b+(e^g&(c^e))+d[13]+2850285829&4294967295;b=c+(f<<5&4294967295|f>>>27);f=g+(c^e&(b^c))+d[2]+4243563512&4294967295;g=b+(f<<9&4294967295|f>>>23);f=e+(b^c&(g^b))+d[7]+1735328473&4294967295;
+e=g+(f<<14&4294967295|f>>>18);f=c+(g^b&(e^g))+d[12]+2368359562&4294967295;c=e+(f<<20&4294967295|f>>>12);f=b+(c^e^g)+d[5]+4294588738&4294967295;b=c+(f<<4&4294967295|f>>>28);f=g+(b^c^e)+d[8]+2272392833&4294967295;g=b+(f<<11&4294967295|f>>>21);f=e+(g^b^c)+d[11]+1839030562&4294967295;e=g+(f<<16&4294967295|f>>>16);f=c+(e^g^b)+d[14]+4259657740&4294967295;c=e+(f<<23&4294967295|f>>>9);f=b+(c^e^g)+d[1]+2763975236&4294967295;b=c+(f<<4&4294967295|f>>>28);f=g+(b^c^e)+d[4]+1272893353&4294967295;g=b+(f<<11&4294967295|
+f>>>21);f=e+(g^b^c)+d[7]+4139469664&4294967295;e=g+(f<<16&4294967295|f>>>16);f=c+(e^g^b)+d[10]+3200236656&4294967295;c=e+(f<<23&4294967295|f>>>9);f=b+(c^e^g)+d[13]+681279174&4294967295;b=c+(f<<4&4294967295|f>>>28);f=g+(b^c^e)+d[0]+3936430074&4294967295;g=b+(f<<11&4294967295|f>>>21);f=e+(g^b^c)+d[3]+3572445317&4294967295;e=g+(f<<16&4294967295|f>>>16);f=c+(e^g^b)+d[6]+76029189&4294967295;c=e+(f<<23&4294967295|f>>>9);f=b+(c^e^g)+d[9]+3654602809&4294967295;b=c+(f<<4&4294967295|f>>>28);f=g+(b^c^e)+d[12]+
+3873151461&4294967295;g=b+(f<<11&4294967295|f>>>21);f=e+(g^b^c)+d[15]+530742520&4294967295;e=g+(f<<16&4294967295|f>>>16);f=c+(e^g^b)+d[2]+3299628645&4294967295;c=e+(f<<23&4294967295|f>>>9);f=b+(e^(c|~g))+d[0]+4096336452&4294967295;b=c+(f<<6&4294967295|f>>>26);f=g+(c^(b|~e))+d[7]+1126891415&4294967295;g=b+(f<<10&4294967295|f>>>22);f=e+(b^(g|~c))+d[14]+2878612391&4294967295;e=g+(f<<15&4294967295|f>>>17);f=c+(g^(e|~b))+d[5]+4237533241&4294967295;c=e+(f<<21&4294967295|f>>>11);f=b+(e^(c|~g))+d[12]+1700485571&
+4294967295;b=c+(f<<6&4294967295|f>>>26);f=g+(c^(b|~e))+d[3]+2399980690&4294967295;g=b+(f<<10&4294967295|f>>>22);f=e+(b^(g|~c))+d[10]+4293915773&4294967295;e=g+(f<<15&4294967295|f>>>17);f=c+(g^(e|~b))+d[1]+2240044497&4294967295;c=e+(f<<21&4294967295|f>>>11);f=b+(e^(c|~g))+d[8]+1873313359&4294967295;b=c+(f<<6&4294967295|f>>>26);f=g+(c^(b|~e))+d[15]+4264355552&4294967295;g=b+(f<<10&4294967295|f>>>22);f=e+(b^(g|~c))+d[6]+2734768916&4294967295;e=g+(f<<15&4294967295|f>>>17);f=c+(g^(e|~b))+d[13]+1309151649&
+4294967295;c=e+(f<<21&4294967295|f>>>11);f=b+(e^(c|~g))+d[4]+4149444226&4294967295;b=c+(f<<6&4294967295|f>>>26);f=g+(c^(b|~e))+d[11]+3174756917&4294967295;g=b+(f<<10&4294967295|f>>>22);f=e+(b^(g|~c))+d[2]+718787259&4294967295;e=g+(f<<15&4294967295|f>>>17);f=c+(g^(e|~b))+d[9]+3951481745&4294967295;a.chain_[0]=a.chain_[0]+b&4294967295;a.chain_[1]=a.chain_[1]+(e+(f<<21&4294967295|f>>>11))&4294967295;a.chain_[2]=a.chain_[2]+e&4294967295;a.chain_[3]=a.chain_[3]+g&4294967295};
+nd.prototype.update=function(a,b){l(b)||(b=a.length);for(var c=b-this.blockSize,d=this.block_,e=this.blockLength_,g=0;g<b;){if(0==e)for(;g<=c;)od(this,a,g),g+=this.blockSize;if(q(a))for(;g<b;){if(d[e++]=a.charCodeAt(g++),e==this.blockSize){od(this,d);e=0;break}}else for(;g<b;)if(d[e++]=a[g++],e==this.blockSize){od(this,d);e=0;break}}this.blockLength_=e;this.totalLength_+=b};var X=function(){Q.call(this);this.callback_=this.element_=null;this.md5_=new nd};u(X,Q);var pd=function(a,b,c,d,e){a.unwatch();a.element_=b;a.callback_=e;a.listen(b,"keyup",s(a.onChanged_,a,c,d))};X.prototype.unwatch=function(){this.element_&&this.callback_&&(this.removeAll(),this.callback_=this.element_=null)};
+X.prototype.onChanged_=function(a,b){var c;c=(c=this.element_.value)?c.replace(/[\s\xa0]+/g,"").toLowerCase():"";this.md5_.reset();this.md5_.update(c+"."+b);c=this.md5_;var d=Array((56>c.blockLength_?c.blockSize:2*c.blockSize)-c.blockLength_);d[0]=128;for(var e=1;e<d.length-8;++e)d[e]=0;for(var g=8*c.totalLength_,e=d.length-8;e<d.length;++e)d[e]=g&255,g/=256;c.update(d);d=Array(16);for(e=g=0;4>e;++e)for(var f=0;32>f;f+=8)d[g++]=c.chain_[e]>>>f&255;jb(d).toLowerCase()==a.toLowerCase()&&this.callback_()};
+X.prototype.disposeInternal=function(){this.element_=null;X.superClass_.disposeInternal.call(this)};var rd=function(a,b,c){this.adObject_=a;this.captchaImageUrl_=b;this.opt_successCallback_=c||null;qd(this)};u(rd,H);var qd=function(a){var b=new S;nb(a,ga(ob,b));Mc(b,"recaptcha_challenge_image",a.captchaImageUrl_);Mc(b,"recaptcha_ad_image",a.adObject_.imageAdUrl);var c={};xc(b,"load",s(function(a,b){a[b.target.id]=b.target},a,c));xc(b,"complete",s(a.handleImagesLoaded_,a,c));b.start()};
+rd.prototype.handleImagesLoaded_=function(a){a=new V(a.recaptcha_challenge_image,a.recaptcha_ad_image);nb(this,ga(ob,a));var b=tb(document,"recaptcha_image");Ab(b);Oc(a,b);a.adImage_&&Rc(a.adImage_)&&(kd(this.adObject_.delayedImpressionUrl),a=new X,nb(this,ga(ob,a)),pd(a,tb(document,"recaptcha_response_field"),this.adObject_.hashedAnswer,this.adObject_.salt,s(function(a,b){a.unwatch();kd(b)},this,a,this.adObject_.engagementUrl)),this.opt_successCallback_&&this.opt_successCallback_("04"+this.adObject_.token))};var W=w;t("RecaptchaStr",W);var Y=k.RecaptchaOptions;t("RecaptchaOptions",Y);var sd={tabindex:0,theme:"red",callback:null,lang:null,custom_theme_widget:null,custom_translations:null};t("RecaptchaDefaultOptions",sd);
+var Z={widget:null,timer_id:-1,style_set:!1,theme:null,type:"image",ajax_verify_cb:null,th1:null,th2:null,th3:null,element:"",ad_captcha_plugin:null,reload_timeout:-1,force_reload:!1,$:function(a){return"string"==typeof a?document.getElementById(a):a},attachEvent:function(a,b,c){a&&a.addEventListener?a.addEventListener(b,c,!1):a&&a.attachEvent&&a.attachEvent("on"+b,c)},create:function(a,b,c){Z.destroy();b&&(Z.widget=Z.$(b),Z.element=b);Z._init_options(c);Z._call_challenge(a)},destroy:function(){var a=
+Z.$("recaptcha_challenge_field");a&&a.parentNode.removeChild(a);-1!=Z.timer_id&&clearInterval(Z.timer_id);Z.timer_id=-1;if(a=Z.$("recaptcha_image"))a.innerHTML="";Z.update_widget();Z.widget&&("custom"!=Z.theme?Z.widget.innerHTML="":Z.widget.style.display="none",Z.widget=null)},focus_response_field:function(){var a=Z.$("recaptcha_response_field");a&&a.focus()},get_challenge:function(){return"undefined"==typeof RecaptchaState?null:RecaptchaState.challenge},get_response:function(){var a=Z.$("recaptcha_response_field");
+return a?a.value:null},ajax_verify:function(a){Z.ajax_verify_cb=a;a=Z.get_challenge()||"";var b=Z.get_response()||"";a=Z._get_api_server()+"/ajaxverify?c="+encodeURIComponent(a)+"&response="+encodeURIComponent(b);Z._add_script(a)},_ajax_verify_callback:function(a){Z.ajax_verify_cb(a)},_get_overridable_url:function(a){var b=window.location.protocol;if("undefined"!=typeof _RecaptchaOverrideApiServer)a=_RecaptchaOverrideApiServer;else if("undefined"!=typeof RecaptchaState&&"string"==typeof RecaptchaState.server&&
+0<RecaptchaState.server.length)return RecaptchaState.server.replace(/\/+$/,"");return b+"//"+a},_get_api_server:function(){return Z._get_overridable_url("www.google.com/recaptcha/api")},_get_static_url_root:function(){return Z._get_overridable_url("www.gstatic.com/recaptcha/api")},_call_challenge:function(a){a=Z._get_api_server()+"/challenge?k="+a+"&ajax=1&cachestop="+Math.random();Z.getLang_()&&(a+="&lang="+Z.getLang_());"undefined"!=typeof Y.extra_challenge_params&&(a+="&"+Y.extra_challenge_params);
+Z._add_script(a)},_add_script:function(a){var b=document.createElement("script");b.type="text/javascript";b.src=a;Z._get_script_area().appendChild(b)},_get_script_area:function(){var a=document.getElementsByTagName("head");return a=!a||1>a.length?document.body:a[0]},_hash_merge:function(a){for(var b={},c=0;c<a.length;c++)for(var d in a[c])b[d]=a[c][d];return b},_init_options:function(a){Y=Z._hash_merge([sd,a||{}])},challenge_callback_internal:function(){Z.update_widget();Z._reset_timer();W=Z._hash_merge([w,
+sa[Z.getLang_()]||{},Y.custom_translations||{}]);window.addEventListener&&window.addEventListener("unload",function(){Z.destroy()},!1);Z._is_ie()&&window.attachEvent&&window.attachEvent("onbeforeunload",function(){});if(0<navigator.userAgent.indexOf("KHTML")){var a=document.createElement("iframe");a.src="about:blank";a.style.height="0px";a.style.width="0px";a.style.visibility="hidden";a.style.border="none";a.appendChild(document.createTextNode("This frame prevents back/forward cache problems in Safari."));
+document.body.appendChild(a)}Z._finish_widget()},_add_css:function(a){if(-1!=navigator.appVersion.indexOf("MSIE 5"))document.write('<style type="text/css">'+a+"</style>");else{var b=document.createElement("style");b.type="text/css";b.styleSheet?b.styleSheet.cssText=a:b.appendChild(document.createTextNode(a));Z._get_script_area().appendChild(b)}},_set_style:function(a){Z.style_set||(Z.style_set=!0,Z._add_css(a+"\n\n.recaptcha_is_showing_audio .recaptcha_only_if_image,.recaptcha_isnot_showing_audio .recaptcha_only_if_audio,.recaptcha_had_incorrect_sol .recaptcha_only_if_no_incorrect_sol,.recaptcha_nothad_incorrect_sol .recaptcha_only_if_incorrect_sol{display:none !important}"))},
+_init_builtin_theme:function(){var a=Z.$,b=Z._get_static_url_root(),c=v.VertCss,d=v.VertHtml,e=b+"/img/"+Z.theme,g="gif",b=Z.theme;"clean"==b&&(c=v.CleanCss,d=v.CleanHtml,g="png");c=c.replace(/IMGROOT/g,e);Z._set_style(c);Z.update_widget();Z.widget.innerHTML='<div id="recaptcha_area">'+d+"</div>";c=Z.getLang_();a("recaptcha_privacy")&&null!=c&&"en"==c.substring(0,2).toLowerCase()&&null!=W.privacy_and_terms&&0<W.privacy_and_terms.length&&(c=document.createElement("a"),c.href="http://www.google.com/intl/en/policies/",
+c.target="_blank",c.innerHTML=W.privacy_and_terms,a("recaptcha_privacy").appendChild(c));c=function(b,c,d,K){var G=a(b);G.src=e+"/"+c+"."+g;c=W[d];G.alt=c;b=a(b+"_btn");b.title=c;Z.attachEvent(b,"click",K)};c("recaptcha_reload","refresh","refresh_btn",function(){Z.reload_internal("r")});c("recaptcha_switch_audio","audio","audio_challenge",function(){Z.switch_type("audio")});c("recaptcha_switch_img","text","visual_challenge",function(){Z.switch_type("image")});c("recaptcha_whatsthis","help","help_btn",
+Z.showhelp);"clean"==b&&(a("recaptcha_logo").src=e+"/logo."+g);a("recaptcha_table").className="recaptchatable recaptcha_theme_"+Z.theme;b=function(b,c){var d=a(b);d&&(RecaptchaState.rtl&&"span"==d.tagName.toLowerCase()&&(d.dir="rtl"),d.appendChild(document.createTextNode(W[c])))};b("recaptcha_instructions_image","instructions_visual");b("recaptcha_instructions_audio","instructions_audio");b("recaptcha_instructions_error","incorrect_try_again");a("recaptcha_instructions_image")||a("recaptcha_instructions_audio")||
+(b="audio"==Z.type?W.instructions_audio:W.instructions_visual,b=b.replace(/:$/,""),a("recaptcha_response_field").setAttribute("placeholder",b))},_finish_widget:function(){var a=Z.$,b=Y,c=b.theme;c in{blackglass:1,clean:1,custom:1,red:1,white:1}||(c="red");Z.theme||(Z.theme=c);"custom"!=Z.theme?Z._init_builtin_theme():Z._set_style("");c=document.createElement("span");c.id="recaptcha_challenge_field_holder";c.style.display="none";a("recaptcha_response_field").parentNode.insertBefore(c,a("recaptcha_response_field"));
+a("recaptcha_response_field").setAttribute("autocomplete","off");a("recaptcha_image").style.width="300px";a("recaptcha_image").style.height="57px";a("recaptcha_challenge_field_holder").innerHTML='<input type="hidden" name="recaptcha_challenge_field" id="recaptcha_challenge_field" value=""/>';Z.th_init();Z.should_focus=!1;Z.th3||Z.force_reload?(Z._set_challenge(RecaptchaState.challenge,"image",!0),setTimeout(function(){Z.reload_internal("i")},100)):Z._set_challenge(RecaptchaState.challenge,"image",
+!1);Z.updateTabIndexes_();Z.update_widget();Z.widget&&(Z.widget.style.display="");b.callback&&b.callback()},updateTabIndexes_:function(){var a=Z.$,b=Y;b.tabindex&&(b=b.tabindex,a("recaptcha_response_field").tabIndex=b++,"audio"==Z.type&&a("recaptcha_audio_play_again")&&(a("recaptcha_audio_play_again").tabIndex=b++,a("recaptcha_audio_download"),a("recaptcha_audio_download").tabIndex=b++),"custom"!=Z.theme&&(a("recaptcha_reload_btn").tabIndex=b++,a("recaptcha_switch_audio_btn").tabIndex=b++,a("recaptcha_switch_img_btn").tabIndex=
+b++,a("recaptcha_whatsthis_btn").tabIndex=b,a("recaptcha_privacy").tabIndex=b++))},switch_type:function(a){if(!((new Date).getTime()<Z.reload_timeout)&&(Z.type=a,Z.reload_internal("audio"==Z.type?"a":"v"),"custom"!=Z.theme)){a=Z.$;var b="audio"==Z.type?W.instructions_audio:W.instructions_visual,b=b.replace(/:$/,"");a("recaptcha_response_field").setAttribute("placeholder",b)}},reload:function(){Z.reload_internal("r")},reload_internal:function(a){var b=Y,c=RecaptchaState,d=(new Date).getTime();d<Z.reload_timeout||
+(Z.reload_timeout=d+1E3,"undefined"==typeof a&&(a="r"),d=Z._get_api_server()+"/reload?c="+c.challenge+"&k="+c.site+"&reason="+a+"&type="+Z.type,Z.getLang_()&&(d+="&lang="+Z.getLang_()),"undefined"!=typeof b.extra_challenge_params&&(d+="&"+b.extra_challenge_params),Z.th_callback_invoke(),Z.th1&&(d+="&th="+Z.th1,Z.th1=""),"audio"==Z.type&&(d=b.audio_beta_12_08?d+"&audio_beta_12_08=1":d+"&new_audio_default=1"),Z.should_focus="t"!=a&&"i"!=a,Z._add_script(d),ob(Z.ad_captcha_plugin),c.publisher_id=null)},
+th_callback_invoke:function(){if(Z.th3)try{var a=Z.th3.exec();a&&1600>a.length&&(Z.th1=a)}catch(b){Z.th1=""}},finish_reload:function(a,b,c,d){RecaptchaState.payload_url=c;RecaptchaState.is_incorrect=!1;RecaptchaState.publisher_id=d;Z._set_challenge(a,b,!1);Z.updateTabIndexes_()},_set_challenge:function(a,b,c){"image"==b&&RecaptchaState.publisher_id?ld(a,function(){Z._set_challenge_internal(a,b,c)}):Z._set_challenge_internal(a,b,c)},_set_challenge_internal:function(a,b,c){var d=Z.$,e=RecaptchaState;
+e.challenge=a;Z.type=b;d("recaptcha_challenge_field").value=e.challenge;c||("audio"==b?(d("recaptcha_image").innerHTML=Z.getAudioCaptchaHtml(),Z._loop_playback()):"image"==b&&(a=e.payload_url,a||(a=Z._get_api_server()+"/image?c="+e.challenge,Z.th_callback_invoke(),Z.th1&&(a+="&th="+Z.th1,Z.th1="")),id(d("recaptcha_widget_div")),Wa()?Z.ad_captcha_plugin=new rd(Xa(),a,function(a){RecaptchaState.challenge=a;d("recaptcha_challenge_field").value=a}):d("recaptcha_image").innerHTML='<img id="recaptcha_challenge_image" alt="'+
+W.image_alt_text+'" height="57" width="300" src="'+a+'" />',hd()));Z._css_toggle("recaptcha_had_incorrect_sol","recaptcha_nothad_incorrect_sol",e.is_incorrect);Z._css_toggle("recaptcha_is_showing_audio","recaptcha_isnot_showing_audio","audio"==b);Z._clear_input();Z.should_focus&&Z.focus_response_field();Z._reset_timer()},_reset_timer:function(){clearInterval(Z.timer_id);var a=Math.max(1E3*(RecaptchaState.timeout-60),6E4);Z.timer_id=setInterval(function(){Z.reload_internal("t")},a);return a},showhelp:function(){window.open(Z._get_help_link(),
+"recaptcha_popup","width=460,height=580,location=no,menubar=no,status=no,toolbar=no,scrollbars=yes,resizable=yes")},_clear_input:function(){Z.$("recaptcha_response_field").value=""},_displayerror:function(a){var b=Z.$;b("recaptcha_image").innerHTML="";b("recaptcha_image").appendChild(document.createTextNode(a))},reloaderror:function(a){Z._displayerror(a)},_is_ie:function(){return 0<navigator.userAgent.indexOf("MSIE")&&!window.opera},_css_toggle:function(a,b,c){Z.update_widget();var d=Z.widget;d||
+(d=document.body);var e=d.className,e=e.replace(new RegExp("(^|\\s+)"+a+"(\\s+|$)")," "),e=e.replace(new RegExp("(^|\\s+)"+b+"(\\s+|$)")," ");d.className=e+(" "+(c?a:b))},_get_help_link:function(){var a="https://support.google.com/recaptcha/";Z.getLang_()&&(a+="?hl="+Z.getLang_());return a},playAgain:function(){Z.$("recaptcha_image").innerHTML=Z.getAudioCaptchaHtml();Z._loop_playback()},_loop_playback:function(){var a=Z.$("recaptcha_audio_play_again");a&&Z.attachEvent(a,"click",function(){Z.playAgain();
+return!1})},getAudioCaptchaHtml:function(){var a=RecaptchaState.payload_url;a||(a=Z._get_api_server()+"/audio.mp3?c="+RecaptchaState.challenge,Z.th_callback_invoke(),Z.th1&&(a+="&th="+Z.th1,Z.th1=""));var b=Z._get_api_server()+"/swf/audiocaptcha.swf?v2",b=Z._is_ie()?'<object classid="clsid:D27CDB6E-AE6D-11cf-96B8-444553540000" id="audiocaptcha" width="0" height="0" codebase="https://fpdownload.macromedia.com/get/flashplayer/current/swflash.cab"><param name="movie" value="'+b+'" /><param name="quality" value="high" /><param name="bgcolor" value="#869ca7" /><param name="allowScriptAccess" value="always" /></object><br/>':
+'<embed src="'+b+'" quality="high" bgcolor="#869ca7" width="0" height="0" name="audiocaptcha" align="middle" play="true" loop="false" quality="high" allowScriptAccess="always" type="application/x-shockwave-flash" pluginspage="http://www.adobe.com/go/getflashplayer" /></embed>',c="";Z.checkFlashVer()&&(c="<br/>"+Z.getSpan_('<a id="recaptcha_audio_play_again" class="recaptcha_audio_cant_hear_link">'+W.play_again+"</a>"));c+="<br/>"+Z.getSpan_('<a id="recaptcha_audio_download" class="recaptcha_audio_cant_hear_link" target="_blank" href="'+
+a+'">'+W.cant_hear_this+"</a>");return b+c},getSpan_:function(a){return"<span"+(RecaptchaState&&RecaptchaState.rtl?' dir="rtl"':"")+">"+a+"</span>"},gethttpwavurl:function(){if("audio"!=Z.type)return"";var a=RecaptchaState.payload_url;a||(a=Z._get_api_server()+"/image?c="+RecaptchaState.challenge,Z.th_callback_invoke(),Z.th1&&(a+="&th="+Z.th1,Z.th1=""));return a},checkFlashVer:function(){var a=-1!=navigator.appVersion.indexOf("MSIE"),b=-1!=navigator.appVersion.toLowerCase().indexOf("win"),c=-1!=navigator.userAgent.indexOf("Opera"),
+d=-1;if(null!=navigator.plugins&&0<navigator.plugins.length){if(navigator.plugins["Shockwave Flash 2.0"]||navigator.plugins["Shockwave Flash"])d=navigator.plugins["Shockwave Flash"+(navigator.plugins["Shockwave Flash 2.0"]?" 2.0":"")].description.split(" ")[2].split(".")[0]}else if(a&&b&&!c)try{d=(new ActiveXObject("ShockwaveFlash.ShockwaveFlash.7")).GetVariable("$version").split(" ")[1].split(",")[0]}catch(e){}return 9<=d},getLang_:function(){return Y.lang?Y.lang:"undefined"!=typeof RecaptchaState&&
+RecaptchaState.lang?RecaptchaState.lang:null},challenge_callback:function(){Z.force_reload=!!RecaptchaState.force_reload;if(RecaptchaState.t3){var a=RecaptchaState.t1?ib(mb(RecaptchaState.t1)):"",b=RecaptchaState.t2?ib(mb(RecaptchaState.t2)):"",c=RecaptchaState.t3?ib(mb(RecaptchaState.t3)):"";Z.th2=c;if(a)b=kc(a),cc(b,Z.challenge_callback_internal,null,void 0),cc(b,null,Z.challenge_callback_internal,void 0);else{if(k.execScript)k.execScript(b,"JavaScript");else if(k.eval)null==ia&&(k.eval("var _et_ = 1;"),
+"undefined"!=typeof k._et_?(delete k._et_,ia=!0):ia=!1),ia?k.eval(b):(a=k.document,c=a.createElement("script"),c.type="text/javascript",c.defer=!1,c.appendChild(a.createTextNode(b)),a.body.appendChild(c),a.body.removeChild(c));else throw Error("goog.globalEval not available");Z.challenge_callback_internal()}}else Z.challenge_callback_internal()},th_init:function(){try{k.thintinel&&k.thintinel.th&&(Z.th3=new k.thintinel.th(Z.th2),Z.th2="")}catch(a){}},update_widget:function(){Z.element&&(Z.widget=
+Z.$(Z.element))}};t("Recaptcha",Z);})()

From 8ba2dac6ea1c6400e1194323b27ad1eb46eb5f79 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 13 Aug 2018 18:54:01 +0100
Subject: [PATCH 379/830] Name changelog after PR, not bug

---
 changelog.d/{1932.bugfix => 3677.bugfix} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename changelog.d/{1932.bugfix => 3677.bugfix} (100%)

diff --git a/changelog.d/1932.bugfix b/changelog.d/3677.bugfix
similarity index 100%
rename from changelog.d/1932.bugfix
rename to changelog.d/3677.bugfix

From b6d55588e1055ad99d0640b4311551e959de462f Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 13 Aug 2018 18:56:55 +0100
Subject: [PATCH 380/830] Changelog should link to PRs

The changelog makes much more sense if it consistently links to PRs, rather
than mostly PRs with the occasional issue #.
---
 CONTRIBUTING.rst | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
index aa2738eea39d..3d75853aa7e2 100644
--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -30,11 +30,11 @@ use github's pull request workflow to review the contribution, and either ask
 you to make any refinements needed or merge it and make them ourselves. The
 changes will then land on master when we next do a release.
 
-We use `Jenkins <http://matrix.org/jenkins>`_ and 
+We use `Jenkins <http://matrix.org/jenkins>`_ and
 `Travis <https://travis-ci.org/matrix-org/synapse>`_ for continuous
-integration. All pull requests to synapse get automatically tested by Travis; 
-the Jenkins builds require an adminstrator to start them. If your change 
-breaks the build, this will be shown in github, so please keep an eye on the 
+integration. All pull requests to synapse get automatically tested by Travis;
+the Jenkins builds require an adminstrator to start them. If your change
+breaks the build, this will be shown in github, so please keep an eye on the
 pull request for feedback.
 
 Code style
@@ -56,17 +56,17 @@ entry. These are managed by Towncrier
 (https://github.com/hawkowl/towncrier).
 
 To create a changelog entry, make a new file in the ``changelog.d``
-file named in the format of ``issuenumberOrPR.type``. The type can be
+file named in the format of ``PRnumber.type``. The type can be
 one of ``feature``, ``bugfix``, ``removal`` (also used for
 deprecations), or ``misc`` (for internal-only changes). The content of
 the file is your changelog entry, which can contain RestructuredText
 formatting. A note of contributors is welcomed in changelogs for
 non-misc changes (the content of misc changes is not displayed).
 
-For example, a fix for a bug reported in #1234 would have its
-changelog entry in ``changelog.d/1234.bugfix``, and contain content
-like "The security levels of Florbs are now validated when
-recieved over federation. Contributed by Jane Matrix".
+For example, a fix in PR #1234 would have its changelog entry in
+``changelog.d/1234.bugfix``, and contain content like "The security levels of
+Florbs are now validated when recieved over federation. Contributed by Jane
+Matrix".
 
 Attribution
 ~~~~~~~~~~~
@@ -125,7 +125,7 @@ the contribution or otherwise have the right to contribute it to Matrix::
         personal information I submit with it, including my sign-off) is
         maintained indefinitely and may be redistributed consistent with
         this project or the open source license(s) involved.
-        
+
 If you agree to this for your contribution, then all that's needed is to
 include the line in your commit or pull request comment::
 

From f4b49152e27593dd6c863e71479a2ab712c4ada2 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Mon, 13 Aug 2018 18:00:23 +0100
Subject: [PATCH 381/830] support admin_email config and pass through into
 blocking errors, return AuthError in all cases

---
 synapse/api/auth.py             |  8 ++++++--
 synapse/api/errors.py           | 13 +++++++++++--
 synapse/config/server.py        |  4 ++++
 synapse/handlers/register.py    | 27 ++++++++++++++-------------
 tests/api/test_auth.py          |  6 +++++-
 tests/handlers/test_register.py |  8 ++++----
 tests/utils.py                  |  1 +
 7 files changed, 45 insertions(+), 22 deletions(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 9c62ec43742d..4f028078fa27 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -781,11 +781,15 @@ def check_auth_blocking(self):
         """
         if self.hs.config.hs_disabled:
             raise AuthError(
-                403, self.hs.config.hs_disabled_message, errcode=Codes.HS_DISABLED
+                403, self.hs.config.hs_disabled_message,
+                errcode=Codes.HS_DISABLED,
+                admin_email=self.hs.config.admin_email,
             )
         if self.hs.config.limit_usage_by_mau is True:
             current_mau = yield self.store.get_monthly_active_count()
             if current_mau >= self.hs.config.max_mau_value:
                 raise AuthError(
-                    403, "MAU Limit Exceeded", errcode=Codes.MAU_LIMIT_EXCEEDED
+                    403, "MAU Limit Exceeded",
+                    admin_email=self.hs.config.admin_email,
+                    errcode=Codes.MAU_LIMIT_EXCEEDED
                 )
diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index dc3bed5fcbbd..d74848159e44 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -225,11 +225,20 @@ def __init__(self, msg="Not found", errcode=Codes.NOT_FOUND):
 
 class AuthError(SynapseError):
     """An error raised when there was a problem authorising an event."""
-
     def __init__(self, *args, **kwargs):
         if "errcode" not in kwargs:
             kwargs["errcode"] = Codes.FORBIDDEN
-        super(AuthError, self).__init__(*args, **kwargs)
+        self.admin_email = kwargs.get('admin_email')
+        self.msg = kwargs.get('msg')
+        self.errcode = kwargs.get('errcode')
+        super(AuthError, self).__init__(*args, errcode=kwargs["errcode"])
+
+    def error_dict(self):
+        return cs_error(
+            self.msg,
+            self.errcode,
+            admin_email=self.admin_email,
+        )
 
 
 class EventSizeError(SynapseError):
diff --git a/synapse/config/server.py b/synapse/config/server.py
index 3b078d72cabd..64a5121a452f 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -82,6 +82,10 @@ def read_config(self, config):
         self.hs_disabled = config.get("hs_disabled", False)
         self.hs_disabled_message = config.get("hs_disabled_message", "")
 
+        # Admin email to direct users at should their instance become blocked
+        # due to resource constraints
+        self.admin_email = config.get("admin_email", None)
+
         # FIXME: federation_domain_whitelist needs sytests
         self.federation_domain_whitelist = None
         federation_domain_whitelist = config.get(
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index 3526b20d5a57..ef7222d7b860 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -144,7 +144,8 @@ def register(
         Raises:
             RegistrationError if there was a problem registering.
         """
-        yield self._check_mau_limits()
+
+        yield self.auth.check_auth_blocking()
         password_hash = None
         if password:
             password_hash = yield self.auth_handler().hash(password)
@@ -289,7 +290,7 @@ def register_saml2(self, localpart):
                 400,
                 "User ID can only contain characters a-z, 0-9, or '=_-./'",
             )
-        yield self._check_mau_limits()
+        yield self.auth.check_auth_blocking()
         user = UserID(localpart, self.hs.hostname)
         user_id = user.to_string()
 
@@ -439,7 +440,7 @@ def get_or_create_user(self, requester, localpart, displayname,
         """
         if localpart is None:
             raise SynapseError(400, "Request must include user id")
-        yield self._check_mau_limits()
+        yield self.auth.check_auth_blocking()
         need_register = True
 
         try:
@@ -534,13 +535,13 @@ def _join_user_to_room(self, requester, room_identifier):
             action="join",
         )
 
-    @defer.inlineCallbacks
-    def _check_mau_limits(self):
-        """
-        Do not accept registrations if monthly active user limits exceeded
-         and limiting is enabled
-        """
-        try:
-            yield self.auth.check_auth_blocking()
-        except AuthError as e:
-            raise RegistrationError(e.code, str(e), e.errcode)
+    # @defer.inlineCallbacks
+    # def _s(self):
+    #     """
+    #     Do not accept registrations if monthly active user limits exceeded
+    #      and limiting is enabled
+    #     """
+    #     try:
+    #         yield self.auth.check_auth_blocking()
+    #     except AuthError as e:
+    #         raise RegistrationError(e.code, str(e), e.errcode)
diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py
index a65689ba896e..e8a1894e651c 100644
--- a/tests/api/test_auth.py
+++ b/tests/api/test_auth.py
@@ -455,8 +455,11 @@ def test_blocking_mau(self):
             return_value=defer.succeed(lots_of_users)
         )
 
-        with self.assertRaises(AuthError):
+        with self.assertRaises(AuthError) as e:
             yield self.auth.check_auth_blocking()
+        self.assertEquals(e.exception.admin_email, self.hs.config.admin_email)
+        self.assertEquals(e.exception.errcode, Codes.MAU_LIMIT_EXCEEDED)
+        self.assertEquals(e.exception.code, 403)
 
         # Ensure does not throw an error
         self.store.get_monthly_active_count = Mock(
@@ -470,5 +473,6 @@ def test_hs_disabled(self):
         self.hs.config.hs_disabled_message = "Reason for being disabled"
         with self.assertRaises(AuthError) as e:
             yield self.auth.check_auth_blocking()
+        self.assertEquals(e.exception.admin_email, self.hs.config.admin_email)
         self.assertEquals(e.exception.errcode, Codes.HS_DISABLED)
         self.assertEquals(e.exception.code, 403)
diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py
index d48d40c8dd97..35d1bcab3e9f 100644
--- a/tests/handlers/test_register.py
+++ b/tests/handlers/test_register.py
@@ -17,7 +17,7 @@
 
 from twisted.internet import defer
 
-from synapse.api.errors import RegistrationError
+from synapse.api.errors import AuthError
 from synapse.handlers.register import RegistrationHandler
 from synapse.types import UserID, create_requester
 
@@ -109,7 +109,7 @@ def test_get_or_create_user_mau_blocked(self):
         self.store.get_monthly_active_count = Mock(
             return_value=defer.succeed(self.lots_of_users)
         )
-        with self.assertRaises(RegistrationError):
+        with self.assertRaises(AuthError):
             yield self.handler.get_or_create_user("requester", 'b', "display_name")
 
     @defer.inlineCallbacks
@@ -118,7 +118,7 @@ def test_register_mau_blocked(self):
         self.store.get_monthly_active_count = Mock(
             return_value=defer.succeed(self.lots_of_users)
         )
-        with self.assertRaises(RegistrationError):
+        with self.assertRaises(AuthError):
             yield self.handler.register(localpart="local_part")
 
     @defer.inlineCallbacks
@@ -127,5 +127,5 @@ def test_register_saml2_mau_blocked(self):
         self.store.get_monthly_active_count = Mock(
             return_value=defer.succeed(self.lots_of_users)
         )
-        with self.assertRaises(RegistrationError):
+        with self.assertRaises(AuthError):
             yield self.handler.register_saml2(localpart="local_part")
diff --git a/tests/utils.py b/tests/utils.py
index 90378326f898..4af81624eb39 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -139,6 +139,7 @@ def setup_test_homeserver(
         config.hs_disabled_message = ""
         config.max_mau_value = 50
         config.mau_limits_reserved_threepids = []
+        config.admin_email = None
 
         # we need a sane default_room_version, otherwise attempts to create rooms will
         # fail.

From 1f24d8681b74debeb8842ecf1df9fcc1b25b522e Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Mon, 13 Aug 2018 21:26:43 +0100
Subject: [PATCH 382/830] set admin email via config

---
 changelog.d/3687.feature | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3687.feature

diff --git a/changelog.d/3687.feature b/changelog.d/3687.feature
new file mode 100644
index 000000000000..93b24d1acb46
--- /dev/null
+++ b/changelog.d/3687.feature
@@ -0,0 +1 @@
+set admin email via config, to be used in error messages where the user should contact the administrator

From 63417c31e9d32992fbb422b630024946494ff26b Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Mon, 13 Aug 2018 22:36:52 +0100
Subject: [PATCH 383/830] fix typo

---
 synapse/server_notices/resource_limits_server_notices.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/server_notices/resource_limits_server_notices.py b/synapse/server_notices/resource_limits_server_notices.py
index 017828f81e72..e4be83643956 100644
--- a/synapse/server_notices/resource_limits_server_notices.py
+++ b/synapse/server_notices/resource_limits_server_notices.py
@@ -34,7 +34,7 @@ def __init__(self, hs):
         self._store = hs.get_datastore()
         self.auth = hs.get_auth()
         self._server_notice_content = hs.config.user_consent_server_notice_content
-        self._limit_usage_by_mau = hs.config.limit_usage_by_mau = False
+        self._limit_usage_by_mau = hs.config.limit_usage_by_mau
         self._hs_disabled = hs.config.hs_disabled
 
         self._notified_of_blocking = set()

From 9b75c78b4df276fe40d3f1b92a57481c3e970886 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Tue, 14 Aug 2018 11:20:41 +0100
Subject: [PATCH 384/830] support server notice state events for resource
 limits

---
 synapse/api/constants.py                      |  2 ++
 .../server_notices/consent_server_notices.py  |  3 ++-
 .../resource_limits_server_notices.py         | 21 ++++++++++++++-----
 .../server_notices/server_notices_manager.py  |  4 ++--
 4 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/synapse/api/constants.py b/synapse/api/constants.py
index b0da506f6d64..ef6084199593 100644
--- a/synapse/api/constants.py
+++ b/synapse/api/constants.py
@@ -79,6 +79,8 @@ class EventTypes(object):
 
     ServerACL = "m.room.server_acl"
 
+    ServerNoticeLimitReached = "m.server_notice.usage_limit_reached"
+
 
 class RejectedReason(object):
     AUTH_ERROR = "auth_error"
diff --git a/synapse/server_notices/consent_server_notices.py b/synapse/server_notices/consent_server_notices.py
index 5e3044d164ca..6d5caedb084e 100644
--- a/synapse/server_notices/consent_server_notices.py
+++ b/synapse/server_notices/consent_server_notices.py
@@ -22,6 +22,7 @@
 from synapse.api.urls import ConsentURIBuilder
 from synapse.config import ConfigError
 from synapse.types import get_localpart_from_id
+from synapse.api.constants import EventTypes
 
 logger = logging.getLogger(__name__)
 
@@ -103,7 +104,7 @@ def maybe_send_server_notice_to_user(self, user_id):
                     },
                 )
                 yield self._server_notices_manager.send_notice(
-                    user_id, content,
+                    user_id, content, EventTypes.Message
                 )
                 yield self._store.user_set_consent_server_notice_sent(
                     user_id, self._current_consent_version,
diff --git a/synapse/server_notices/resource_limits_server_notices.py b/synapse/server_notices/resource_limits_server_notices.py
index e4be83643956..2f49dae16822 100644
--- a/synapse/server_notices/resource_limits_server_notices.py
+++ b/synapse/server_notices/resource_limits_server_notices.py
@@ -17,6 +17,7 @@
 from twisted.internet import defer
 
 from synapse.api.errors import AuthError, SynapseError
+from synapse.api.constants import EventTypes
 
 logger = logging.getLogger(__name__)
 
@@ -61,22 +62,32 @@ def maybe_send_server_notice_to_user(self, user_id):
                 # In practice, not sure we can ever get here
                 return
             try:
+                # Normally should always pass in user_id if you have it, but in
+                # this case are checking what would happen to other users if they
+                # were to arrive.
                 yield self.auth.check_auth_blocking()
                 self._resouce_limited = False
                 # Need to start removing notices
                 if user_id in self._notified_of_blocking:
                     # Send message to remove warning - needs updating
-                    content = "remove warning"
+                    content = {
+                        'body': '',
+                        'admin_email': '',
+                    }
                     self._send_server_notice(user_id, content)
                     self._notified_of_blocking.remove(user_id)
 
-            except AuthError:
+            except AuthError as e:
                 # Need to start notifying of blocking
 
                 self._resouce_limited = True
                 if user_id not in self._notified_of_blocking:
-                    # Send message to add warning - needs updating
-                    content = "add warning"
+                    # TODO use admin email contained in error once PR lands
+                    content = {
+                        'body': e.msg,
+                        'admin_email': 'stunt@adminemail.com',
+                        'msgtype': 'm.text'
+                    }
                     self._send_server_notice(user_id, content)
                     self._notified_of_blocking.add(user_id)
 
@@ -93,7 +104,7 @@ def _send_server_notice(self, user_id, content):
         """
         try:
             yield self._server_notices_manager.send_notice(
-                user_id, content,
+                user_id, content, EventTypes.ServerNoticeLimitReached
             )
         except SynapseError as e:
             logger.error("Error sending server notice about resource limits: %s", e)
diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py
index a26deace5318..4806977a8933 100644
--- a/synapse/server_notices/server_notices_manager.py
+++ b/synapse/server_notices/server_notices_manager.py
@@ -46,7 +46,7 @@ def is_enabled(self):
         return self._config.server_notices_mxid is not None
 
     @defer.inlineCallbacks
-    def send_notice(self, user_id, event_content):
+    def send_notice(self, user_id, event_content, type):
         """Send a notice to the given user
 
         Creates the server notices room, if none exists.
@@ -67,7 +67,7 @@ def send_notice(self, user_id, event_content):
 
         yield self._event_creation_handler.create_and_send_nonmember_event(
             requester, {
-                "type": EventTypes.Message,
+                "type": type,
                 "room_id": room_id,
                 "sender": system_mxid,
                 "content": event_content,

From bdfbd934d6eb42b899779abbe17f72ec4c77e720 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Tue, 14 Aug 2018 20:53:43 +1000
Subject: [PATCH 385/830] Implement a new test baseclass to cut down on
 boilerplate (#3684)

---
 changelog.d/3684.misc               |   1 +
 tests/rest/client/v1/test_typing.py |  66 ++++++-------
 tests/unittest.py                   | 144 ++++++++++++++++++++++++++++
 3 files changed, 174 insertions(+), 37 deletions(-)
 create mode 100644 changelog.d/3684.misc

diff --git a/changelog.d/3684.misc b/changelog.d/3684.misc
new file mode 100644
index 000000000000..4c013263c453
--- /dev/null
+++ b/changelog.d/3684.misc
@@ -0,0 +1 @@
+Implemented a new testing base class to reduce test boilerplate.
diff --git a/tests/rest/client/v1/test_typing.py b/tests/rest/client/v1/test_typing.py
index 677265edf602..0ad814c5e588 100644
--- a/tests/rest/client/v1/test_typing.py
+++ b/tests/rest/client/v1/test_typing.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2018 New Vector
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,41 +18,32 @@
 
 from mock import Mock, NonCallableMock
 
-# twisted imports
 from twisted.internet import defer
 
-import synapse.rest.client.v1.room
+from synapse.rest.client.v1 import room
 from synapse.types import UserID
 
-from ....utils import MockClock, MockHttpResource, setup_test_homeserver
-from .utils import RestTestCase
+from tests import unittest
 
 PATH_PREFIX = "/_matrix/client/api/v1"
 
 
-class RoomTypingTestCase(RestTestCase):
+class RoomTypingTestCase(unittest.HomeserverTestCase):
     """ Tests /rooms/$room_id/typing/$user_id REST API. """
 
     user_id = "@sid:red"
 
     user = UserID.from_string(user_id)
+    servlets = [room.register_servlets]
 
-    @defer.inlineCallbacks
-    def setUp(self):
-        self.clock = MockClock()
+    def make_homeserver(self, reactor, clock):
 
-        self.mock_resource = MockHttpResource(prefix=PATH_PREFIX)
-        self.auth_user_id = self.user_id
-
-        hs = yield setup_test_homeserver(
-            self.addCleanup,
+        hs = self.setup_test_homeserver(
             "red",
-            clock=self.clock,
             http_client=None,
             federation_client=Mock(),
             ratelimiter=NonCallableMock(spec_set=["send_message"]),
         )
-        self.hs = hs
 
         self.event_source = hs.get_event_sources().sources["typing"]
 
@@ -100,25 +92,24 @@ def fetch_room_distributions_into(
             fetch_room_distributions_into
         )
 
-        synapse.rest.client.v1.room.register_servlets(hs, self.mock_resource)
+        return hs
 
-        self.room_id = yield self.create_room_as(self.user_id)
+    def prepare(self, reactor, clock, hs):
+        self.room_id = self.helper.create_room_as(self.user_id)
         # Need another user to make notifications actually work
-        yield self.join(self.room_id, user="@jim:red")
+        self.helper.join(self.room_id, user="@jim:red")
 
-    @defer.inlineCallbacks
     def test_set_typing(self):
-        (code, _) = yield self.mock_resource.trigger(
+        request, channel = self.make_request(
             "PUT",
             "/rooms/%s/typing/%s" % (self.room_id, self.user_id),
-            '{"typing": true, "timeout": 30000}',
+            b'{"typing": true, "timeout": 30000}',
         )
-        self.assertEquals(200, code)
+        self.render(request)
+        self.assertEquals(200, channel.code)
 
         self.assertEquals(self.event_source.get_current_key(), 1)
-        events = yield self.event_source.get_new_events(
-            from_key=0, room_ids=[self.room_id]
-        )
+        events = self.event_source.get_new_events(from_key=0, room_ids=[self.room_id])
         self.assertEquals(
             events[0],
             [
@@ -130,35 +121,36 @@ def test_set_typing(self):
             ],
         )
 
-    @defer.inlineCallbacks
     def test_set_not_typing(self):
-        (code, _) = yield self.mock_resource.trigger(
+        request, channel = self.make_request(
             "PUT",
             "/rooms/%s/typing/%s" % (self.room_id, self.user_id),
-            '{"typing": false}',
+            b'{"typing": false}',
         )
-        self.assertEquals(200, code)
+        self.render(request)
+        self.assertEquals(200, channel.code)
 
-    @defer.inlineCallbacks
     def test_typing_timeout(self):
-        (code, _) = yield self.mock_resource.trigger(
+        request, channel = self.make_request(
             "PUT",
             "/rooms/%s/typing/%s" % (self.room_id, self.user_id),
-            '{"typing": true, "timeout": 30000}',
+            b'{"typing": true, "timeout": 30000}',
         )
-        self.assertEquals(200, code)
+        self.render(request)
+        self.assertEquals(200, channel.code)
 
         self.assertEquals(self.event_source.get_current_key(), 1)
 
-        self.clock.advance_time(36)
+        self.reactor.advance(36)
 
         self.assertEquals(self.event_source.get_current_key(), 2)
 
-        (code, _) = yield self.mock_resource.trigger(
+        request, channel = self.make_request(
             "PUT",
             "/rooms/%s/typing/%s" % (self.room_id, self.user_id),
-            '{"typing": true, "timeout": 30000}',
+            b'{"typing": true, "timeout": 30000}',
         )
-        self.assertEquals(200, code)
+        self.render(request)
+        self.assertEquals(200, channel.code)
 
         self.assertEquals(self.event_source.get_current_key(), 3)
diff --git a/tests/unittest.py b/tests/unittest.py
index f448a6dfbd40..e6afe3b96d61 100644
--- a/tests/unittest.py
+++ b/tests/unittest.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2018 New Vector
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,12 +16,19 @@
 
 import logging
 
+from mock import Mock
+
 import twisted
 import twisted.logger
 from twisted.trial import unittest
 
+from synapse.http.server import JsonResource
+from synapse.server import HomeServer
+from synapse.types import UserID, create_requester
 from synapse.util.logcontext import LoggingContextFilter
 
+from tests.server import get_clock, make_request, render, setup_test_homeserver
+
 # Set up putting Synapse's logs into Trial's.
 rootLogger = logging.getLogger()
 
@@ -129,3 +137,139 @@ def DEBUG(target):
     Can apply to either a TestCase or an individual test method."""
     target.loglevel = logging.DEBUG
     return target
+
+
+class HomeserverTestCase(TestCase):
+    """
+    A base TestCase that reduces boilerplate for HomeServer-using test cases.
+
+    Attributes:
+        servlets (list[function]): List of servlet registration function.
+        user_id (str): The user ID to assume if auth is hijacked.
+        hijack_auth (bool): Whether to hijack auth to return the user specified
+        in user_id.
+    """
+    servlets = []
+    hijack_auth = True
+
+    def setUp(self):
+        """
+        Set up the TestCase by calling the homeserver constructor, optionally
+        hijacking the authentication system to return a fixed user, and then
+        calling the prepare function.
+        """
+        self.reactor, self.clock = get_clock()
+        self._hs_args = {"clock": self.clock, "reactor": self.reactor}
+        self.hs = self.make_homeserver(self.reactor, self.clock)
+
+        if self.hs is None:
+            raise Exception("No homeserver returned from make_homeserver.")
+
+        if not isinstance(self.hs, HomeServer):
+            raise Exception("A homeserver wasn't returned, but %r" % (self.hs,))
+
+        # Register the resources
+        self.resource = JsonResource(self.hs)
+
+        for servlet in self.servlets:
+            servlet(self.hs, self.resource)
+
+        if hasattr(self, "user_id"):
+            from tests.rest.client.v1.utils import RestHelper
+
+            self.helper = RestHelper(self.hs, self.resource, self.user_id)
+
+            if self.hijack_auth:
+
+                def get_user_by_access_token(token=None, allow_guest=False):
+                    return {
+                        "user": UserID.from_string(self.helper.auth_user_id),
+                        "token_id": 1,
+                        "is_guest": False,
+                    }
+
+                def get_user_by_req(request, allow_guest=False, rights="access"):
+                    return create_requester(
+                        UserID.from_string(self.helper.auth_user_id), 1, False, None
+                    )
+
+                self.hs.get_auth().get_user_by_req = get_user_by_req
+                self.hs.get_auth().get_user_by_access_token = get_user_by_access_token
+                self.hs.get_auth().get_access_token_from_request = Mock(
+                    return_value="1234"
+                )
+
+        if hasattr(self, "prepare"):
+            self.prepare(self.reactor, self.clock, self.hs)
+
+    def make_homeserver(self, reactor, clock):
+        """
+        Make and return a homeserver.
+
+        Args:
+            reactor: A Twisted Reactor, or something that pretends to be one.
+            clock (synapse.util.Clock): The Clock, associated with the reactor.
+
+        Returns:
+            A homeserver (synapse.server.HomeServer) suitable for testing.
+
+        Function to be overridden in subclasses.
+        """
+        raise NotImplementedError()
+
+    def prepare(self, reactor, clock, homeserver):
+        """
+        Prepare for the test.  This involves things like mocking out parts of
+        the homeserver, or building test data common across the whole test
+        suite.
+
+        Args:
+            reactor: A Twisted Reactor, or something that pretends to be one.
+            clock (synapse.util.Clock): The Clock, associated with the reactor.
+            homeserver (synapse.server.HomeServer): The HomeServer to test
+            against.
+
+        Function to optionally be overridden in subclasses.
+        """
+
+    def make_request(self, method, path, content=b""):
+        """
+        Create a SynapseRequest at the path using the method and containing the
+        given content.
+
+        Args:
+            method (bytes/unicode): The HTTP request method ("verb").
+            path (bytes/unicode): The HTTP path, suitably URL encoded (e.g.
+            escaped UTF-8 & spaces and such).
+            content (bytes): The body of the request.
+
+        Returns:
+            A synapse.http.site.SynapseRequest.
+        """
+        return make_request(method, path, content)
+
+    def render(self, request):
+        """
+        Render a request against the resources registered by the test class's
+        servlets.
+
+        Args:
+            request (synapse.http.site.SynapseRequest): The request to render.
+        """
+        render(request, self.resource, self.reactor)
+
+    def setup_test_homeserver(self, *args, **kwargs):
+        """
+        Set up the test homeserver, meant to be called by the overridable
+        make_homeserver. It automatically passes through the test class's
+        clock & reactor.
+
+        Args:
+            See tests.utils.setup_test_homeserver.
+
+        Returns:
+            synapse.server.HomeServer
+        """
+        kwargs = dict(kwargs)
+        kwargs.update(self._hs_args)
+        return setup_test_homeserver(self.addCleanup, *args, **kwargs)

From 614e6d517d90922cbc54e59d0117a4daaed9a9fc Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Tue, 14 Aug 2018 21:30:34 +1000
Subject: [PATCH 386/830] Dockerised sytest (#3660)

---
 .circleci/config.yml  | 48 +++++++++++++++++++++++++++++++++++++++++++
 .dockerignore         |  3 +++
 MANIFEST.in           |  1 +
 changelog.d/3660.misc |  1 +
 4 files changed, 53 insertions(+)
 create mode 100644 .circleci/config.yml
 create mode 100644 changelog.d/3660.misc

diff --git a/.circleci/config.yml b/.circleci/config.yml
new file mode 100644
index 000000000000..e03f01b83703
--- /dev/null
+++ b/.circleci/config.yml
@@ -0,0 +1,48 @@
+version: 2
+jobs:
+  sytestpy2:
+    machine: true
+    steps:
+      - checkout
+      - run: docker pull matrixdotorg/sytest-synapsepy2
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy2
+      - store_artifacts:
+          path: ~/project/logs
+          destination: logs
+  sytestpy2postgres:
+    machine: true
+    steps:
+      - checkout
+      - run: docker pull matrixdotorg/sytest-synapsepy2
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy2
+      - store_artifacts:
+          path: ~/project/logs
+          destination: logs
+  sytestpy3:
+    machine: true
+    steps:
+      - checkout
+      - run: docker pull matrixdotorg/sytest-synapsepy3
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs hawkowl/sytestpy3
+      - store_artifacts:
+          path: ~/project/logs
+          destination: logs
+  sytestpy3postgres:
+    machine: true
+    steps:
+      - checkout
+      - run: docker pull matrixdotorg/sytest-synapsepy3
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy3
+      - store_artifacts:
+          path: ~/project/logs
+          destination: logs
+
+workflows:
+  version: 2
+  build:
+    jobs:
+      - sytestpy2
+      - sytestpy2postgres
+# Currently broken while the Python 3 port is incomplete
+#      - sytestpy3
+#      - sytestpy3postgres
diff --git a/.dockerignore b/.dockerignore
index f36f86fbb722..6cdb8532d332 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -3,3 +3,6 @@ Dockerfile
 .gitignore
 demo/etc
 tox.ini
+synctl
+.git/*
+.tox/*
diff --git a/MANIFEST.in b/MANIFEST.in
index 1ff98d95dff0..e0826ba544c8 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -36,3 +36,4 @@ recursive-include changelog.d *
 prune .github
 prune demo/etc
 prune docker
+prune .circleci
diff --git a/changelog.d/3660.misc b/changelog.d/3660.misc
new file mode 100644
index 000000000000..acd814c27391
--- /dev/null
+++ b/changelog.d/3660.misc
@@ -0,0 +1 @@
+Sytests can now be run inside a Docker container.

From e2c9fe0a6a6dbe4f81402b79b6605e17fb05e5d4 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Tue, 14 Aug 2018 13:32:56 +0100
Subject: [PATCH 387/830] backout ability to pass in event type to server
 notices

---
 synapse/server_notices/consent_server_notices.py         | 2 +-
 synapse/server_notices/resource_limits_server_notices.py | 3 ++-
 synapse/server_notices/server_notices_manager.py         | 4 ++--
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/synapse/server_notices/consent_server_notices.py b/synapse/server_notices/consent_server_notices.py
index 6d5caedb084e..783ceae1ad27 100644
--- a/synapse/server_notices/consent_server_notices.py
+++ b/synapse/server_notices/consent_server_notices.py
@@ -104,7 +104,7 @@ def maybe_send_server_notice_to_user(self, user_id):
                     },
                 )
                 yield self._server_notices_manager.send_notice(
-                    user_id, content, EventTypes.Message
+                    user_id, content
                 )
                 yield self._store.user_set_consent_server_notice_sent(
                     user_id, self._current_consent_version,
diff --git a/synapse/server_notices/resource_limits_server_notices.py b/synapse/server_notices/resource_limits_server_notices.py
index 2f49dae16822..0470f6620574 100644
--- a/synapse/server_notices/resource_limits_server_notices.py
+++ b/synapse/server_notices/resource_limits_server_notices.py
@@ -91,6 +91,7 @@ def maybe_send_server_notice_to_user(self, user_id):
                     self._send_server_notice(user_id, content)
                     self._notified_of_blocking.add(user_id)
 
+
     @defer.inlineCallbacks
     def _send_server_notice(self, user_id, content):
         """Sends Server notice
@@ -104,7 +105,7 @@ def _send_server_notice(self, user_id, content):
         """
         try:
             yield self._server_notices_manager.send_notice(
-                user_id, content, EventTypes.ServerNoticeLimitReached
+                user_id, content
             )
         except SynapseError as e:
             logger.error("Error sending server notice about resource limits: %s", e)
diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py
index 4806977a8933..a26deace5318 100644
--- a/synapse/server_notices/server_notices_manager.py
+++ b/synapse/server_notices/server_notices_manager.py
@@ -46,7 +46,7 @@ def is_enabled(self):
         return self._config.server_notices_mxid is not None
 
     @defer.inlineCallbacks
-    def send_notice(self, user_id, event_content, type):
+    def send_notice(self, user_id, event_content):
         """Send a notice to the given user
 
         Creates the server notices room, if none exists.
@@ -67,7 +67,7 @@ def send_notice(self, user_id, event_content, type):
 
         yield self._event_creation_handler.create_and_send_nonmember_event(
             requester, {
-                "type": type,
+                "type": EventTypes.Message,
                 "room_id": room_id,
                 "sender": system_mxid,
                 "content": event_content,

From 99ebaed8e6c5a65872a91f7a50e914ad9e574f7f Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Tue, 14 Aug 2018 14:55:55 +0100
Subject: [PATCH 388/830] Update register.py

remove comments
---
 synapse/handlers/register.py | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index ef7222d7b860..54e343492854 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -535,13 +535,3 @@ def _join_user_to_room(self, requester, room_identifier):
             action="join",
         )
 
-    # @defer.inlineCallbacks
-    # def _s(self):
-    #     """
-    #     Do not accept registrations if monthly active user limits exceeded
-    #      and limiting is enabled
-    #     """
-    #     try:
-    #         yield self.auth.check_auth_blocking()
-    #     except AuthError as e:
-    #         raise RegistrationError(e.code, str(e), e.errcode)

From ed4bc3d2fc7242e27b3cdd36bc6c27c98fac09c8 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Tue, 14 Aug 2018 15:04:48 +0100
Subject: [PATCH 389/830] fix off by 1s on mau

---
 synapse/handlers/auth.py        |  4 ++--
 tests/handlers/test_auth.py     | 39 ++++++++++++++++++++++++++++++++-
 tests/handlers/test_register.py | 14 ++++++++----
 3 files changed, 50 insertions(+), 7 deletions(-)

diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py
index 7ea8ce9f9479..7baaa394471f 100644
--- a/synapse/handlers/auth.py
+++ b/synapse/handlers/auth.py
@@ -520,7 +520,7 @@ def get_access_token_for_user_id(self, user_id, device_id=None):
         """
         logger.info("Logging in user %s on device %s", user_id, device_id)
         access_token = yield self.issue_access_token(user_id, device_id)
-        yield self.auth.check_auth_blocking()
+        yield self.auth.check_auth_blocking(user_id)
 
         # the device *should* have been registered before we got here; however,
         # it's possible we raced against a DELETE operation. The thing we
@@ -734,7 +734,6 @@ def issue_access_token(self, user_id, device_id=None):
 
     @defer.inlineCallbacks
     def validate_short_term_login_token_and_get_user_id(self, login_token):
-        yield self.auth.check_auth_blocking()
         auth_api = self.hs.get_auth()
         user_id = None
         try:
@@ -743,6 +742,7 @@ def validate_short_term_login_token_and_get_user_id(self, login_token):
             auth_api.validate_macaroon(macaroon, "login", True, user_id)
         except Exception:
             raise AuthError(403, "Invalid token", errcode=Codes.FORBIDDEN)
+        yield self.auth.check_auth_blocking(user_id)
         defer.returnValue(user_id)
 
     @defer.inlineCallbacks
diff --git a/tests/handlers/test_auth.py b/tests/handlers/test_auth.py
index 56c0f87fb772..9ca7b2ee4ea6 100644
--- a/tests/handlers/test_auth.py
+++ b/tests/handlers/test_auth.py
@@ -124,7 +124,7 @@ def test_mau_limits_disabled(self):
         )
 
     @defer.inlineCallbacks
-    def test_mau_limits_exceeded(self):
+    def test_mau_limits_exceeded_large(self):
         self.hs.config.limit_usage_by_mau = True
         self.hs.get_datastore().get_monthly_active_count = Mock(
             return_value=defer.succeed(self.large_number_of_users)
@@ -141,6 +141,43 @@ def test_mau_limits_exceeded(self):
                 self._get_macaroon().serialize()
             )
 
+    @defer.inlineCallbacks
+    def test_mau_limits_parity(self):
+        self.hs.config.limit_usage_by_mau = True
+
+        # If not in monthly active cohort
+        self.hs.get_datastore().get_monthly_active_count = Mock(
+            return_value=defer.succeed(self.hs.config.max_mau_value)
+        )
+        with self.assertRaises(AuthError):
+            yield self.auth_handler.get_access_token_for_user_id('user_a')
+
+        self.hs.get_datastore().get_monthly_active_count = Mock(
+            return_value=defer.succeed(self.hs.config.max_mau_value)
+        )
+        with self.assertRaises(AuthError):
+            yield self.auth_handler.validate_short_term_login_token_and_get_user_id(
+                self._get_macaroon().serialize()
+            )
+        # If in monthly active cohort
+        self.hs.get_datastore().user_last_seen_monthly_active = Mock(
+            return_value=defer.succeed(self.hs.get_clock().time_msec())
+        )
+        self.hs.get_datastore().get_monthly_active_count = Mock(
+            return_value=defer.succeed(self.hs.config.max_mau_value)
+        )
+        yield self.auth_handler.get_access_token_for_user_id('user_a')
+        self.hs.get_datastore().user_last_seen_monthly_active = Mock(
+            return_value=defer.succeed(self.hs.get_clock().time_msec())
+        )
+        self.hs.get_datastore().get_monthly_active_count = Mock(
+            return_value=defer.succeed(self.hs.config.max_mau_value)
+        )
+        yield self.auth_handler.validate_short_term_login_token_and_get_user_id(
+            self._get_macaroon().serialize()
+        )
+
+
     @defer.inlineCallbacks
     def test_mau_limits_not_exceeded(self):
         self.hs.config.limit_usage_by_mau = True
diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py
index 35d1bcab3e9f..a821da075035 100644
--- a/tests/handlers/test_register.py
+++ b/tests/handlers/test_register.py
@@ -17,7 +17,7 @@
 
 from twisted.internet import defer
 
-from synapse.api.errors import AuthError
+from synapse.api.errors import RegistrationError
 from synapse.handlers.register import RegistrationHandler
 from synapse.types import UserID, create_requester
 
@@ -109,7 +109,7 @@ def test_get_or_create_user_mau_blocked(self):
         self.store.get_monthly_active_count = Mock(
             return_value=defer.succeed(self.lots_of_users)
         )
-        with self.assertRaises(AuthError):
+        with self.assertRaises(RegistrationError):
             yield self.handler.get_or_create_user("requester", 'b', "display_name")
 
     @defer.inlineCallbacks
@@ -118,7 +118,13 @@ def test_register_mau_blocked(self):
         self.store.get_monthly_active_count = Mock(
             return_value=defer.succeed(self.lots_of_users)
         )
-        with self.assertRaises(AuthError):
+        with self.assertRaises(RegistrationError):
+            yield self.handler.register(localpart="local_part")
+
+        self.store.get_monthly_active_count = Mock(
+            return_value=defer.succeed(self.hs.config.max_mau_value)
+        )
+        with self.assertRaises(RegistrationError):
             yield self.handler.register(localpart="local_part")
 
     @defer.inlineCallbacks
@@ -127,5 +133,5 @@ def test_register_saml2_mau_blocked(self):
         self.store.get_monthly_active_count = Mock(
             return_value=defer.succeed(self.lots_of_users)
         )
-        with self.assertRaises(AuthError):
+        with self.assertRaises(RegistrationError):
             yield self.handler.register_saml2(localpart="local_part")

From c74c71128d164ce7aa6b50538ab960ec85f7a8da Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Tue, 14 Aug 2018 15:06:24 +0100
Subject: [PATCH 390/830] remove blank line

---
 synapse/handlers/register.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index 54e343492854..f03ee1476bac 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -534,4 +534,3 @@ def _join_user_to_room(self, requester, room_identifier):
             remote_room_hosts=remote_room_hosts,
             action="join",
         )
-

From 8f9a7eb58de214cd489ba233c381521e9bf79dec Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Mon, 13 Aug 2018 18:00:23 +0100
Subject: [PATCH 391/830] support admin_email config and pass through into
 blocking errors, return AuthError in all cases

---
 synapse/handlers/register.py    | 1 -
 tests/handlers/test_register.py | 8 ++++----
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index 54e343492854..f03ee1476bac 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -534,4 +534,3 @@ def _join_user_to_room(self, requester, room_identifier):
             remote_room_hosts=remote_room_hosts,
             action="join",
         )
-
diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py
index a821da075035..6699d251217e 100644
--- a/tests/handlers/test_register.py
+++ b/tests/handlers/test_register.py
@@ -17,7 +17,7 @@
 
 from twisted.internet import defer
 
-from synapse.api.errors import RegistrationError
+from synapse.api.errors import AuthError
 from synapse.handlers.register import RegistrationHandler
 from synapse.types import UserID, create_requester
 
@@ -109,7 +109,7 @@ def test_get_or_create_user_mau_blocked(self):
         self.store.get_monthly_active_count = Mock(
             return_value=defer.succeed(self.lots_of_users)
         )
-        with self.assertRaises(RegistrationError):
+        with self.assertRaises(AuthError):
             yield self.handler.get_or_create_user("requester", 'b', "display_name")
 
     @defer.inlineCallbacks
@@ -118,7 +118,7 @@ def test_register_mau_blocked(self):
         self.store.get_monthly_active_count = Mock(
             return_value=defer.succeed(self.lots_of_users)
         )
-        with self.assertRaises(RegistrationError):
+        with self.assertRaises(AuthError):
             yield self.handler.register(localpart="local_part")
 
         self.store.get_monthly_active_count = Mock(
@@ -133,5 +133,5 @@ def test_register_saml2_mau_blocked(self):
         self.store.get_monthly_active_count = Mock(
             return_value=defer.succeed(self.lots_of_users)
         )
-        with self.assertRaises(RegistrationError):
+        with self.assertRaises(AuthError):
             yield self.handler.register_saml2(localpart="local_part")

From 06b331ff4035558500e196a5dce79ffe9d2da807 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Tue, 14 Aug 2018 15:28:15 +0100
Subject: [PATCH 392/830] fix off by 1 errors

---
 tests/handlers/test_auth.py     |  1 -
 tests/handlers/test_register.py | 16 ++++++++++++++--
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/tests/handlers/test_auth.py b/tests/handlers/test_auth.py
index 9ca7b2ee4ea6..3046bd609365 100644
--- a/tests/handlers/test_auth.py
+++ b/tests/handlers/test_auth.py
@@ -177,7 +177,6 @@ def test_mau_limits_parity(self):
             self._get_macaroon().serialize()
         )
 
-
     @defer.inlineCallbacks
     def test_mau_limits_not_exceeded(self):
         self.hs.config.limit_usage_by_mau = True
diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py
index 6699d251217e..7154816a34a7 100644
--- a/tests/handlers/test_register.py
+++ b/tests/handlers/test_register.py
@@ -98,7 +98,7 @@ def test_mau_limits_when_disabled(self):
     def test_get_or_create_user_mau_not_blocked(self):
         self.hs.config.limit_usage_by_mau = True
         self.store.count_monthly_users = Mock(
-            return_value=defer.succeed(self.small_number_of_users)
+            return_value=defer.succeed(self.hs.config.max_mau_value - 1)
         )
         # Ensure does not throw exception
         yield self.handler.get_or_create_user("@user:server", 'c', "User")
@@ -112,6 +112,12 @@ def test_get_or_create_user_mau_blocked(self):
         with self.assertRaises(AuthError):
             yield self.handler.get_or_create_user("requester", 'b', "display_name")
 
+        self.store.get_monthly_active_count = Mock(
+            return_value=defer.succeed(self.hs.config.max_mau_value)
+        )
+        with self.assertRaises(AuthError):
+            yield self.handler.get_or_create_user("requester", 'b', "display_name")
+
     @defer.inlineCallbacks
     def test_register_mau_blocked(self):
         self.hs.config.limit_usage_by_mau = True
@@ -124,7 +130,7 @@ def test_register_mau_blocked(self):
         self.store.get_monthly_active_count = Mock(
             return_value=defer.succeed(self.hs.config.max_mau_value)
         )
-        with self.assertRaises(RegistrationError):
+        with self.assertRaises(AuthError):
             yield self.handler.register(localpart="local_part")
 
     @defer.inlineCallbacks
@@ -135,3 +141,9 @@ def test_register_saml2_mau_blocked(self):
         )
         with self.assertRaises(AuthError):
             yield self.handler.register_saml2(localpart="local_part")
+
+        self.store.get_monthly_active_count = Mock(
+            return_value=defer.succeed(self.hs.config.max_mau_value)
+        )
+        with self.assertRaises(AuthError):
+            yield self.handler.register_saml2(localpart="local_part")

From 2545993ce430606f76926723898c2de3840da94a Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Tue, 14 Aug 2018 15:48:12 +0100
Subject: [PATCH 393/830] make comments clearer

---
 synapse/api/auth.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index c31c6a6a0867..18c73f054929 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -780,7 +780,8 @@ def check_auth_blocking(self, user_id=None):
         such as monthly active user limiting or global disable flag
 
         Args:
-            user_id(str): If present, checks for presence against existing MAU cohort
+            user_id(str|None): If present, checks for presence against existing
+            MAU cohort
         """
         if self.hs.config.hs_disabled:
             raise AuthError(

From e7d091fb867ade42889af7b8f9ce6a94a3d5e85e Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Tue, 14 Aug 2018 16:26:55 +0100
Subject: [PATCH 394/830] combine mau metrics into one group

---
 synapse/app/homeserver.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 37a9b126a5c1..f13584e190c3 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -303,9 +303,8 @@ def run_startup_checks(self, db_conn, database_engine):
 
 
 # Gauges to expose monthly active user control metrics
-current_mau_gauge = Gauge("synapse_admin_current_mau", "Current MAU")
-max_mau_value_gauge = Gauge("synapse_admin_max_mau_value", "MAU Limit")
-
+current_mau_gauge = Gauge("synapse_admin_mau:current", "Current MAU")
+max_mau_gauge = Gauge("synapse_admin_mau:max", "MAU Limit")
 
 def setup(config_options):
     """
@@ -532,7 +531,7 @@ def generate_monthly_active_users():
         if hs.config.limit_usage_by_mau:
             count = yield hs.get_datastore().get_monthly_active_count()
         current_mau_gauge.set(float(count))
-        max_mau_value_gauge.set(float(hs.config.max_mau_value))
+        max_mau_gauge.set(float(hs.config.max_mau_value))
 
     hs.get_datastore().initialise_reserved_users(
         hs.config.mau_limits_reserved_threepids

From e5962f845c2e93564a2855696e0776bd05c5f126 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Tue, 14 Aug 2018 16:36:14 +0100
Subject: [PATCH 395/830] pep8

---
 synapse/app/homeserver.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index f13584e190c3..a98bb506e5fb 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -306,6 +306,7 @@ def run_startup_checks(self, db_conn, database_engine):
 current_mau_gauge = Gauge("synapse_admin_mau:current", "Current MAU")
 max_mau_gauge = Gauge("synapse_admin_mau:max", "MAU Limit")
 
+
 def setup(config_options):
     """
     Args:

From 629f390035f2c1eb7c9d5e4a881e2586a9fdc3ab Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Tue, 14 Aug 2018 16:38:05 +0100
Subject: [PATCH 396/830] Rename MAU prometheus metric

---
 changelog.d/3690.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3690.misc

diff --git a/changelog.d/3690.misc b/changelog.d/3690.misc
new file mode 100644
index 000000000000..710add02433d
--- /dev/null
+++ b/changelog.d/3690.misc
@@ -0,0 +1 @@
+Rename MAU prometheus metrics

From 1522ed9c07b11a97064a6642c6fd6c3e786f9cb5 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Tue, 14 Aug 2018 16:52:30 +0100
Subject: [PATCH 397/830] in case max_mau is less than I think

---
 tests/storage/test_monthly_active_users.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/storage/test_monthly_active_users.py b/tests/storage/test_monthly_active_users.py
index 0d58dcebc52c..511acbde9bc2 100644
--- a/tests/storage/test_monthly_active_users.py
+++ b/tests/storage/test_monthly_active_users.py
@@ -33,7 +33,7 @@ def setUp(self):
 
     @defer.inlineCallbacks
     def test_initialise_reserved_users(self):
-
+        self.hs.config.max_mau_value = 5
         user1 = "@user1:server"
         user1_email = "user1@matrix.org"
         user2 = "@user2:server"

From 9ecbaf8ba8b12537862e3045a650fffd438ba8a5 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Tue, 14 Aug 2018 16:55:28 +0100
Subject: [PATCH 398/830] adding missing yield

---
 synapse/storage/monthly_active_users.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py
index 7b36accdb6ca..7e417f811e9e 100644
--- a/synapse/storage/monthly_active_users.py
+++ b/synapse/storage/monthly_active_users.py
@@ -46,7 +46,7 @@ def initialise_reserved_users(self, threepids):
                 tp["medium"], tp["address"]
             )
             if user_id:
-                self.upsert_monthly_active_user(user_id)
+                yield self.upsert_monthly_active_user(user_id)
                 reserved_user_list.append(user_id)
             else:
                 logger.warning(

From cd0c749c4f86e9350be1152a91958d0064d41c84 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Tue, 14 Aug 2018 17:06:51 +0100
Subject: [PATCH 399/830] Fix missing yield in
 synapse.storage.monthly_active_users.initialise_reserved_users

---
 changelog.d/3692.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3692.bugfix

diff --git a/changelog.d/3692.bugfix b/changelog.d/3692.bugfix
new file mode 100644
index 000000000000..f44e13dca154
--- /dev/null
+++ b/changelog.d/3692.bugfix
@@ -0,0 +1 @@
+Fix missing yield in synapse.storage.monthly_active_users.initialise_reserved_users

From 7277216d01261f055886d0ac7b1ae5e5c5fc33cf Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Tue, 14 Aug 2018 17:14:39 +0100
Subject: [PATCH 400/830] fix setup_test_homeserver to be postgres compatible

---
 tests/handlers/test_sync.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py
index cfd37f3138d3..8c8b65e04ede 100644
--- a/tests/handlers/test_sync.py
+++ b/tests/handlers/test_sync.py
@@ -29,7 +29,7 @@ class SyncTestCase(tests.unittest.TestCase):
 
     @defer.inlineCallbacks
     def setUp(self):
-        self.hs = yield setup_test_homeserver()
+        self.hs = yield setup_test_homeserver(self.addCleanup)
         self.sync_handler = SyncHandler(self.hs)
         self.store = self.hs.get_datastore()
 

From ab035bdeaca5019289e6587a625e87c995d3deeb Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 15 Aug 2018 10:16:41 +0100
Subject: [PATCH 401/830] replace admin_email with admin_uri for greater
 flexibility

---
 synapse/api/auth.py      | 10 +++++-----
 synapse/api/errors.py    |  4 ++--
 synapse/config/server.py |  2 +-
 tests/api/test_auth.py   |  4 ++--
 tests/utils.py           |  2 +-
 5 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 108ea0ea093f..3b2a2ab77ade 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -786,8 +786,8 @@ def check_auth_blocking(self, user_id=None):
         if self.hs.config.hs_disabled:
             raise AuthError(
                 403, self.hs.config.hs_disabled_message,
-                errcode=Codes.HS_DISABLED,
-                admin_email=self.hs.config.admin_email,
+                errcode=Codes.RESOURCE_LIMIT_EXCEED,
+                admin_uri=self.hs.config.admin_uri,
             )
         if self.hs.config.limit_usage_by_mau is True:
             # If the user is already part of the MAU cohort
@@ -799,7 +799,7 @@ def check_auth_blocking(self, user_id=None):
             current_mau = yield self.store.get_monthly_active_count()
             if current_mau >= self.hs.config.max_mau_value:
                 raise AuthError(
-                    403, "MAU Limit Exceeded",
-                    admin_email=self.hs.config.admin_email,
-                    errcode=Codes.MAU_LIMIT_EXCEEDED
+                    403, "Monthly Active User Limits AU Limit Exceeded",
+                    admin_uri=self.hs.config.admin_uri,
+                    errcode=Codes.RESOURCE_LIMIT_EXCEED
                 )
diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index d74848159e44..b677087e7360 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -228,7 +228,7 @@ class AuthError(SynapseError):
     def __init__(self, *args, **kwargs):
         if "errcode" not in kwargs:
             kwargs["errcode"] = Codes.FORBIDDEN
-        self.admin_email = kwargs.get('admin_email')
+        self.admin_uri = kwargs.get('admin_uri')
         self.msg = kwargs.get('msg')
         self.errcode = kwargs.get('errcode')
         super(AuthError, self).__init__(*args, errcode=kwargs["errcode"])
@@ -237,7 +237,7 @@ def error_dict(self):
         return cs_error(
             self.msg,
             self.errcode,
-            admin_email=self.admin_email,
+            admin_uri=self.admin_uri,
         )
 
 
diff --git a/synapse/config/server.py b/synapse/config/server.py
index 64a5121a452f..442ee4fde441 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -84,7 +84,7 @@ def read_config(self, config):
 
         # Admin email to direct users at should their instance become blocked
         # due to resource constraints
-        self.admin_email = config.get("admin_email", None)
+        self.admin_uri = config.get("admin_uri", None)
 
         # FIXME: federation_domain_whitelist needs sytests
         self.federation_domain_whitelist = None
diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py
index e8a1894e651c..8c1ee9cdcc5c 100644
--- a/tests/api/test_auth.py
+++ b/tests/api/test_auth.py
@@ -457,7 +457,7 @@ def test_blocking_mau(self):
 
         with self.assertRaises(AuthError) as e:
             yield self.auth.check_auth_blocking()
-        self.assertEquals(e.exception.admin_email, self.hs.config.admin_email)
+        self.assertEquals(e.exception.admin_uri, self.hs.config.admin_uri)
         self.assertEquals(e.exception.errcode, Codes.MAU_LIMIT_EXCEEDED)
         self.assertEquals(e.exception.code, 403)
 
@@ -473,6 +473,6 @@ def test_hs_disabled(self):
         self.hs.config.hs_disabled_message = "Reason for being disabled"
         with self.assertRaises(AuthError) as e:
             yield self.auth.check_auth_blocking()
-        self.assertEquals(e.exception.admin_email, self.hs.config.admin_email)
+        self.assertEquals(e.exception.admin_uri, self.hs.config.admin_uri)
         self.assertEquals(e.exception.errcode, Codes.HS_DISABLED)
         self.assertEquals(e.exception.code, 403)
diff --git a/tests/utils.py b/tests/utils.py
index 4af81624eb39..52326d4f670b 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -139,7 +139,7 @@ def setup_test_homeserver(
         config.hs_disabled_message = ""
         config.max_mau_value = 50
         config.mau_limits_reserved_threepids = []
-        config.admin_email = None
+        config.admin_uri = None
 
         # we need a sane default_room_version, otherwise attempts to create rooms will
         # fail.

From b8429c7c81f96fdf24f8e297e788fd2b8b3a712a Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 15 Aug 2018 10:19:48 +0100
Subject: [PATCH 402/830] update error codes for resource limiting

---
 synapse/api/errors.py  | 3 +--
 tests/api/test_auth.py | 4 ++--
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index b677087e7360..10b0356e86f5 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -56,8 +56,7 @@ class Codes(object):
     SERVER_NOT_TRUSTED = "M_SERVER_NOT_TRUSTED"
     CONSENT_NOT_GIVEN = "M_CONSENT_NOT_GIVEN"
     CANNOT_LEAVE_SERVER_NOTICE_ROOM = "M_CANNOT_LEAVE_SERVER_NOTICE_ROOM"
-    MAU_LIMIT_EXCEEDED = "M_MAU_LIMIT_EXCEEDED"
-    HS_DISABLED = "M_HS_DISABLED"
+    RESOURCE_LIMIT_EXCEED = "M_RESOURCE_LIMIT_EXCEED"
     UNSUPPORTED_ROOM_VERSION = "M_UNSUPPORTED_ROOM_VERSION"
     INCOMPATIBLE_ROOM_VERSION = "M_INCOMPATIBLE_ROOM_VERSION"
 
diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py
index 8c1ee9cdcc5c..32a2b5fc3d0f 100644
--- a/tests/api/test_auth.py
+++ b/tests/api/test_auth.py
@@ -458,7 +458,7 @@ def test_blocking_mau(self):
         with self.assertRaises(AuthError) as e:
             yield self.auth.check_auth_blocking()
         self.assertEquals(e.exception.admin_uri, self.hs.config.admin_uri)
-        self.assertEquals(e.exception.errcode, Codes.MAU_LIMIT_EXCEEDED)
+        self.assertEquals(e.exception.errcode, Codes.RESOURCE_LIMIT_EXCEED)
         self.assertEquals(e.exception.code, 403)
 
         # Ensure does not throw an error
@@ -474,5 +474,5 @@ def test_hs_disabled(self):
         with self.assertRaises(AuthError) as e:
             yield self.auth.check_auth_blocking()
         self.assertEquals(e.exception.admin_uri, self.hs.config.admin_uri)
-        self.assertEquals(e.exception.errcode, Codes.HS_DISABLED)
+        self.assertEquals(e.exception.errcode, Codes.RESOURCE_LIMIT_EXCEED)
         self.assertEquals(e.exception.code, 403)

From 596ce635764667f1d824eec2cade73ca6063b73e Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 15 Aug 2018 10:23:28 +0100
Subject: [PATCH 403/830] update resource limit error codes

---
 changelog.d/3697.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3697.misc

diff --git a/changelog.d/3697.misc b/changelog.d/3697.misc
new file mode 100644
index 000000000000..9082fc466285
--- /dev/null
+++ b/changelog.d/3697.misc
@@ -0,0 +1 @@
+update resource limit error codes

From 75c663c7b97733cf1e217d0a973a6c4c64228444 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 15 Aug 2018 11:27:48 +0100
Subject: [PATCH 404/830] update error codes

---
 tests/handlers/test_sync.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py
index 8c8b65e04ede..33d861bd64ae 100644
--- a/tests/handlers/test_sync.py
+++ b/tests/handlers/test_sync.py
@@ -51,7 +51,7 @@ def test_wait_for_sync_for_user_auth_blocking(self):
         self.hs.config.hs_disabled = True
         with self.assertRaises(AuthError) as e:
             yield self.sync_handler.wait_for_sync_for_user(sync_config)
-        self.assertEquals(e.exception.errcode, Codes.HS_DISABLED)
+        self.assertEquals(e.exception.errcode, Codes.RESOURCE_LIMIT_EXCEED)
 
         self.hs.config.hs_disabled = False
 
@@ -59,7 +59,7 @@ def test_wait_for_sync_for_user_auth_blocking(self):
 
         with self.assertRaises(AuthError) as e:
             yield self.sync_handler.wait_for_sync_for_user(sync_config)
-        self.assertEquals(e.exception.errcode, Codes.MAU_LIMIT_EXCEEDED)
+        self.assertEquals(e.exception.errcode, Codes.RESOURCE_LIMIT_EXCEED)
 
     def _generate_sync_config(self, user_id):
         return SyncConfig(

From 55afba0fc5164d3be08848668ddec4bba7fba923 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 15 Aug 2018 11:41:18 +0100
Subject: [PATCH 405/830] update admin email to uri

---
 changelog.d/3687.feature | 2 +-
 synapse/config/server.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/changelog.d/3687.feature b/changelog.d/3687.feature
index 93b24d1acb46..64b89f6411ba 100644
--- a/changelog.d/3687.feature
+++ b/changelog.d/3687.feature
@@ -1 +1 @@
-set admin email via config, to be used in error messages where the user should contact the administrator
+set admin uri via config, to be used in error messages where the user should contact the administrator
diff --git a/synapse/config/server.py b/synapse/config/server.py
index 442ee4fde441..2190f3210a73 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -82,7 +82,7 @@ def read_config(self, config):
         self.hs_disabled = config.get("hs_disabled", False)
         self.hs_disabled_message = config.get("hs_disabled_message", "")
 
-        # Admin email to direct users at should their instance become blocked
+        # Admin uri to direct users at should their instance become blocked
         # due to resource constraints
         self.admin_uri = config.get("admin_uri", None)
 

From 87a824bad1c93e8c9cd35d0667ee366c8ea6b41e Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 15 Aug 2018 11:58:03 +0100
Subject: [PATCH 406/830] clean up AuthError

---
 synapse/api/errors.py | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index 10b0356e86f5..08f0cb55548e 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -224,13 +224,9 @@ def __init__(self, msg="Not found", errcode=Codes.NOT_FOUND):
 
 class AuthError(SynapseError):
     """An error raised when there was a problem authorising an event."""
-    def __init__(self, *args, **kwargs):
-        if "errcode" not in kwargs:
-            kwargs["errcode"] = Codes.FORBIDDEN
-        self.admin_uri = kwargs.get('admin_uri')
-        self.msg = kwargs.get('msg')
-        self.errcode = kwargs.get('errcode')
-        super(AuthError, self).__init__(*args, errcode=kwargs["errcode"])
+    def __init__(self, code, msg, errcode=Codes.FORBIDDEN, admin_uri=None):
+        self.admin_uri = admin_uri
+        super(AuthError, self).__init__(code, msg, errcode=errcode)
 
     def error_dict(self):
         return cs_error(

From eaaa2248ffdda0c913a915a5c5f3e7ea5e2aa8c4 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 15 Aug 2018 08:49:59 +0100
Subject: [PATCH 407/830] Refactor request logging code

This commit moves a bunch of the logic for deciding when to log the receipt and
completion of HTTP requests into SynapseRequest, rather than in the request
handling wrappers.

Advantages of this are:
 * we get logs for *all* requests (including OPTIONS and HEADs), rather than
   just those that end up hitting handlers we've remembered to decorate
   correctly.

 * when a request handler wires up a Producer (as the media stuff does
   currently, and as other things will do soon), we log at the point that all
   of the traffic has been sent to the client.
---
 synapse/http/server.py |  70 +++++---------
 synapse/http/site.py   | 201 ++++++++++++++++++++++++++++++++---------
 2 files changed, 179 insertions(+), 92 deletions(-)

diff --git a/synapse/http/server.py b/synapse/http/server.py
index 6dacb31037aa..4e4c02e1700c 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -25,7 +25,7 @@
 
 from twisted.internet import defer
 from twisted.python import failure
-from twisted.web import resource, server
+from twisted.web import resource
 from twisted.web.server import NOT_DONE_YET
 from twisted.web.util import redirectTo
 
@@ -37,10 +37,8 @@
     SynapseError,
     UnrecognizedRequestError,
 )
-from synapse.http.request_metrics import requests_counter
 from synapse.util.caches import intern_dict
-from synapse.util.logcontext import LoggingContext, PreserveLoggingContext
-from synapse.util.metrics import Measure
+from synapse.util.logcontext import preserve_fn
 
 logger = logging.getLogger(__name__)
 
@@ -60,11 +58,10 @@
 def wrap_json_request_handler(h):
     """Wraps a request handler method with exception handling.
 
-    Also adds logging as per wrap_request_handler_with_logging.
+    Also does the wrapping with request.processing as per wrap_async_request_handler.
 
     The handler method must have a signature of "handle_foo(self, request)",
-    where "self" must have a "clock" attribute (and "request" must be a
-    SynapseRequest).
+    where "request" must be a SynapseRequest.
 
     The handler must return a deferred. If the deferred succeeds we assume that
     a response has been sent. If the deferred fails with a SynapseError we use
@@ -108,24 +105,23 @@ def wrapped_request_handler(self, request):
                 pretty_print=_request_user_agent_is_curl(request),
             )
 
-    return wrap_request_handler_with_logging(wrapped_request_handler)
+    return wrap_async_request_handler(wrapped_request_handler)
 
 
 def wrap_html_request_handler(h):
     """Wraps a request handler method with exception handling.
 
-    Also adds logging as per wrap_request_handler_with_logging.
+    Also does the wrapping with request.processing as per wrap_async_request_handler.
 
     The handler method must have a signature of "handle_foo(self, request)",
-    where "self" must have a "clock" attribute (and "request" must be a
-    SynapseRequest).
+    where "request" must be a SynapseRequest.
     """
     def wrapped_request_handler(self, request):
         d = defer.maybeDeferred(h, self, request)
         d.addErrback(_return_html_error, request)
         return d
 
-    return wrap_request_handler_with_logging(wrapped_request_handler)
+    return wrap_async_request_handler(wrapped_request_handler)
 
 
 def _return_html_error(f, request):
@@ -170,46 +166,26 @@ def _return_html_error(f, request):
     finish_request(request)
 
 
-def wrap_request_handler_with_logging(h):
-    """Wraps a request handler to provide logging and metrics
+def wrap_async_request_handler(h):
+    """Wraps an async request handler so that it calls request.processing.
+
+    This helps ensure that work done by the request handler after the request is completed
+    is correctly recorded against the request metrics/logs.
 
     The handler method must have a signature of "handle_foo(self, request)",
-    where "self" must have a "clock" attribute (and "request" must be a
-    SynapseRequest).
+    where "request" must be a SynapseRequest.
 
-    As well as calling `request.processing` (which will log the response and
-    duration for this request), the wrapped request handler will insert the
-    request id into the logging context.
+    The handler may return a deferred, in which case the completion of the request isn't
+    logged until the deferred completes.
     """
     @defer.inlineCallbacks
-    def wrapped_request_handler(self, request):
-        """
-        Args:
-            self:
-            request (synapse.http.site.SynapseRequest):
-        """
+    def wrapped_async_request_handler(self, request):
+        with request.processing():
+            yield h(self, request)
 
-        request_id = request.get_request_id()
-        with LoggingContext(request_id) as request_context:
-            request_context.request = request_id
-            with Measure(self.clock, "wrapped_request_handler"):
-                # we start the request metrics timer here with an initial stab
-                # at the servlet name. For most requests that name will be
-                # JsonResource (or a subclass), and JsonResource._async_render
-                # will update it once it picks a servlet.
-                servlet_name = self.__class__.__name__
-                with request.processing(servlet_name):
-                    with PreserveLoggingContext(request_context):
-                        d = defer.maybeDeferred(h, self, request)
-
-                        # record the arrival of the request *after*
-                        # dispatching to the handler, so that the handler
-                        # can update the servlet name in the request
-                        # metrics
-                        requests_counter.labels(request.method,
-                                                request.request_metrics.name).inc()
-                        yield d
-    return wrapped_request_handler
+    # we need to preserve_fn here, because the synchronous render method won't yield for
+    # us (obviously)
+    return preserve_fn(wrapped_async_request_handler)
 
 
 class HttpServer(object):
@@ -272,7 +248,7 @@ def render(self, request):
         """ This gets called by twisted every time someone sends us a request.
         """
         self._async_render(request)
-        return server.NOT_DONE_YET
+        return NOT_DONE_YET
 
     @wrap_json_request_handler
     @defer.inlineCallbacks
diff --git a/synapse/http/site.py b/synapse/http/site.py
index 5fd30a4c2c4a..f5a8f7840654 100644
--- a/synapse/http/site.py
+++ b/synapse/http/site.py
@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 import contextlib
 import logging
 import time
@@ -19,8 +18,8 @@
 from twisted.web.server import Request, Site
 
 from synapse.http import redact_uri
-from synapse.http.request_metrics import RequestMetrics
-from synapse.util.logcontext import ContextResourceUsage, LoggingContext
+from synapse.http.request_metrics import RequestMetrics, requests_counter
+from synapse.util.logcontext import LoggingContext, PreserveLoggingContext
 
 logger = logging.getLogger(__name__)
 
@@ -34,25 +33,43 @@ class SynapseRequest(Request):
 
     It extends twisted's twisted.web.server.Request, and adds:
      * Unique request ID
+     * A log context associated with the request
      * Redaction of access_token query-params in __repr__
      * Logging at start and end
      * Metrics to record CPU, wallclock and DB time by endpoint.
 
-    It provides a method `processing` which should be called by the Resource
-    which is handling the request, and returns a context manager.
+    It also provides a method `processing`, which returns a context manager. If this
+    method is called, the request won't be logged until the context manager is closed;
+    this is useful for asynchronous request handlers which may go on processing the
+    request even after the client has disconnected.
 
+    Attributes:
+        logcontext(LoggingContext) : the log context for this request
     """
     def __init__(self, site, channel, *args, **kw):
         Request.__init__(self, channel, *args, **kw)
         self.site = site
-        self._channel = channel
+        self._channel = channel     # this is used by the tests
         self.authenticated_entity = None
         self.start_time = 0
 
+        # we can't yet create the logcontext, as we don't know the method.
+        self.logcontext = None
+
         global _next_request_seq
         self.request_seq = _next_request_seq
         _next_request_seq += 1
 
+        # whether an asynchronous request handler has called processing()
+        self._is_processing = False
+
+        # the time when the asynchronous request handler completed its processing
+        self._processing_finished_time = None
+
+        # what time we finished sending the response to the client (or the connection
+        # dropped)
+        self.finish_time = None
+
     def __repr__(self):
         # We overwrite this so that we don't log ``access_token``
         return '<%s at 0x%x method=%r uri=%r clientproto=%r site=%r>' % (
@@ -74,11 +91,116 @@ def get_user_agent(self):
         return self.requestHeaders.getRawHeaders(b"User-Agent", [None])[-1]
 
     def render(self, resrc):
+        # this is called once a Resource has been found to serve the request; in our
+        # case the Resource in question will normally be a JsonResource.
+
+        # create a LogContext for this request
+        request_id = self.get_request_id()
+        logcontext = self.logcontext = LoggingContext(request_id)
+        logcontext.request = request_id
+
         # override the Server header which is set by twisted
         self.setHeader("Server", self.site.server_version_string)
-        return Request.render(self, resrc)
+
+        with PreserveLoggingContext(self.logcontext):
+            # we start the request metrics timer here with an initial stab
+            # at the servlet name. For most requests that name will be
+            # JsonResource (or a subclass), and JsonResource._async_render
+            # will update it once it picks a servlet.
+            servlet_name = resrc.__class__.__name__
+            self._started_processing(servlet_name)
+
+            Request.render(self, resrc)
+
+            # record the arrival of the request *after*
+            # dispatching to the handler, so that the handler
+            # can update the servlet name in the request
+            # metrics
+            requests_counter.labels(self.method,
+                                    self.request_metrics.name).inc()
+
+    @contextlib.contextmanager
+    def processing(self):
+        """Record the fact that we are processing this request.
+
+        Returns a context manager; the correct way to use this is:
+
+        @defer.inlineCallbacks
+        def handle_request(request):
+            with request.processing("FooServlet"):
+                yield really_handle_the_request()
+
+        Once the context manager is closed, the completion of the request will be logged,
+        and the various metrics will be updated.
+        """
+        if self._is_processing:
+            raise RuntimeError("Request is already processing")
+        self._is_processing = True
+
+        try:
+            yield
+        except Exception:
+            # this should already have been caught, and sent back to the client as a 500.
+            logger.exception("Asynchronous messge handler raised an uncaught exception")
+        finally:
+            # the request handler has finished its work and either sent the whole response
+            # back, or handed over responsibility to a Producer.
+
+            self._processing_finished_time = time.time()
+            self._is_processing = False
+
+            # if we've already sent the response, log it now; otherwise, we wait for the
+            # response to be sent.
+            if self.finish_time is not None:
+                self._finished_processing()
+
+    def finish(self):
+        """Called when all response data has been written to this Request.
+
+        Overrides twisted.web.server.Request.finish to record the finish time and do
+        logging.
+        """
+        self.finish_time = time.time()
+        Request.finish(self)
+        if not self._is_processing:
+            with PreserveLoggingContext(self.logcontext):
+                self._finished_processing()
+
+    def connectionLost(self, reason):
+        """Called when the client connection is closed before the response is written.
+
+        Overrides twisted.web.server.Request.connectionLost to record the finish time and
+        do logging.
+        """
+        self.finish_time = time.time()
+        Request.connectionLost(self, reason)
+
+        # we only get here if the connection to the client drops before we send
+        # the response.
+        #
+        # It's useful to log it here so that we can get an idea of when
+        # the client disconnects.
+        with PreserveLoggingContext(self.logcontext):
+            logger.warn(
+                "Error processing request: %s %s", reason.type, reason.value,
+            )
+
+            if not self._is_processing:
+                self._finished_processing()
 
     def _started_processing(self, servlet_name):
+        """Record the fact that we are processing this request.
+
+        This will log the request's arrival. Once the request completes,
+        be sure to call finished_processing.
+
+        Args:
+            servlet_name (str): the name of the servlet which will be
+                processing this request. This is used in the metrics.
+
+                It is possible to update this afterwards by updating
+                self.request_metrics.name.
+        """
         self.start_time = time.time()
         self.request_metrics = RequestMetrics()
         self.request_metrics.start(
@@ -94,13 +216,21 @@ def _started_processing(self, servlet_name):
         )
 
     def _finished_processing(self):
-        try:
-            context = LoggingContext.current_context()
-            usage = context.get_resource_usage()
-        except Exception:
-            usage = ContextResourceUsage()
+        """Log the completion of this request and update the metrics
+        """
+
+        usage = self.logcontext.get_resource_usage()
+
+        if self._processing_finished_time is None:
+            # we completed the request without anything calling processing()
+            self._processing_finished_time = time.time()
 
-        end_time = time.time()
+        # the time between receiving the request and the request handler finishing
+        processing_time = self._processing_finished_time - self.start_time
+
+        # the time between the request handler finishing and the response being sent
+        # to the client (nb may be negative)
+        response_send_time = self.finish_time - self._processing_finished_time
 
         # need to decode as it could be raw utf-8 bytes
         # from a IDN servname in an auth header
@@ -116,22 +246,31 @@ def _finished_processing(self):
         user_agent = self.get_user_agent()
         if user_agent is not None:
             user_agent = user_agent.decode("utf-8", "replace")
+        else:
+            user_agent = "-"
+
+        code = str(self.code)
+        if not self.finished:
+            # we didn't send the full response before we gave up (presumably because
+            # the connection dropped)
+            code += "!"
 
         self.site.access_logger.info(
             "%s - %s - {%s}"
-            " Processed request: %.3fsec (%.3fsec, %.3fsec) (%.3fsec/%.3fsec/%d)"
+            " Processed request: %.3fsec/%.3fsec (%.3fsec, %.3fsec) (%.3fsec/%.3fsec/%d)"
             " %sB %s \"%s %s %s\" \"%s\" [%d dbevts]",
             self.getClientIP(),
             self.site.site_tag,
             authenticated_entity,
-            end_time - self.start_time,
+            processing_time,
+            response_send_time,
             usage.ru_utime,
             usage.ru_stime,
             usage.db_sched_duration_sec,
             usage.db_txn_duration_sec,
             int(usage.db_txn_count),
             self.sentLength,
-            self.code,
+            code,
             self.method,
             self.get_redacted_uri(),
             self.clientproto,
@@ -140,38 +279,10 @@ def _finished_processing(self):
         )
 
         try:
-            self.request_metrics.stop(end_time, self)
+            self.request_metrics.stop(self.finish_time, self)
         except Exception as e:
             logger.warn("Failed to stop metrics: %r", e)
 
-    @contextlib.contextmanager
-    def processing(self, servlet_name):
-        """Record the fact that we are processing this request.
-
-        Returns a context manager; the correct way to use this is:
-
-        @defer.inlineCallbacks
-        def handle_request(request):
-            with request.processing("FooServlet"):
-                yield really_handle_the_request()
-
-        This will log the request's arrival. Once the context manager is
-        closed, the completion of the request will be logged, and the various
-        metrics will be updated.
-
-        Args:
-            servlet_name (str): the name of the servlet which will be
-                processing this request. This is used in the metrics.
-
-                It is possible to update this afterwards by updating
-                self.request_metrics.servlet_name.
-        """
-        # TODO: we should probably just move this into render() and finish(),
-        # to save having to call a separate method.
-        self._started_processing(servlet_name)
-        yield
-        self._finished_processing()
-
 
 class XForwardedForRequest(SynapseRequest):
     def __init__(self, *args, **kw):

From 2de813a61176e4db892aa07dec4cb3a390da5b4b Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 15 Aug 2018 13:49:03 +0100
Subject: [PATCH 408/830] changelog

---
 changelog.d/3700.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3700.bugfix

diff --git a/changelog.d/3700.bugfix b/changelog.d/3700.bugfix
new file mode 100644
index 000000000000..492cce1dfc33
--- /dev/null
+++ b/changelog.d/3700.bugfix
@@ -0,0 +1 @@
+Improve HTTP request logging to include all requests
\ No newline at end of file

From 39176f27f79914ec4608235e1f875b16d0018c92 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 15 Aug 2018 13:53:51 +0100
Subject: [PATCH 409/830] remove changelog referencing another PR

---
 changelog.d/3697.misc | 1 -
 1 file changed, 1 deletion(-)
 delete mode 100644 changelog.d/3697.misc

diff --git a/changelog.d/3697.misc b/changelog.d/3697.misc
deleted file mode 100644
index 9082fc466285..000000000000
--- a/changelog.d/3697.misc
+++ /dev/null
@@ -1 +0,0 @@
-update resource limit error codes

From 773db62a22ca8be06c9537ae0296952d0e53837e Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 15 Aug 2018 14:10:32 +0100
Subject: [PATCH 410/830] Rename slave TransactionStore to
 SlaveTransactionStore

---
 synapse/app/client_reader.py                      | 4 ++--
 synapse/app/event_creator.py                      | 4 ++--
 synapse/app/federation_reader.py                  | 4 ++--
 synapse/app/federation_sender.py                  | 4 ++--
 synapse/app/media_repository.py                   | 4 ++--
 synapse/replication/slave/storage/transactions.py | 2 +-
 6 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/synapse/app/client_reader.py b/synapse/app/client_reader.py
index e2c91123db8c..86129ed544bc 100644
--- a/synapse/app/client_reader.py
+++ b/synapse/app/client_reader.py
@@ -39,7 +39,7 @@
 from synapse.replication.slave.storage.keys import SlavedKeyStore
 from synapse.replication.slave.storage.registration import SlavedRegistrationStore
 from synapse.replication.slave.storage.room import RoomStore
-from synapse.replication.slave.storage.transactions import TransactionStore
+from synapse.replication.slave.storage.transactions import SlavedTransactionStore
 from synapse.replication.tcp.client import ReplicationClientHandler
 from synapse.rest.client.v1.room import (
     JoinedRoomMemberListRestServlet,
@@ -66,7 +66,7 @@ class ClientReaderSlavedStore(
     DirectoryStore,
     SlavedApplicationServiceStore,
     SlavedRegistrationStore,
-    TransactionStore,
+    SlavedTransactionStore,
     SlavedClientIpStore,
     BaseSlavedStore,
 ):
diff --git a/synapse/app/event_creator.py b/synapse/app/event_creator.py
index 374f1156449c..95bfff181bb2 100644
--- a/synapse/app/event_creator.py
+++ b/synapse/app/event_creator.py
@@ -43,7 +43,7 @@
 from synapse.replication.slave.storage.receipts import SlavedReceiptsStore
 from synapse.replication.slave.storage.registration import SlavedRegistrationStore
 from synapse.replication.slave.storage.room import RoomStore
-from synapse.replication.slave.storage.transactions import TransactionStore
+from synapse.replication.slave.storage.transactions import SlavedTransactionStore
 from synapse.replication.tcp.client import ReplicationClientHandler
 from synapse.rest.client.v1.room import (
     JoinRoomAliasServlet,
@@ -63,7 +63,7 @@
 
 class EventCreatorSlavedStore(
     DirectoryStore,
-    TransactionStore,
+    SlavedTransactionStore,
     SlavedProfileStore,
     SlavedAccountDataStore,
     SlavedPusherStore,
diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py
index 0c557a824227..7e342c178594 100644
--- a/synapse/app/federation_reader.py
+++ b/synapse/app/federation_reader.py
@@ -41,7 +41,7 @@
 from synapse.replication.slave.storage.pushers import SlavedPusherStore
 from synapse.replication.slave.storage.receipts import SlavedReceiptsStore
 from synapse.replication.slave.storage.room import RoomStore
-from synapse.replication.slave.storage.transactions import TransactionStore
+from synapse.replication.slave.storage.transactions import SlavedTransactionStore
 from synapse.replication.tcp.client import ReplicationClientHandler
 from synapse.server import HomeServer
 from synapse.storage.engines import create_engine
@@ -63,7 +63,7 @@ class FederationReaderSlavedStore(
     SlavedKeyStore,
     RoomStore,
     DirectoryStore,
-    TransactionStore,
+    SlavedTransactionStore,
     BaseSlavedStore,
 ):
     pass
diff --git a/synapse/app/federation_sender.py b/synapse/app/federation_sender.py
index 18469013faf0..29911ec68710 100644
--- a/synapse/app/federation_sender.py
+++ b/synapse/app/federation_sender.py
@@ -36,7 +36,7 @@
 from synapse.replication.slave.storage.presence import SlavedPresenceStore
 from synapse.replication.slave.storage.receipts import SlavedReceiptsStore
 from synapse.replication.slave.storage.registration import SlavedRegistrationStore
-from synapse.replication.slave.storage.transactions import TransactionStore
+from synapse.replication.slave.storage.transactions import SlavedTransactionStore
 from synapse.replication.tcp.client import ReplicationClientHandler
 from synapse.server import HomeServer
 from synapse.storage.engines import create_engine
@@ -50,7 +50,7 @@
 
 
 class FederationSenderSlaveStore(
-    SlavedDeviceInboxStore, TransactionStore, SlavedReceiptsStore, SlavedEventStore,
+    SlavedDeviceInboxStore, SlavedTransactionStore, SlavedReceiptsStore, SlavedEventStore,
     SlavedRegistrationStore, SlavedDeviceStore, SlavedPresenceStore,
 ):
     def __init__(self, db_conn, hs):
diff --git a/synapse/app/media_repository.py b/synapse/app/media_repository.py
index 749bbf37d021..ee16755660d9 100644
--- a/synapse/app/media_repository.py
+++ b/synapse/app/media_repository.py
@@ -34,7 +34,7 @@
 from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore
 from synapse.replication.slave.storage.client_ips import SlavedClientIpStore
 from synapse.replication.slave.storage.registration import SlavedRegistrationStore
-from synapse.replication.slave.storage.transactions import TransactionStore
+from synapse.replication.slave.storage.transactions import SlavedTransactionStore
 from synapse.replication.tcp.client import ReplicationClientHandler
 from synapse.rest.media.v0.content_repository import ContentRepoResource
 from synapse.server import HomeServer
@@ -52,7 +52,7 @@ class MediaRepositorySlavedStore(
     SlavedApplicationServiceStore,
     SlavedRegistrationStore,
     SlavedClientIpStore,
-    TransactionStore,
+    SlavedTransactionStore,
     BaseSlavedStore,
     MediaRepositoryStore,
 ):
diff --git a/synapse/replication/slave/storage/transactions.py b/synapse/replication/slave/storage/transactions.py
index cd6416c47c8a..3527beb3c926 100644
--- a/synapse/replication/slave/storage/transactions.py
+++ b/synapse/replication/slave/storage/transactions.py
@@ -18,5 +18,5 @@
 from ._base import BaseSlavedStore
 
 
-class TransactionStore(TransactionStore, BaseSlavedStore):
+class SlavedTransactionStore(TransactionStore, BaseSlavedStore):
     pass

From 488ffe6fdb36c7479052096489c683777597c2aa Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 15 Aug 2018 14:17:18 +0100
Subject: [PATCH 411/830] Use federation handler function rather than duplicate

This involves renaming _persist_events to be a public function.
---
 synapse/handlers/federation.py         | 14 ++++----
 synapse/replication/http/federation.py | 44 ++------------------------
 2 files changed, 10 insertions(+), 48 deletions(-)

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index acabca1d258c..9a37d627ca9a 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -1175,7 +1175,7 @@ def on_invite_request(self, origin, pdu):
         )
 
         context = yield self.state_handler.compute_event_context(event)
-        yield self._persist_events([(event, context)])
+        yield self.persist_events_and_notify([(event, context)])
 
         defer.returnValue(event)
 
@@ -1206,7 +1206,7 @@ def do_remotely_reject_invite(self, target_hosts, room_id, user_id):
         )
 
         context = yield self.state_handler.compute_event_context(event)
-        yield self._persist_events([(event, context)])
+        yield self.persist_events_and_notify([(event, context)])
 
         defer.returnValue(event)
 
@@ -1449,7 +1449,7 @@ def _handle_new_event(self, origin, event, state=None, auth_events=None,
                     event, context
                 )
 
-            yield self._persist_events(
+            yield self.persist_events_and_notify(
                 [(event, context)],
                 backfilled=backfilled,
             )
@@ -1487,7 +1487,7 @@ def _handle_new_events(self, origin, event_infos, backfilled=False):
             ], consumeErrors=True,
         ))
 
-        yield self._persist_events(
+        yield self.persist_events_and_notify(
             [
                 (ev_info["event"], context)
                 for ev_info, context in zip(event_infos, contexts)
@@ -1575,7 +1575,7 @@ def _persist_auth_tree(self, origin, auth_events, state, event):
                     raise
                 events_to_context[e.event_id].rejected = RejectedReason.AUTH_ERROR
 
-        yield self._persist_events(
+        yield self.persist_events_and_notify(
             [
                 (e, events_to_context[e.event_id])
                 for e in itertools.chain(auth_events, state)
@@ -1586,7 +1586,7 @@ def _persist_auth_tree(self, origin, auth_events, state, event):
             event, old_state=state
         )
 
-        yield self._persist_events(
+        yield self.persist_events_and_notify(
             [(event, new_event_context)],
         )
 
@@ -2327,7 +2327,7 @@ def _check_key_revocation(self, public_key, url):
             raise AuthError(403, "Third party certificate was invalid")
 
     @defer.inlineCallbacks
-    def _persist_events(self, event_and_contexts, backfilled=False):
+    def persist_events_and_notify(self, event_and_contexts, backfilled=False):
         """Persists events and tells the notifier/pushers about them, if
         necessary.
 
diff --git a/synapse/replication/http/federation.py b/synapse/replication/http/federation.py
index 7b0b1cd32ed1..2ddd18f73b31 100644
--- a/synapse/replication/http/federation.py
+++ b/synapse/replication/http/federation.py
@@ -17,13 +17,10 @@
 
 from twisted.internet import defer
 
-from synapse.api.constants import EventTypes, Membership
 from synapse.events import FrozenEvent
 from synapse.events.snapshot import EventContext
 from synapse.http.servlet import parse_json_object_from_request
 from synapse.replication.http._base import ReplicationEndpoint
-from synapse.types import UserID
-from synapse.util.logcontext import run_in_background
 from synapse.util.metrics import Measure
 
 logger = logging.getLogger(__name__)
@@ -55,9 +52,7 @@ def __init__(self, hs):
 
         self.store = hs.get_datastore()
         self.clock = hs.get_clock()
-        self.is_mine_id = hs.is_mine_id
-        self.notifier = hs.get_notifier()
-        self.pusher_pool = hs.get_pusherpool()
+        self.federation_handler = hs.get_handlers().federation_handler
 
     @staticmethod
     @defer.inlineCallbacks
@@ -114,45 +109,12 @@ def _handle_request(self, request):
             len(event_and_contexts),
         )
 
-        max_stream_id = yield self.store.persist_events(
-            event_and_contexts,
-            backfilled=backfilled
+        yield self.federation_handler.persist_events_and_notify(
+            event_and_contexts, backfilled,
         )
 
-        if not backfilled:
-            for event, _ in event_and_contexts:
-                self._notify_persisted_event(event, max_stream_id)
-
         defer.returnValue((200, {}))
 
-    def _notify_persisted_event(self, event, max_stream_id):
-        extra_users = []
-        if event.type == EventTypes.Member:
-            target_user_id = event.state_key
-
-            # We notify for memberships if its an invite for one of our
-            # users
-            if event.internal_metadata.is_outlier():
-                if event.membership != Membership.INVITE:
-                    if not self.is_mine_id(target_user_id):
-                        return
-
-            target_user = UserID.from_string(target_user_id)
-            extra_users.append(target_user)
-        elif event.internal_metadata.is_outlier():
-            return
-
-        event_stream_id = event.internal_metadata.stream_ordering
-        self.notifier.on_new_room_event(
-            event, event_stream_id, max_stream_id,
-            extra_users=extra_users
-        )
-
-        run_in_background(
-            self.pusher_pool.on_new_notifications,
-            event_stream_id, max_stream_id,
-        )
-
 
 class ReplicationFederationSendEduRestServlet(ReplicationEndpoint):
     """Handles EDUs newly received from federation, including persisting and

From afcd655ab67611e3bbac3bc992574baa464bc030 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 14 Aug 2018 16:13:17 +0100
Subject: [PATCH 412/830] Use a producer to stream back responses

The problem with dumping all of the json response into the Request object at
once is that doing so starts the timeout for the next request to be received:
so if it takes longer than 60s to stream back the response to the client, the
client never gets it.

The correct solution is to use a Producer; then the timeout is only started
once all of the content is sent over the TCP connection.
---
 synapse/http/server.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/synapse/http/server.py b/synapse/http/server.py
index 6dacb31037aa..5733abb5dcbb 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -27,6 +27,7 @@
 from twisted.python import failure
 from twisted.web import resource, server
 from twisted.web.server import NOT_DONE_YET
+from twisted.web.static import NoRangeStaticProducer
 from twisted.web.util import redirectTo
 
 import synapse.events
@@ -42,6 +43,11 @@
 from synapse.util.logcontext import LoggingContext, PreserveLoggingContext
 from synapse.util.metrics import Measure
 
+if PY3:
+    from io import BytesIO
+else:
+    from cStringIO import StringIO as BytesIO
+
 logger = logging.getLogger(__name__)
 
 HTML_ERROR_TEMPLATE = """<!DOCTYPE html>
@@ -413,8 +419,7 @@ def respond_with_json(request, code, json_object, send_cors=False,
         return
 
     if pretty_print:
-        json_bytes = (encode_pretty_printed_json(json_object) + "\n"
-                      ).encode("utf-8")
+        json_bytes = encode_pretty_printed_json(json_object) + b"\n"
     else:
         if canonical_json or synapse.events.USE_FROZEN_DICTS:
             # canonicaljson already encodes to bytes
@@ -450,8 +455,12 @@ def respond_with_json_bytes(request, code, json_bytes, send_cors=False,
     if send_cors:
         set_cors_headers(request)
 
-    request.write(json_bytes)
-    finish_request(request)
+    # todo: we can almost certainly avoid this copy and encode the json straight into
+    # the bytesIO, but it would involve faffing around with string->bytes wrappers.
+    bytes_io = BytesIO(json_bytes)
+
+    producer = NoRangeStaticProducer(request, bytes_io)
+    producer.start()
     return NOT_DONE_YET
 
 

From c24fc9797bc0dd59c0d8131ea09c1178e39b76f6 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 15 Aug 2018 15:04:30 +0100
Subject: [PATCH 413/830] add new event types

---
 synapse/api/constants.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/synapse/api/constants.py b/synapse/api/constants.py
index ef6084199593..1fb24578e265 100644
--- a/synapse/api/constants.py
+++ b/synapse/api/constants.py
@@ -78,6 +78,7 @@ class EventTypes(object):
     Name = "m.room.name"
 
     ServerACL = "m.room.server_acl"
+    Pinned = "m.room.pinned_events"
 
     ServerNoticeLimitReached = "m.server_notice.usage_limit_reached"
 

From eabc5f827107a24717655e0eec376adf88b574c4 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 15 Aug 2018 15:04:52 +0100
Subject: [PATCH 414/830] wip cut at sending resource server notices

---
 .../resource_limits_server_notices.py         | 69 ++++++++++---------
 .../server_notices/server_notices_manager.py  | 30 +++++---
 2 files changed, 55 insertions(+), 44 deletions(-)

diff --git a/synapse/server_notices/resource_limits_server_notices.py b/synapse/server_notices/resource_limits_server_notices.py
index 0470f6620574..94d0f98189de 100644
--- a/synapse/server_notices/resource_limits_server_notices.py
+++ b/synapse/server_notices/resource_limits_server_notices.py
@@ -40,6 +40,7 @@ def __init__(self, hs):
 
         self._notified_of_blocking = set()
         self._resouce_limited = False
+
         # Config checks?
 
     @defer.inlineCallbacks
@@ -69,43 +70,43 @@ def maybe_send_server_notice_to_user(self, user_id):
                 self._resouce_limited = False
                 # Need to start removing notices
                 if user_id in self._notified_of_blocking:
-                    # Send message to remove warning - needs updating
+                    # Send message to remove warning
+                    # send state event here
+                    # How do I do this? if drop the id, how to refer to it?
                     content = {
-                        'body': '',
-                        'admin_email': '',
+                        "pinned":[]
                     }
-                    self._send_server_notice(user_id, content)
+                    yield self._server_notices_manager.send_notice(
+                        user_id, content, EventTypes.Pinned, '',
+                    )
+
                     self._notified_of_blocking.remove(user_id)
 
             except AuthError as e:
                 # Need to start notifying of blocking
-
-                self._resouce_limited = True
-                if user_id not in self._notified_of_blocking:
-                    # TODO use admin email contained in error once PR lands
-                    content = {
-                        'body': e.msg,
-                        'admin_email': 'stunt@adminemail.com',
-                        'msgtype': 'm.text'
-                    }
-                    self._send_server_notice(user_id, content)
-                    self._notified_of_blocking.add(user_id)
-
-
-    @defer.inlineCallbacks
-    def _send_server_notice(self, user_id, content):
-        """Sends Server notice
-
-        Args:
-            user_id(str): The user to send to
-            content(str): The content of the message
-
-        Returns:
-            Deferred[]
-        """
-        try:
-            yield self._server_notices_manager.send_notice(
-                user_id, content
-            )
-        except SynapseError as e:
-            logger.error("Error sending server notice about resource limits: %s", e)
+                try:
+                    self._resouce_limited = True
+                    if user_id not in self._notified_of_blocking:
+                        # TODO use admin email contained in error once PR lands
+                        content = {
+                            'body': e.msg,
+                            'admin_email': 'stunt@adminemail.com',
+                        }
+                        event = yield self._server_notices_manager.send_notice(
+                            user_id, content, EventTypes.ServerNoticeLimitReached
+                        )
+
+                        # send server notices state event here
+                        # TODO Over writing pinned events
+                        content = {
+                            "pinned":[
+                                event.event_id,
+                            ]
+                        }
+                        yield self._server_notices_manager.send_notice(
+                            user_id, content, EventTypes.Pinned, '',
+                        )
+
+                        self._notified_of_blocking.add(user_id)
+                except SynapseError as e:
+                    logger.error("Error sending server notice about resource limits: %s", e)
diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py
index a26deace5318..3e57f8211ae9 100644
--- a/synapse/server_notices/server_notices_manager.py
+++ b/synapse/server_notices/server_notices_manager.py
@@ -46,7 +46,10 @@ def is_enabled(self):
         return self._config.server_notices_mxid is not None
 
     @defer.inlineCallbacks
-    def send_notice(self, user_id, event_content):
+    def send_notice(
+        self, user_id, event_content,
+        type=EventTypes.Message, state_key=None
+    ):
         """Send a notice to the given user
 
         Creates the server notices room, if none exists.
@@ -54,9 +57,11 @@ def send_notice(self, user_id, event_content):
         Args:
             user_id (str): mxid of user to send event to.
             event_content (dict): content of event to send
+            type(EventTypes): type of event
+            is_state_event(bool): Is the event a state event
 
         Returns:
-            Deferred[None]
+            Deferred[FrozenEvent]
         """
         room_id = yield self.get_notice_room_for_user(user_id)
 
@@ -65,15 +70,19 @@ def send_notice(self, user_id, event_content):
 
         logger.info("Sending server notice to %s", user_id)
 
-        yield self._event_creation_handler.create_and_send_nonmember_event(
-            requester, {
-                "type": EventTypes.Message,
-                "room_id": room_id,
-                "sender": system_mxid,
-                "content": event_content,
-            },
-            ratelimit=False,
+        event_dict = {
+            "type": type,
+            "room_id": room_id,
+            "sender": system_mxid,
+            "content": event_content,
+        }
+        if state_key:
+            event_dict['state_key'] = state_key
+
+        res = yield self._event_creation_handler.create_and_send_nonmember_event(
+            requester, event_dict, ratelimit=False,
         )
+        defer.returnValue(res)
 
     @cachedInlineCallbacks()
     def get_notice_room_for_user(self, user_id):
@@ -141,6 +150,7 @@ def get_notice_room_for_user(self, user_id):
             creator_join_profile=join_profile,
         )
         room_id = info['room_id']
+        yield self._store.add_tag_to_room(user_id, room_id, 'm.server_notice', None)
 
         logger.info("Created server notices room %s for %s", room_id, user_id)
         defer.returnValue(room_id)

From a87af25fbb9d8938b1557ef42e31edecf1ee731c Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Wed, 15 Aug 2018 23:43:41 +1000
Subject: [PATCH 415/830] Fix the tests

---
 tests/rest/client/v1/test_register.py       |  7 ++----
 tests/rest/client/v1/utils.py               |  8 +++----
 tests/rest/client/v2_alpha/test_filter.py   | 23 +++++++-----------
 tests/rest/client/v2_alpha/test_register.py | 26 +++++++--------------
 tests/rest/client/v2_alpha/test_sync.py     |  5 ++--
 tests/server.py                             | 23 ++++++++++++++----
 tests/test_server.py                        | 17 +++++---------
 7 files changed, 49 insertions(+), 60 deletions(-)

diff --git a/tests/rest/client/v1/test_register.py b/tests/rest/client/v1/test_register.py
index 4be88b8a39fe..6b7ff813d567 100644
--- a/tests/rest/client/v1/test_register.py
+++ b/tests/rest/client/v1/test_register.py
@@ -25,7 +25,7 @@
 from synapse.util import Clock
 
 from tests import unittest
-from tests.server import make_request, setup_test_homeserver
+from tests.server import make_request, render, setup_test_homeserver
 
 
 class CreateUserServletTestCase(unittest.TestCase):
@@ -77,10 +77,7 @@ def test_POST_createuser_with_valid_user(self):
         )
 
         request, channel = make_request(b"POST", url, request_data)
-        request.render(res)
-
-        # Advance the clock because it waits
-        self.clock.advance(1)
+        render(request, res, self.clock)
 
         self.assertEquals(channel.result["code"], b"200")
 
diff --git a/tests/rest/client/v1/utils.py b/tests/rest/client/v1/utils.py
index 9f862f9dfa02..40dc4ea256da 100644
--- a/tests/rest/client/v1/utils.py
+++ b/tests/rest/client/v1/utils.py
@@ -23,7 +23,7 @@
 from synapse.api.constants import Membership
 
 from tests import unittest
-from tests.server import make_request, wait_until_result
+from tests.server import make_request, render
 
 
 class RestTestCase(unittest.TestCase):
@@ -171,8 +171,7 @@ def create_room_as(self, room_creator, is_public=True, tok=None):
         request, channel = make_request(
             "POST", path, json.dumps(content).encode('utf8')
         )
-        request.render(self.resource)
-        wait_until_result(self.hs.get_reactor(), channel)
+        render(request, self.resource, self.hs.get_reactor())
 
         assert channel.result["code"] == b"200", channel.result
         self.auth_user_id = temp_id
@@ -220,8 +219,7 @@ def change_membership(self, room, src, targ, membership, tok=None, expect_code=2
 
         request, channel = make_request("PUT", path, json.dumps(data).encode('utf8'))
 
-        request.render(self.resource)
-        wait_until_result(self.hs.get_reactor(), channel)
+        render(request, self.resource, self.hs.get_reactor())
 
         assert int(channel.result["code"]) == expect_code, (
             "Expected: %d, got: %d, resp: %r"
diff --git a/tests/rest/client/v2_alpha/test_filter.py b/tests/rest/client/v2_alpha/test_filter.py
index 8260c130f8e5..6a886ee3b832 100644
--- a/tests/rest/client/v2_alpha/test_filter.py
+++ b/tests/rest/client/v2_alpha/test_filter.py
@@ -24,8 +24,8 @@
 from tests.server import (
     ThreadedMemoryReactorClock as MemoryReactorClock,
     make_request,
+    render,
     setup_test_homeserver,
-    wait_until_result,
 )
 
 PATH_PREFIX = "/_matrix/client/v2_alpha"
@@ -76,8 +76,7 @@ def test_add_filter(self):
             "/_matrix/client/r0/user/%s/filter" % (self.USER_ID),
             self.EXAMPLE_FILTER_JSON,
         )
-        request.render(self.resource)
-        wait_until_result(self.clock, channel)
+        render(request, self.resource, self.clock)
 
         self.assertEqual(channel.result["code"], b"200")
         self.assertEqual(channel.json_body, {"filter_id": "0"})
@@ -91,8 +90,7 @@ def test_add_filter_for_other_user(self):
             "/_matrix/client/r0/user/%s/filter" % ("@watermelon:test"),
             self.EXAMPLE_FILTER_JSON,
         )
-        request.render(self.resource)
-        wait_until_result(self.clock, channel)
+        render(request, self.resource, self.clock)
 
         self.assertEqual(channel.result["code"], b"403")
         self.assertEquals(channel.json_body["errcode"], Codes.FORBIDDEN)
@@ -105,8 +103,7 @@ def test_add_filter_non_local_user(self):
             "/_matrix/client/r0/user/%s/filter" % (self.USER_ID),
             self.EXAMPLE_FILTER_JSON,
         )
-        request.render(self.resource)
-        wait_until_result(self.clock, channel)
+        render(request, self.resource, self.clock)
 
         self.hs.is_mine = _is_mine
         self.assertEqual(channel.result["code"], b"403")
@@ -121,8 +118,7 @@ def test_get_filter(self):
         request, channel = make_request(
             "GET", "/_matrix/client/r0/user/%s/filter/%s" % (self.USER_ID, filter_id)
         )
-        request.render(self.resource)
-        wait_until_result(self.clock, channel)
+        render(request, self.resource, self.clock)
 
         self.assertEqual(channel.result["code"], b"200")
         self.assertEquals(channel.json_body, self.EXAMPLE_FILTER)
@@ -131,8 +127,7 @@ def test_get_filter_non_existant(self):
         request, channel = make_request(
             "GET", "/_matrix/client/r0/user/%s/filter/12382148321" % (self.USER_ID)
         )
-        request.render(self.resource)
-        wait_until_result(self.clock, channel)
+        render(request, self.resource, self.clock)
 
         self.assertEqual(channel.result["code"], b"400")
         self.assertEquals(channel.json_body["errcode"], Codes.NOT_FOUND)
@@ -143,8 +138,7 @@ def test_get_filter_invalid_id(self):
         request, channel = make_request(
             "GET", "/_matrix/client/r0/user/%s/filter/foobar" % (self.USER_ID)
         )
-        request.render(self.resource)
-        wait_until_result(self.clock, channel)
+        render(request, self.resource, self.clock)
 
         self.assertEqual(channel.result["code"], b"400")
 
@@ -153,7 +147,6 @@ def test_get_filter_no_id(self):
         request, channel = make_request(
             "GET", "/_matrix/client/r0/user/%s/filter/" % (self.USER_ID)
         )
-        request.render(self.resource)
-        wait_until_result(self.clock, channel)
+        render(request, self.resource, self.clock)
 
         self.assertEqual(channel.result["code"], b"400")
diff --git a/tests/rest/client/v2_alpha/test_register.py b/tests/rest/client/v2_alpha/test_register.py
index b72bd0fb7fab..1c128e81f5ac 100644
--- a/tests/rest/client/v2_alpha/test_register.py
+++ b/tests/rest/client/v2_alpha/test_register.py
@@ -11,7 +11,7 @@
 from synapse.util import Clock
 
 from tests import unittest
-from tests.server import make_request, setup_test_homeserver, wait_until_result
+from tests.server import make_request, render, setup_test_homeserver
 
 
 class RegisterRestServletTestCase(unittest.TestCase):
@@ -72,8 +72,7 @@ def test_POST_appservice_registration_valid(self):
         request, channel = make_request(
             b"POST", self.url + b"?access_token=i_am_an_app_service", request_data
         )
-        request.render(self.resource)
-        wait_until_result(self.clock, channel)
+        render(request, self.resource, self.clock)
 
         self.assertEquals(channel.result["code"], b"200", channel.result)
         det_data = {
@@ -89,16 +88,14 @@ def test_POST_appservice_registration_invalid(self):
         request, channel = make_request(
             b"POST", self.url + b"?access_token=i_am_an_app_service", request_data
         )
-        request.render(self.resource)
-        wait_until_result(self.clock, channel)
+        render(request, self.resource, self.clock)
 
         self.assertEquals(channel.result["code"], b"401", channel.result)
 
     def test_POST_bad_password(self):
         request_data = json.dumps({"username": "kermit", "password": 666})
         request, channel = make_request(b"POST", self.url, request_data)
-        request.render(self.resource)
-        wait_until_result(self.clock, channel)
+        render(request, self.resource, self.clock)
 
         self.assertEquals(channel.result["code"], b"400", channel.result)
         self.assertEquals(channel.json_body["error"], "Invalid password")
@@ -106,8 +103,7 @@ def test_POST_bad_password(self):
     def test_POST_bad_username(self):
         request_data = json.dumps({"username": 777, "password": "monkey"})
         request, channel = make_request(b"POST", self.url, request_data)
-        request.render(self.resource)
-        wait_until_result(self.clock, channel)
+        render(request, self.resource, self.clock)
 
         self.assertEquals(channel.result["code"], b"400", channel.result)
         self.assertEquals(channel.json_body["error"], "Invalid username")
@@ -126,8 +122,7 @@ def test_POST_user_valid(self):
         self.device_handler.check_device_registered = Mock(return_value=device_id)
 
         request, channel = make_request(b"POST", self.url, request_data)
-        request.render(self.resource)
-        wait_until_result(self.clock, channel)
+        render(request, self.resource, self.clock)
 
         det_data = {
             "user_id": user_id,
@@ -149,8 +144,7 @@ def test_POST_disabled_registration(self):
         self.registration_handler.register = Mock(return_value=("@user:id", "t"))
 
         request, channel = make_request(b"POST", self.url, request_data)
-        request.render(self.resource)
-        wait_until_result(self.clock, channel)
+        render(request, self.resource, self.clock)
 
         self.assertEquals(channel.result["code"], b"403", channel.result)
         self.assertEquals(channel.json_body["error"], "Registration has been disabled")
@@ -162,8 +156,7 @@ def test_POST_guest_registration(self):
         self.registration_handler.register = Mock(return_value=(user_id, None))
 
         request, channel = make_request(b"POST", self.url + b"?kind=guest", b"{}")
-        request.render(self.resource)
-        wait_until_result(self.clock, channel)
+        render(request, self.resource, self.clock)
 
         det_data = {
             "user_id": user_id,
@@ -177,8 +170,7 @@ def test_POST_disabled_guest_registration(self):
         self.hs.config.allow_guest_access = False
 
         request, channel = make_request(b"POST", self.url + b"?kind=guest", b"{}")
-        request.render(self.resource)
-        wait_until_result(self.clock, channel)
+        render(request, self.resource, self.clock)
 
         self.assertEquals(channel.result["code"], b"403", channel.result)
         self.assertEquals(channel.json_body["error"], "Guest access is disabled")
diff --git a/tests/rest/client/v2_alpha/test_sync.py b/tests/rest/client/v2_alpha/test_sync.py
index 2e1d06c50982..9f3d8bd1dbb4 100644
--- a/tests/rest/client/v2_alpha/test_sync.py
+++ b/tests/rest/client/v2_alpha/test_sync.py
@@ -23,8 +23,8 @@
 from tests.server import (
     ThreadedMemoryReactorClock as MemoryReactorClock,
     make_request,
+    render,
     setup_test_homeserver,
-    wait_until_result,
 )
 
 PATH_PREFIX = "/_matrix/client/v2_alpha"
@@ -69,8 +69,7 @@ def get_user_by_req(request, allow_guest=False, rights="access"):
 
     def test_sync_argless(self):
         request, channel = make_request("GET", "/_matrix/client/r0/sync")
-        request.render(self.resource)
-        wait_until_result(self.clock, channel)
+        render(request, self.resource, self.clock)
 
         self.assertEqual(channel.result["code"], b"200")
         self.assertTrue(
diff --git a/tests/server.py b/tests/server.py
index beb24cf0320a..c63b2c310000 100644
--- a/tests/server.py
+++ b/tests/server.py
@@ -24,6 +24,7 @@ class FakeChannel(object):
     """
 
     result = attr.ib(default=attr.Factory(dict))
+    _producer = None
 
     @property
     def json_body(self):
@@ -49,6 +50,15 @@ def write(self, content):
 
         self.result["body"] += content
 
+    def registerProducer(self, producer, streaming):
+        self._producer = producer
+
+    def unregisterProducer(self):
+        if self._producer is None:
+            return
+
+        self._producer = None
+
     def requestDone(self, _self):
         self.result["done"] = True
 
@@ -111,14 +121,19 @@ def make_request(method, path, content=b""):
     return req, channel
 
 
-def wait_until_result(clock, channel, timeout=100):
+def wait_until_result(clock, request, timeout=100):
     """
-    Wait until the channel has a result.
+    Wait until the request is finished.
     """
     clock.run()
     x = 0
 
-    while not channel.result:
+    while not request.finished:
+
+        # If there's a producer, tell it to resume producing so we get content
+        if request._channel._producer:
+            request._channel._producer.resumeProducing()
+
         x += 1
 
         if x > timeout:
@@ -129,7 +144,7 @@ def wait_until_result(clock, channel, timeout=100):
 
 def render(request, resource, clock):
     request.render(resource)
-    wait_until_result(clock, request._channel)
+    wait_until_result(clock, request)
 
 
 class ThreadedMemoryReactorClock(MemoryReactorClock):
diff --git a/tests/test_server.py b/tests/test_server.py
index 895e49040688..ef74544e93cc 100644
--- a/tests/test_server.py
+++ b/tests/test_server.py
@@ -8,7 +8,7 @@
 from synapse.util import Clock
 
 from tests import unittest
-from tests.server import make_request, setup_test_homeserver
+from tests.server import make_request, render, setup_test_homeserver
 
 
 class JsonResourceTests(unittest.TestCase):
@@ -37,7 +37,7 @@ def _callback(request, **kwargs):
         )
 
         request, channel = make_request(b"GET", b"/_matrix/foo/%E2%98%83?a=%E2%98%83")
-        request.render(res)
+        render(request, res, self.reactor)
 
         self.assertEqual(request.args, {b'a': [u"\N{SNOWMAN}".encode('utf8')]})
         self.assertEqual(got_kwargs, {u"room_id": u"\N{SNOWMAN}"})
@@ -55,7 +55,7 @@ def _callback(request, **kwargs):
         res.register_paths("GET", [re.compile("^/_matrix/foo$")], _callback)
 
         request, channel = make_request(b"GET", b"/_matrix/foo")
-        request.render(res)
+        render(request, res, self.reactor)
 
         self.assertEqual(channel.result["code"], b'500')
 
@@ -78,13 +78,8 @@ def _callback(request, **kwargs):
         res.register_paths("GET", [re.compile("^/_matrix/foo$")], _callback)
 
         request, channel = make_request(b"GET", b"/_matrix/foo")
-        request.render(res)
+        render(request, res, self.reactor)
 
-        # No error has been raised yet
-        self.assertTrue("code" not in channel.result)
-
-        # Advance time, now there's an error
-        self.reactor.advance(1)
         self.assertEqual(channel.result["code"], b'500')
 
     def test_callback_synapseerror(self):
@@ -100,7 +95,7 @@ def _callback(request, **kwargs):
         res.register_paths("GET", [re.compile("^/_matrix/foo$")], _callback)
 
         request, channel = make_request(b"GET", b"/_matrix/foo")
-        request.render(res)
+        render(request, res, self.reactor)
 
         self.assertEqual(channel.result["code"], b'403')
         self.assertEqual(channel.json_body["error"], "Forbidden!!one!")
@@ -121,7 +116,7 @@ def _callback(request, **kwargs):
         res.register_paths("GET", [re.compile("^/_matrix/foo$")], _callback)
 
         request, channel = make_request(b"GET", b"/_matrix/foobar")
-        request.render(res)
+        render(request, res, self.reactor)
 
         self.assertEqual(channel.result["code"], b'400')
         self.assertEqual(channel.json_body["error"], "Unrecognized request")

From d82fa0e9a6e64bf29b727fb73fff0a3aec20ce8a Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 15 Aug 2018 15:11:50 +0100
Subject: [PATCH 416/830] changelog

---
 changelog.d/3701.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3701.bugfix

diff --git a/changelog.d/3701.bugfix b/changelog.d/3701.bugfix
new file mode 100644
index 000000000000..c22de3453747
--- /dev/null
+++ b/changelog.d/3701.bugfix
@@ -0,0 +1 @@
+Avoid timing out requests while we are streaming back the response

From 2f78f432c421702e3756d029b3c669db837d8bcd Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@arasphere.net>
Date: Wed, 15 Aug 2018 17:35:22 +0200
Subject: [PATCH 417/830] speed up /members and add at= and membership params
 (#3568)

---
 changelog.d/3331.feature       |  1 +
 changelog.d/3567.feature       |  1 +
 changelog.d/3568.feature       |  1 +
 synapse/handlers/message.py    | 88 +++++++++++++++++++++++++++++-----
 synapse/rest/client/v1/room.py | 32 +++++++++++--
 synapse/storage/events.py      |  2 +-
 synapse/storage/state.py       | 66 ++++++++++++++++++++++++-
 tests/storage/test_state.py    |  2 +-
 8 files changed, 174 insertions(+), 19 deletions(-)
 create mode 100644 changelog.d/3331.feature
 create mode 100644 changelog.d/3567.feature
 create mode 100644 changelog.d/3568.feature

diff --git a/changelog.d/3331.feature b/changelog.d/3331.feature
new file mode 100644
index 000000000000..e574b9bcc3af
--- /dev/null
+++ b/changelog.d/3331.feature
@@ -0,0 +1 @@
+add support for the include_redundant_members filter param as per MSC1227
diff --git a/changelog.d/3567.feature b/changelog.d/3567.feature
new file mode 100644
index 000000000000..c74c1f57a90a
--- /dev/null
+++ b/changelog.d/3567.feature
@@ -0,0 +1 @@
+make the /context API filter & lazy-load aware as per MSC1227
diff --git a/changelog.d/3568.feature b/changelog.d/3568.feature
new file mode 100644
index 000000000000..247f02ba4e13
--- /dev/null
+++ b/changelog.d/3568.feature
@@ -0,0 +1 @@
+speed up /members API and add `at` and `membership` params as per MSC1227
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 01a362360e92..893c9bcdc4db 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -25,7 +25,13 @@
 from twisted.internet.defer import succeed
 
 from synapse.api.constants import MAX_DEPTH, EventTypes, Membership
-from synapse.api.errors import AuthError, Codes, ConsentNotGivenError, SynapseError
+from synapse.api.errors import (
+    AuthError,
+    Codes,
+    ConsentNotGivenError,
+    NotFoundError,
+    SynapseError,
+)
 from synapse.api.urls import ConsentURIBuilder
 from synapse.crypto.event_signing import add_hashes_and_signatures
 from synapse.events.utils import serialize_event
@@ -36,6 +42,7 @@
 from synapse.util.frozenutils import frozendict_json_encoder
 from synapse.util.logcontext import run_in_background
 from synapse.util.metrics import measure_func
+from synapse.visibility import filter_events_for_client
 
 from ._base import BaseHandler
 
@@ -82,28 +89,85 @@ def get_room_data(self, user_id=None, room_id=None,
         defer.returnValue(data)
 
     @defer.inlineCallbacks
-    def get_state_events(self, user_id, room_id, is_guest=False):
+    def get_state_events(
+        self, user_id, room_id, types=None, filtered_types=None,
+        at_token=None, is_guest=False,
+    ):
         """Retrieve all state events for a given room. If the user is
         joined to the room then return the current state. If the user has
-        left the room return the state events from when they left.
+        left the room return the state events from when they left. If an explicit
+        'at' parameter is passed, return the state events as of that event, if
+        visible.
 
         Args:
             user_id(str): The user requesting state events.
             room_id(str): The room ID to get all state events from.
+            types(list[(str, str|None)]|None): List of (type, state_key) tuples
+                which are used to filter the state fetched. If `state_key` is None,
+                all events are returned of the given type.
+                May be None, which matches any key.
+            filtered_types(list[str]|None): Only apply filtering via `types` to this
+                list of event types.  Other types of events are returned unfiltered.
+                If None, `types` filtering is applied to all events.
+            at_token(StreamToken|None): the stream token of the at which we are requesting
+                the stats. If the user is not allowed to view the state as of that
+                stream token, we raise a 403 SynapseError. If None, returns the current
+                state based on the current_state_events table.
+            is_guest(bool): whether this user is a guest
         Returns:
             A list of dicts representing state events. [{}, {}, {}]
+        Raises:
+            NotFoundError (404) if the at token does not yield an event
+
+            AuthError (403) if the user doesn't have permission to view
+            members of this room.
         """
-        membership, membership_event_id = yield self.auth.check_in_room_or_world_readable(
-            room_id, user_id
-        )
+        if at_token:
+            # FIXME this claims to get the state at a stream position, but
+            # get_recent_events_for_room operates by topo ordering. This therefore
+            # does not reliably give you the state at the given stream position.
+            # (https://github.com/matrix-org/synapse/issues/3305)
+            last_events, _ = yield self.store.get_recent_events_for_room(
+                room_id, end_token=at_token.room_key, limit=1,
+            )
 
-        if membership == Membership.JOIN:
-            room_state = yield self.state.get_current_state(room_id)
-        elif membership == Membership.LEAVE:
-            room_state = yield self.store.get_state_for_events(
-                [membership_event_id], None
+            if not last_events:
+                raise NotFoundError("Can't find event for token %s" % (at_token, ))
+
+            visible_events = yield filter_events_for_client(
+                self.store, user_id, last_events,
+            )
+
+            event = last_events[0]
+            if visible_events:
+                room_state = yield self.store.get_state_for_events(
+                    [event.event_id], types, filtered_types=filtered_types,
+                )
+                room_state = room_state[event.event_id]
+            else:
+                raise AuthError(
+                    403,
+                    "User %s not allowed to view events in room %s at token %s" % (
+                        user_id, room_id, at_token,
+                    )
+                )
+        else:
+            membership, membership_event_id = (
+                yield self.auth.check_in_room_or_world_readable(
+                    room_id, user_id,
+                )
             )
-            room_state = room_state[membership_event_id]
+
+            if membership == Membership.JOIN:
+                state_ids = yield self.store.get_filtered_current_state_ids(
+                    room_id, types, filtered_types=filtered_types,
+                )
+                room_state = yield self.store.get_events(state_ids.values())
+            elif membership == Membership.LEAVE:
+                room_state = yield self.store.get_state_for_events(
+                    [membership_event_id], types, filtered_types=filtered_types,
+                )
+                room_state = room_state[membership_event_id]
 
         now = self.clock.time_msec()
         defer.returnValue(
diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py
index fa5989e74e8c..fcc109176014 100644
--- a/synapse/rest/client/v1/room.py
+++ b/synapse/rest/client/v1/room.py
@@ -34,7 +34,7 @@
     parse_string,
 )
 from synapse.streams.config import PaginationConfig
-from synapse.types import RoomAlias, RoomID, ThirdPartyInstanceID, UserID
+from synapse.types import RoomAlias, RoomID, StreamToken, ThirdPartyInstanceID, UserID
 
 from .base import ClientV1RestServlet, client_path_patterns
 
@@ -384,15 +384,39 @@ def __init__(self, hs):
     def on_GET(self, request, room_id):
         # TODO support Pagination stream API (limit/tokens)
         requester = yield self.auth.get_user_by_req(request)
-        events = yield self.message_handler.get_state_events(
+        handler = self.message_handler
+
+        # request the state as of a given event, as identified by a stream token,
+        # for consistency with /messages etc.
+        # useful for getting the membership in retrospect as of a given /sync
+        # response.
+        at_token_string = parse_string(request, "at")
+        if at_token_string is None:
+            at_token = None
+        else:
+            at_token = StreamToken.from_string(at_token_string)
+
+        # let you filter down on particular memberships.
+        # XXX: this may not be the best shape for this API - we could pass in a filter
+        # instead, except filters aren't currently aware of memberships.
+        # See https://github.com/matrix-org/matrix-doc/issues/1337 for more details.
+        membership = parse_string(request, "membership")
+        not_membership = parse_string(request, "not_membership")
+
+        events = yield handler.get_state_events(
             room_id=room_id,
             user_id=requester.user.to_string(),
+            at_token=at_token,
+            types=[(EventTypes.Member, None)],
         )
 
         chunk = []
 
         for event in events:
-            if event["type"] != EventTypes.Member:
+            if (
+                (membership and event['content'].get("membership") != membership) or
+                (not_membership and event['content'].get("membership") == not_membership)
+            ):
                 continue
             chunk.append(event)
 
@@ -401,6 +425,8 @@ def on_GET(self, request, room_id):
         }))
 
 
+# deprecated in favour of /members?membership=join?
+# except it does custom AS logic and has a simpler return format
 class JoinedRoomMemberListRestServlet(ClientV1RestServlet):
     PATTERNS = client_path_patterns("/rooms/(?P<room_id>[^/]*)/joined_members$")
 
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 135af54fa936..025a7fb6d979 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -1911,7 +1911,7 @@ def _purge_history_txn(
         max_depth = max(row[0] for row in rows)
 
         if max_depth <= token.topological:
-            # We need to ensure we don't delete all the events from the datanase
+            # We need to ensure we don't delete all the events from the database
             # otherwise we wouldn't be able to send any events (due to not
             # having any backwards extremeties)
             raise SynapseError(
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index 17b14d464baf..754dfa6973fc 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -116,6 +116,69 @@ def _get_current_state_ids_txn(txn):
             _get_current_state_ids_txn,
         )
 
+    # FIXME: how should this be cached?
+    def get_filtered_current_state_ids(self, room_id, types, filtered_types=None):
+        """Get the current state event of a given type for a room based on the
+        current_state_events table.  This may not be as up-to-date as the result
+        of doing a fresh state resolution as per state_handler.get_current_state
+        Args:
+            room_id (str)
+            types (list[(Str, (Str|None))]): List of (type, state_key) tuples
+                which are used to filter the state fetched. `state_key` may be
+                None, which matches any `state_key`
+            filtered_types (list[Str]|None): List of types to apply the above filter to.
+        Returns:
+            deferred: dict of (type, state_key) -> event
+        """
+
+        include_other_types = False if filtered_types is None else True
+
+        def _get_filtered_current_state_ids_txn(txn):
+            results = {}
+            sql = """SELECT type, state_key, event_id FROM current_state_events
+                     WHERE room_id = ? %s"""
+            # Turns out that postgres doesn't like doing a list of OR's and
+            # is about 1000x slower, so we just issue a query for each specific
+            # type seperately.
+            if types:
+                clause_to_args = [
+                    (
+                        "AND type = ? AND state_key = ?",
+                        (etype, state_key)
+                    ) if state_key is not None else (
+                        "AND type = ?",
+                        (etype,)
+                    )
+                    for etype, state_key in types
+                ]
+
+                if include_other_types:
+                    unique_types = set(filtered_types)
+                    clause_to_args.append(
+                        (
+                            "AND type <> ? " * len(unique_types),
+                            list(unique_types)
+                        )
+                    )
+            else:
+                # If types is None we fetch all the state, and so just use an
+                # empty where clause with no extra args.
+                clause_to_args = [("", [])]
+            for where_clause, where_args in clause_to_args:
+                args = [room_id]
+                args.extend(where_args)
+                txn.execute(sql % (where_clause,), args)
+                for row in txn:
+                    typ, state_key, event_id = row
+                    key = (intern_string(typ), intern_string(state_key))
+                    results[key] = event_id
+            return results
+
+        return self.runInteraction(
+            "get_filtered_current_state_ids",
+            _get_filtered_current_state_ids_txn,
+        )
+
     @cached(max_entries=10000, iterable=True)
     def get_state_group_delta(self, state_group):
         """Given a state group try to return a previous group and a delta between
@@ -389,8 +452,7 @@ def get_state_for_events(self, event_ids, types, filtered_types=None):
                 If None, `types` filtering is applied to all events.
 
         Returns:
-            deferred: A list of dicts corresponding to the event_ids given.
-            The dicts are mappings from (type, state_key) -> state_events
+            deferred: A dict of (event_id) -> (type, state_key) -> [state_events]
         """
         event_to_groups = yield self._get_state_group_for_events(
             event_ids,
diff --git a/tests/storage/test_state.py b/tests/storage/test_state.py
index 6168c46248d0..ebfd969b36bb 100644
--- a/tests/storage/test_state.py
+++ b/tests/storage/test_state.py
@@ -148,7 +148,7 @@ def test_get_state_for_event(self):
             {(e3.type, e3.state_key): e3, (e5.type, e5.state_key): e5}, state
         )
 
-        # check we can use filter_types to grab a specific room member
+        # check we can use filtered_types to grab a specific room member
         # without filtering out the other event types
         state = yield self.store.get_state_for_event(
             e5.event_id,

From e2e846628d110e49e4486c9a5b1f56734318d063 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 15 Aug 2018 16:38:46 +0100
Subject: [PATCH 418/830] Remove incorrectly re-added changelog files

These were removed for the 0.33.2 release, but were incorrectly re-added in
---
 changelog.d/3331.feature | 1 -
 changelog.d/3567.feature | 1 -
 2 files changed, 2 deletions(-)
 delete mode 100644 changelog.d/3331.feature
 delete mode 100644 changelog.d/3567.feature

diff --git a/changelog.d/3331.feature b/changelog.d/3331.feature
deleted file mode 100644
index e574b9bcc3af..000000000000
--- a/changelog.d/3331.feature
+++ /dev/null
@@ -1 +0,0 @@
-add support for the include_redundant_members filter param as per MSC1227
diff --git a/changelog.d/3567.feature b/changelog.d/3567.feature
deleted file mode 100644
index c74c1f57a90a..000000000000
--- a/changelog.d/3567.feature
+++ /dev/null
@@ -1 +0,0 @@
-make the /context API filter & lazy-load aware as per MSC1227

From da7fe43899a2ad743ab067d5e0c239cc109d4339 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 15 Aug 2018 17:03:30 +0100
Subject: [PATCH 419/830] Fix mau blocking calulation bug on login

---
 changelog.d/3689.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3689.bugfix

diff --git a/changelog.d/3689.bugfix b/changelog.d/3689.bugfix
new file mode 100644
index 000000000000..934d039836ad
--- /dev/null
+++ b/changelog.d/3689.bugfix
@@ -0,0 +1 @@
+Fix mau blocking calulation bug on login

From 3f543dc021dd9456c8ed2da7a1e4769a68c07729 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@arasphere.net>
Date: Thu, 16 Aug 2018 10:46:50 +0200
Subject: [PATCH 420/830] initial cut at a room summary API (#3574)

---
 changelog.d/3574.feature             |   1 +
 synapse/handlers/sync.py             | 159 +++++++++++++++++++++++++--
 synapse/rest/client/v2_alpha/sync.py |   1 +
 synapse/storage/_base.py             |   7 +-
 synapse/storage/state.py             |   5 +-
 synapse/storage/stream.py            |   4 +-
 6 files changed, 159 insertions(+), 18 deletions(-)
 create mode 100644 changelog.d/3574.feature

diff --git a/changelog.d/3574.feature b/changelog.d/3574.feature
new file mode 100644
index 000000000000..87ac32df7249
--- /dev/null
+++ b/changelog.d/3574.feature
@@ -0,0 +1 @@
+implement `summary` block in /sync response as per MSC688
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 3b21a04a5d34..ac3edf0cc9bc 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -75,6 +75,7 @@ class JoinedSyncResult(collections.namedtuple("JoinedSyncResult", [
     "ephemeral",
     "account_data",
     "unread_notifications",
+    "summary",
 ])):
     __slots__ = []
 
@@ -503,10 +504,142 @@ def get_state_at(self, room_id, stream_position, types=None, filtered_types=None
             state = {}
         defer.returnValue(state)
 
+    @defer.inlineCallbacks
+    def compute_summary(self, room_id, sync_config, batch, state, now_token):
+        """ Works out a room summary block for this room, summarising the number
+        of joined members in the room, and providing the 'hero' members if the
+        room has no name so clients can consistently name rooms.  Also adds
+        state events to 'state' if needed to describe the heroes.
+
+        Args:
+            room_id(str):
+            sync_config(synapse.handlers.sync.SyncConfig):
+            batch(synapse.handlers.sync.TimelineBatch): The timeline batch for
+                the room that will be sent to the user.
+            state(dict): dict of (type, state_key) -> Event as returned by
+                compute_state_delta
+            now_token(str): Token of the end of the current batch.
+
+        Returns:
+             A deferred dict describing the room summary
+        """
+
+        # FIXME: this promulgates https://github.com/matrix-org/synapse/issues/3305
+        last_events, _ = yield self.store.get_recent_event_ids_for_room(
+            room_id, end_token=now_token.room_key, limit=1,
+        )
+
+        if not last_events:
+            defer.returnValue(None)
+            return
+
+        last_event = last_events[-1]
+        state_ids = yield self.store.get_state_ids_for_event(
+            last_event.event_id, [
+                (EventTypes.Member, None),
+                (EventTypes.Name, ''),
+                (EventTypes.CanonicalAlias, ''),
+            ]
+        )
+
+        member_ids = {
+            state_key: event_id
+            for (t, state_key), event_id in state_ids.iteritems()
+            if t == EventTypes.Member
+        }
+        name_id = state_ids.get((EventTypes.Name, ''))
+        canonical_alias_id = state_ids.get((EventTypes.CanonicalAlias, ''))
+
+        summary = {}
+
+        # FIXME: it feels very heavy to load up every single membership event
+        # just to calculate the counts.
+        member_events = yield self.store.get_events(member_ids.values())
+
+        joined_user_ids = []
+        invited_user_ids = []
+
+        for ev in member_events.values():
+            if ev.content.get("membership") == Membership.JOIN:
+                joined_user_ids.append(ev.state_key)
+            elif ev.content.get("membership") == Membership.INVITE:
+                invited_user_ids.append(ev.state_key)
+
+        # TODO: only send these when they change.
+        summary["m.joined_member_count"] = len(joined_user_ids)
+        summary["m.invited_member_count"] = len(invited_user_ids)
+
+        if name_id or canonical_alias_id:
+            defer.returnValue(summary)
+
+        # FIXME: order by stream ordering, not alphabetic
+
+        me = sync_config.user.to_string()
+        if (joined_user_ids or invited_user_ids):
+            summary['m.heroes'] = sorted(
+                [
+                    user_id
+                    for user_id in (joined_user_ids + invited_user_ids)
+                    if user_id != me
+                ]
+            )[0:5]
+        else:
+            summary['m.heroes'] = sorted(
+                [user_id for user_id in member_ids.keys() if user_id != me]
+            )[0:5]
+
+        if not sync_config.filter_collection.lazy_load_members():
+            defer.returnValue(summary)
+
+        # ensure we send membership events for heroes if needed
+        cache_key = (sync_config.user.to_string(), sync_config.device_id)
+        cache = self.get_lazy_loaded_members_cache(cache_key)
+
+        # track which members the client should already know about via LL:
+        # Ones which are already in state...
+        existing_members = set(
+            user_id for (typ, user_id) in state.keys()
+            if typ == EventTypes.Member
+        )
+
+        # ...or ones which are in the timeline...
+        for ev in batch.events:
+            if ev.type == EventTypes.Member:
+                existing_members.add(ev.state_key)
+
+        # ...and then ensure any missing ones get included in state.
+        missing_hero_event_ids = [
+            member_ids[hero_id]
+            for hero_id in summary['m.heroes']
+            if (
+                cache.get(hero_id) != member_ids[hero_id] and
+                hero_id not in existing_members
+            )
+        ]
+
+        missing_hero_state = yield self.store.get_events(missing_hero_event_ids)
+        missing_hero_state = missing_hero_state.values()
+
+        for s in missing_hero_state:
+            cache.set(s.state_key, s.event_id)
+            state[(EventTypes.Member, s.state_key)] = s
+
+        defer.returnValue(summary)
+
+    def get_lazy_loaded_members_cache(self, cache_key):
+        cache = self.lazy_loaded_members_cache.get(cache_key)
+        if cache is None:
+            logger.debug("creating LruCache for %r", cache_key)
+            cache = LruCache(LAZY_LOADED_MEMBERS_CACHE_MAX_SIZE)
+            self.lazy_loaded_members_cache[cache_key] = cache
+        else:
+            logger.debug("found LruCache for %r", cache_key)
+        return cache
+
     @defer.inlineCallbacks
     def compute_state_delta(self, room_id, batch, sync_config, since_token, now_token,
                             full_state):
-        """ Works out the differnce in state between the start of the timeline
+        """ Works out the difference in state between the start of the timeline
         and the previous sync.
 
         Args:
@@ -520,7 +653,7 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
             full_state(bool): Whether to force returning the full state.
 
         Returns:
-             A deferred new event dictionary
+             A deferred dict of (type, state_key) -> Event
         """
         # TODO(mjark) Check if the state events were received by the server
         # after the previous sync, since we need to include those state
@@ -618,13 +751,7 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
 
             if lazy_load_members and not include_redundant_members:
                 cache_key = (sync_config.user.to_string(), sync_config.device_id)
-                cache = self.lazy_loaded_members_cache.get(cache_key)
-                if cache is None:
-                    logger.debug("creating LruCache for %r", cache_key)
-                    cache = LruCache(LAZY_LOADED_MEMBERS_CACHE_MAX_SIZE)
-                    self.lazy_loaded_members_cache[cache_key] = cache
-                else:
-                    logger.debug("found LruCache for %r", cache_key)
+                cache = self.get_lazy_loaded_members_cache(cache_key)
 
                 # if it's a new sync sequence, then assume the client has had
                 # amnesia and doesn't want any recent lazy-loaded members
@@ -1425,7 +1552,6 @@ def _generate_room_entry(self, sync_result_builder, ignored_users,
             if events == [] and tags is None:
                 return
 
-        since_token = sync_result_builder.since_token
         now_token = sync_result_builder.now_token
         sync_config = sync_result_builder.sync_config
 
@@ -1468,6 +1594,18 @@ def _generate_room_entry(self, sync_result_builder, ignored_users,
             full_state=full_state
         )
 
+        summary = {}
+        if (
+            sync_config.filter_collection.lazy_load_members() and
+            (
+                any(ev.type == EventTypes.Member for ev in batch.events) or
+                since_token is None
+            )
+        ):
+            summary = yield self.compute_summary(
+                room_id, sync_config, batch, state, now_token
+            )
+
         if room_builder.rtype == "joined":
             unread_notifications = {}
             room_sync = JoinedSyncResult(
@@ -1477,6 +1615,7 @@ def _generate_room_entry(self, sync_result_builder, ignored_users,
                 ephemeral=ephemeral,
                 account_data=account_data_events,
                 unread_notifications=unread_notifications,
+                summary=summary,
             )
 
             if room_sync or always_include:
diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py
index 8aa06faf23d7..1275baa1ba9a 100644
--- a/synapse/rest/client/v2_alpha/sync.py
+++ b/synapse/rest/client/v2_alpha/sync.py
@@ -370,6 +370,7 @@ def serialize(event):
             ephemeral_events = room.ephemeral
             result["ephemeral"] = {"events": ephemeral_events}
             result["unread_notifications"] = room.unread_notifications
+            result["summary"] = room.summary
 
         return result
 
diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 44f37b4c1ee8..08dffd774f81 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -1150,17 +1150,16 @@ def get_user_list_paginate(self, table, keyvalues, pagevalues, retcols,
         defer.returnValue(retval)
 
     def get_user_count_txn(self, txn):
-        """Get a total number of registerd users in the users list.
+        """Get a total number of registered users in the users list.
 
         Args:
             txn : Transaction object
         Returns:
-            defer.Deferred: resolves to int
+            int : number of users
         """
         sql_count = "SELECT COUNT(*) FROM users WHERE is_guest = 0;"
         txn.execute(sql_count)
-        count = txn.fetchone()[0]
-        defer.returnValue(count)
+        return txn.fetchone()[0]
 
     def _simple_search_list(self, table, term, col, retcols,
                             desc="_simple_search_list"):
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index 754dfa6973fc..dd03c4168b6c 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -480,7 +480,8 @@ def get_state_for_events(self, event_ids, types, filtered_types=None):
     @defer.inlineCallbacks
     def get_state_ids_for_events(self, event_ids, types=None, filtered_types=None):
         """
-        Get the state dicts corresponding to a list of events
+        Get the state dicts corresponding to a list of events, containing the event_ids
+        of the state events (as opposed to the events themselves)
 
         Args:
             event_ids(list(str)): events whose state should be returned
@@ -493,7 +494,7 @@ def get_state_ids_for_events(self, event_ids, types=None, filtered_types=None):
                 If None, `types` filtering is applied to all events.
 
         Returns:
-            A deferred dict from event_id -> (type, state_key) -> state_event
+            A deferred dict from event_id -> (type, state_key) -> event_id
         """
         event_to_groups = yield self._get_state_group_for_events(
             event_ids,
diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py
index b9f2b74ac64b..4c296d72c0f8 100644
--- a/synapse/storage/stream.py
+++ b/synapse/storage/stream.py
@@ -348,7 +348,7 @@ def get_recent_events_for_room(self, room_id, limit, end_token):
             end_token (str): The stream token representing now.
 
         Returns:
-            Deferred[tuple[list[FrozenEvent],  str]]: Returns a list of
+            Deferred[tuple[list[FrozenEvent], str]]: Returns a list of
             events and a token pointing to the start of the returned
             events.
             The events returned are in ascending order.
@@ -379,7 +379,7 @@ def get_recent_event_ids_for_room(self, room_id, limit, end_token):
             end_token (str): The stream token representing now.
 
         Returns:
-            Deferred[tuple[list[_EventDictReturn],  str]]: Returns a list of
+            Deferred[tuple[list[_EventDictReturn], str]]: Returns a list of
             _EventDictReturn and a token pointing to the start of the returned
             events.
             The events returned are in ascending order.

From c055c91655cbe5d4e6b36d86a00dafb02c585b6d Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Thu, 16 Aug 2018 11:10:19 +0100
Subject: [PATCH 421/830] fix case where empty string state check is evaulated
 as False

---
 synapse/server_notices/server_notices_manager.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py
index 3e57f8211ae9..22c819cc5ba2 100644
--- a/synapse/server_notices/server_notices_manager.py
+++ b/synapse/server_notices/server_notices_manager.py
@@ -76,7 +76,8 @@ def send_notice(
             "sender": system_mxid,
             "content": event_content,
         }
-        if state_key:
+
+        if state_key is not None:
             event_dict['state_key'] = state_key
 
         res = yield self._event_creation_handler.create_and_send_nonmember_event(

From df1e4f259fda3009dacbb3211e999f7d89adaf87 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Thu, 16 Aug 2018 11:10:53 +0100
Subject: [PATCH 422/830] WIP impl commiting to get feedback

---
 .../resource_limits_server_notices.py         | 80 +++++++++++++++++--
 1 file changed, 74 insertions(+), 6 deletions(-)

diff --git a/synapse/server_notices/resource_limits_server_notices.py b/synapse/server_notices/resource_limits_server_notices.py
index 94d0f98189de..2b714a1016e5 100644
--- a/synapse/server_notices/resource_limits_server_notices.py
+++ b/synapse/server_notices/resource_limits_server_notices.py
@@ -40,7 +40,8 @@ def __init__(self, hs):
 
         self._notified_of_blocking = set()
         self._resouce_limited = False
-
+        self._message_handler = hs.get_message_handler()
+        self._state = hs.get_state_handler()
         # Config checks?
 
     @defer.inlineCallbacks
@@ -57,6 +58,72 @@ def maybe_send_server_notice_to_user(self, user_id):
             return
 
         if self._limit_usage_by_mau is True:
+            room_id = yield self._server_notices_manager.get_notice_room_for_user(user_id)
+
+
+            # Alternate impl - currently inlcuded because I'm not sure I am on
+            # the right track and want to share WIP
+
+            # logger.info("GET STATE EVENTS")
+            # currently_blocked = False
+            # events = []
+            # try:
+            #     events = yield self._message_handler.get_state_events(user_id, room_id, types=[(EventTypes.Pinned, None)])
+            # except AuthError as e:
+            #     # The user has yet to join the server notices room
+            #     pass
+            #
+            # pinned_event_refs = []
+            # for e in events:
+            #     logger.info('events %s' % e)
+            #     logger.info(type(e))
+            #     for key, event_ids in e['content'].items():
+            #         logger.info('Key Event %s %s' % (key, event_ids))
+            #         if key == 'pinned':
+            #             pinned_event_refs = event_ids
+            #
+            # logger.info('pinned_event_refs %s' % pinned_event_refs)
+            #
+            # events = yield self._store.get_events(pinned_event_refs)
+            # logger.info(events)
+            # for event_id, event in events.items():
+            #     logger.info("event_id, event event.type %s %s %s" % (event_id, event, event.type))
+            #     if event.type == 'm.server_notice.usage_limit_reached':
+            #         currently_blocked = True
+            #
+            # logger.info('Currently Blocked is %r' % currently_blocked)
+
+            #for e in events:
+            #    logger.info(e)
+            currently_blocked = False
+            logger.info("GET CURRENT STATE")
+            pinned_state_event = yield self._state.get_current_state(room_id, event_type=EventTypes.Pinned)
+            logger.info(events)
+            logger.info(events.get('content'))
+
+            referenced_events = []
+            if pinned_state_event is not None:
+                content = pinned_state_event.get('content')
+                if content is not None:
+                    referenced_events = content.get('pinned')
+
+            events = yield self._store.get_events(referenced_events)
+            logger.info(events)
+            for event_id, event in events.items():
+                logger.info("event_id, event event.type %s %s %s" % (event_id, event, event.type))
+                if event.type == 'm.server_notice.usage_limit_reached':
+                    currently_blocked = True
+
+            logger.info("currently_blocked is %r" % currently_blocked)
+
+                #event = yield self._store.get_event(events.event_id)
+                #logger.info(event)
+
+            #logger.info("GET CURRENT STATE IDs")
+            #events = yield self._state.get_current_state_ids(room_id)
+            #for k,v in events.items():
+            #    logger.info('%s %s' % (k,v))
+
             timestamp = yield self._store.user_last_seen_monthly_active(user_id)
             if timestamp is None:
                 # This user will be blocked from receiving the notice anyway.
@@ -69,7 +136,8 @@ def maybe_send_server_notice_to_user(self, user_id):
                 yield self.auth.check_auth_blocking()
                 self._resouce_limited = False
                 # Need to start removing notices
-                if user_id in self._notified_of_blocking:
+                # if user_id in self._notified_of_blocking:
+                if currently_blocked:
                     # Send message to remove warning
                     # send state event here
                     # How do I do this? if drop the id, how to refer to it?
@@ -79,14 +147,14 @@ def maybe_send_server_notice_to_user(self, user_id):
                     yield self._server_notices_manager.send_notice(
                         user_id, content, EventTypes.Pinned, '',
                     )
-
-                    self._notified_of_blocking.remove(user_id)
+                    logger.info('deactivate block')
 
             except AuthError as e:
                 # Need to start notifying of blocking
                 try:
                     self._resouce_limited = True
-                    if user_id not in self._notified_of_blocking:
+                    #if user_id not in self._notified_of_blocking:
+                    if not currently_blocked:
                         # TODO use admin email contained in error once PR lands
                         content = {
                             'body': e.msg,
@@ -103,10 +171,10 @@ def maybe_send_server_notice_to_user(self, user_id):
                                 event.event_id,
                             ]
                         }
+                        logger.info("active block")
                         yield self._server_notices_manager.send_notice(
                             user_id, content, EventTypes.Pinned, '',
                         )
 
-                        self._notified_of_blocking.add(user_id)
                 except SynapseError as e:
                     logger.error("Error sending server notice about resource limits: %s", e)

From 762a758feae40ccb5baf0ba4808308e99e73e13f Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@arasphere.net>
Date: Thu, 16 Aug 2018 15:22:47 +0200
Subject: [PATCH 423/830] lazyload aware /messages (#3589)

---
 changelog.d/3589.feature        |  1 +
 synapse/handlers/pagination.py  | 35 ++++++++++++++++++++++++++++++++-
 synapse/rest/client/versions.py | 13 +++++++++++-
 3 files changed, 47 insertions(+), 2 deletions(-)
 create mode 100644 changelog.d/3589.feature

diff --git a/changelog.d/3589.feature b/changelog.d/3589.feature
new file mode 100644
index 000000000000..a8d7124719e0
--- /dev/null
+++ b/changelog.d/3589.feature
@@ -0,0 +1 @@
+Add lazy-loading support to /messages as per MSC1227
diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py
index a97d43550f6c..5170d093e365 100644
--- a/synapse/handlers/pagination.py
+++ b/synapse/handlers/pagination.py
@@ -18,7 +18,7 @@
 from twisted.internet import defer
 from twisted.python.failure import Failure
 
-from synapse.api.constants import Membership
+from synapse.api.constants import EventTypes, Membership
 from synapse.api.errors import SynapseError
 from synapse.events.utils import serialize_event
 from synapse.types import RoomStreamToken
@@ -251,6 +251,33 @@ def get_messages(self, requester, room_id=None, pagin_config=None,
             is_peeking=(member_event_id is None),
         )
 
+        state = None
+        if event_filter and event_filter.lazy_load_members():
+            # TODO: remove redundant members
+
+            types = [
+                (EventTypes.Member, state_key)
+                for state_key in set(
+                    event.sender  # FIXME: we also care about invite targets etc.
+                    for event in events
+                )
+            ]
+
+            state_ids = yield self.store.get_state_ids_for_event(
+                events[0].event_id, types=types,
+            )
+
+            if state_ids:
+                state = yield self.store.get_events(list(state_ids.values()))
+
+            if state:
+                state = yield filter_events_for_client(
+                    self.store,
+                    user_id,
+                    state.values(),
+                    is_peeking=(member_event_id is None),
+                )
+
         time_now = self.clock.time_msec()
 
         chunk = {
@@ -262,4 +289,10 @@ def get_messages(self, requester, room_id=None, pagin_config=None,
             "end": next_token.to_string(),
         }
 
+        if state:
+            chunk["state"] = [
+                serialize_event(e, time_now, as_client_event)
+                for e in state
+            ]
+
         defer.returnValue(chunk)
diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py
index 6ac2987b98c8..29e62bfcddea 100644
--- a/synapse/rest/client/versions.py
+++ b/synapse/rest/client/versions.py
@@ -27,11 +27,22 @@ class VersionsRestServlet(RestServlet):
     def on_GET(self, request):
         return (200, {
             "versions": [
+                # XXX: at some point we need to decide whether we need to include
+                # the previous version numbers, given we've defined r0.3.0 to be
+                # backwards compatible with r0.2.0.  But need to check how
+                # conscientious we've been in compatibility, and decide whether the
+                # middle number is the major revision when at 0.X.Y (as opposed to
+                # X.Y.Z).  And we need to decide whether it's fair to make clients
+                # parse the version string to figure out what's going on.
                 "r0.0.1",
                 "r0.1.0",
                 "r0.2.0",
                 "r0.3.0",
-            ]
+            ],
+            # as per MSC1497:
+            "unstable_features": {
+                "m.lazy_load_members": True,
+            }
         })
 
 

From c151b32b1de7a464620d3cc7a827b50dd83388ed Mon Sep 17 00:00:00 2001
From: Will Hunt <will@half-shot.uk>
Date: Thu, 16 Aug 2018 14:23:38 +0100
Subject: [PATCH 424/830] Add GET media/v1/config (#3184)

---
 changelog.d/3184.feature                  |  1 +
 synapse/rest/media/v1/config_resource.py  | 48 +++++++++++++++++++++++
 synapse/rest/media/v1/media_repository.py |  3 +-
 3 files changed, 51 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/3184.feature
 create mode 100644 synapse/rest/media/v1/config_resource.py

diff --git a/changelog.d/3184.feature b/changelog.d/3184.feature
new file mode 100644
index 000000000000..9f746a57a0fa
--- /dev/null
+++ b/changelog.d/3184.feature
@@ -0,0 +1 @@
+Add /_media/r0/config
diff --git a/synapse/rest/media/v1/config_resource.py b/synapse/rest/media/v1/config_resource.py
new file mode 100644
index 000000000000..d6605b6027c4
--- /dev/null
+++ b/synapse/rest/media/v1/config_resource.py
@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 Will Hunt <will@half-shot.uk>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from twisted.internet import defer
+from twisted.web.resource import Resource
+from twisted.web.server import NOT_DONE_YET
+
+from synapse.http.server import respond_with_json, wrap_json_request_handler
+
+
+class MediaConfigResource(Resource):
+    isLeaf = True
+
+    def __init__(self, hs):
+        Resource.__init__(self)
+        config = hs.get_config()
+        self.clock = hs.get_clock()
+        self.auth = hs.get_auth()
+        self.limits_dict = {
+            "m.upload.size": config.max_upload_size,
+        }
+
+    def render_GET(self, request):
+        self._async_render_GET(request)
+        return NOT_DONE_YET
+
+    @wrap_json_request_handler
+    @defer.inlineCallbacks
+    def _async_render_GET(self, request):
+        yield self.auth.get_user_by_req(request)
+        respond_with_json(request, 200, self.limits_dict)
+
+    def render_OPTIONS(self, request):
+        respond_with_json(request, 200, {}, send_cors=True)
+        return NOT_DONE_YET
diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py
index 4c589e05e07d..241c972070e0 100644
--- a/synapse/rest/media/v1/media_repository.py
+++ b/synapse/rest/media/v1/media_repository.py
@@ -42,6 +42,7 @@
 from synapse.util.stringutils import is_ascii, random_string
 
 from ._base import FileInfo, respond_404, respond_with_responder
+from .config_resource import MediaConfigResource
 from .download_resource import DownloadResource
 from .filepath import MediaFilePaths
 from .identicon_resource import IdenticonResource
@@ -754,7 +755,6 @@ def __init__(self, hs):
         Resource.__init__(self)
 
         media_repo = hs.get_media_repository()
-
         self.putChild("upload", UploadResource(hs, media_repo))
         self.putChild("download", DownloadResource(hs, media_repo))
         self.putChild("thumbnail", ThumbnailResource(
@@ -765,3 +765,4 @@ def __init__(self, hs):
             self.putChild("preview_url", PreviewUrlResource(
                 hs, media_repo, media_repo.media_storage,
             ))
+        self.putChild("config", MediaConfigResource(hs))

From a675f9c5560a16655a9337992499690c9f27a7eb Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Thu, 16 Aug 2018 14:53:35 +0100
Subject: [PATCH 425/830] check for room state before deciding on action

---
 .../server_notices/consent_server_notices.py  |  1 -
 .../resource_limits_server_notices.py         | 92 ++++++-------------
 .../test_resource_limits_server_notices.py    | 52 +++++++----
 3 files changed, 58 insertions(+), 87 deletions(-)

diff --git a/synapse/server_notices/consent_server_notices.py b/synapse/server_notices/consent_server_notices.py
index 783ceae1ad27..a9fe58d9ffdd 100644
--- a/synapse/server_notices/consent_server_notices.py
+++ b/synapse/server_notices/consent_server_notices.py
@@ -22,7 +22,6 @@
 from synapse.api.urls import ConsentURIBuilder
 from synapse.config import ConfigError
 from synapse.types import get_localpart_from_id
-from synapse.api.constants import EventTypes
 
 logger = logging.getLogger(__name__)
 
diff --git a/synapse/server_notices/resource_limits_server_notices.py b/synapse/server_notices/resource_limits_server_notices.py
index 2b714a1016e5..43a926fc7975 100644
--- a/synapse/server_notices/resource_limits_server_notices.py
+++ b/synapse/server_notices/resource_limits_server_notices.py
@@ -16,8 +16,8 @@
 
 from twisted.internet import defer
 
-from synapse.api.errors import AuthError, SynapseError
 from synapse.api.constants import EventTypes
+from synapse.api.errors import AuthError, SynapseError
 
 logger = logging.getLogger(__name__)
 
@@ -35,10 +35,10 @@ def __init__(self, hs):
         self._store = hs.get_datastore()
         self.auth = hs.get_auth()
         self._server_notice_content = hs.config.user_consent_server_notice_content
+        self._admin_uri = hs.config.admin_uri
         self._limit_usage_by_mau = hs.config.limit_usage_by_mau
         self._hs_disabled = hs.config.hs_disabled
 
-        self._notified_of_blocking = set()
         self._resouce_limited = False
         self._message_handler = hs.get_message_handler()
         self._state = hs.get_state_handler()
@@ -58,83 +58,45 @@ def maybe_send_server_notice_to_user(self, user_id):
             return
 
         if self._limit_usage_by_mau is True:
-            room_id = yield self._server_notices_manager.get_notice_room_for_user(user_id)
+            timestamp = yield self._store.user_last_seen_monthly_active(user_id)
+            if timestamp is None:
+                # This user will be blocked from receiving the notice anyway.
+                # In practice, not sure we can ever get here
+                return
 
+            room_id = yield self._server_notices_manager.get_notice_room_for_user(user_id)
 
-            # Alternate impl - currently inlcuded because I'm not sure I am on
-            # the right track and want to share WIP
-
-            # logger.info("GET STATE EVENTS")
-            # currently_blocked = False
-            # events = []
-            # try:
-            #     events = yield self._message_handler.get_state_events(user_id, room_id, types=[(EventTypes.Pinned, None)])
-            # except AuthError as e:
-            #     # The user has yet to join the server notices room
-            #     pass
-            #
-            # pinned_event_refs = []
-            # for e in events:
-            #     logger.info('events %s' % e)
-            #     logger.info(type(e))
-            #     for key, event_ids in e['content'].items():
-            #         logger.info('Key Event %s %s' % (key, event_ids))
-            #         if key == 'pinned':
-            #             pinned_event_refs = event_ids
-            #
-            # logger.info('pinned_event_refs %s' % pinned_event_refs)
-            #
-            # events = yield self._store.get_events(pinned_event_refs)
-            # logger.info(events)
-            # for event_id, event in events.items():
-            #     logger.info("event_id, event event.type %s %s %s" % (event_id, event, event.type))
-            #     if event.type == 'm.server_notice.usage_limit_reached':
-            #         currently_blocked = True
-            #
-            # logger.info('Currently Blocked is %r' % currently_blocked)
-
-            #for e in events:
-            #    logger.info(e)
             currently_blocked = False
             logger.info("GET CURRENT STATE")
-            pinned_state_event = yield self._state.get_current_state(room_id, event_type=EventTypes.Pinned)
-            logger.info(events)
-            logger.info(events.get('content'))
+            pinned_state_event = None
+            try:
+                pinned_state_event = yield self._state.get_current_state(
+                    room_id, event_type=EventTypes.Pinned
+                )
+            except AuthError as e:
+                # The user has yet to join the server notices room
+                pass
 
             referenced_events = []
             if pinned_state_event is not None:
-                content = pinned_state_event.get('content')
-                if content is not None:
-                    referenced_events = content.get('pinned')
+                referenced_events = pinned_state_event.content.get('pinned')
 
             events = yield self._store.get_events(referenced_events)
             logger.info(events)
             for event_id, event in events.items():
-                logger.info("event_id, event event.type %s %s %s" % (event_id, event, event.type))
-                if event.type == 'm.server_notice.usage_limit_reached':
+                logger.info("event_id, event event.type %s %s %s" % (
+                    event_id, event, event.type)
+                )
+                if event.type == EventTypes.ServerNoticeLimitReached:
                     currently_blocked = True
 
             logger.info("currently_blocked is %r" % currently_blocked)
-
-                #event = yield self._store.get_event(events.event_id)
-                #logger.info(event)
-
-            #logger.info("GET CURRENT STATE IDs")
-            #events = yield self._state.get_current_state_ids(room_id)
-            #for k,v in events.items():
-            #    logger.info('%s %s' % (k,v))
-
-            timestamp = yield self._store.user_last_seen_monthly_active(user_id)
-            if timestamp is None:
-                # This user will be blocked from receiving the notice anyway.
-                # In practice, not sure we can ever get here
-                return
             try:
                 # Normally should always pass in user_id if you have it, but in
                 # this case are checking what would happen to other users if they
                 # were to arrive.
                 yield self.auth.check_auth_blocking()
-                self._resouce_limited = False
+
                 # Need to start removing notices
                 # if user_id in self._notified_of_blocking:
                 if currently_blocked:
@@ -142,7 +104,7 @@ def maybe_send_server_notice_to_user(self, user_id):
                     # send state event here
                     # How do I do this? if drop the id, how to refer to it?
                     content = {
-                        "pinned":[]
+                        "pinned": []
                     }
                     yield self._server_notices_manager.send_notice(
                         user_id, content, EventTypes.Pinned, '',
@@ -152,13 +114,11 @@ def maybe_send_server_notice_to_user(self, user_id):
             except AuthError as e:
                 # Need to start notifying of blocking
                 try:
-                    self._resouce_limited = True
-                    #if user_id not in self._notified_of_blocking:
                     if not currently_blocked:
                         # TODO use admin email contained in error once PR lands
                         content = {
                             'body': e.msg,
-                            'admin_email': 'stunt@adminemail.com',
+                            'admin_uri': self._admin_uri,
                         }
                         event = yield self._server_notices_manager.send_notice(
                             user_id, content, EventTypes.ServerNoticeLimitReached
@@ -167,7 +127,7 @@ def maybe_send_server_notice_to_user(self, user_id):
                         # send server notices state event here
                         # TODO Over writing pinned events
                         content = {
-                            "pinned":[
+                            "pinned": [
                                 event.event_id,
                             ]
                         }
@@ -177,4 +137,4 @@ def maybe_send_server_notice_to_user(self, user_id):
                         )
 
                 except SynapseError as e:
-                    logger.error("Error sending server notice about resource limits: %s", e)
+                    logger.error("Error sending resource limits server notice: %s", e)
diff --git a/tests/server_notices/test_resource_limits_server_notices.py b/tests/server_notices/test_resource_limits_server_notices.py
index a69253f1b6bf..fb8c593a75e8 100644
--- a/tests/server_notices/test_resource_limits_server_notices.py
+++ b/tests/server_notices/test_resource_limits_server_notices.py
@@ -2,6 +2,7 @@
 
 from twisted.internet import defer
 
+from synapse.api.constants import EventTypes
 from synapse.api.errors import AuthError
 from synapse.handlers.auth import AuthHandler
 from synapse.server_notices.resource_limits_server_notices import (
@@ -20,7 +21,7 @@ def __init__(self, hs):
 class TestResourceLimitsServerNotices(unittest.TestCase):
     @defer.inlineCallbacks
     def setUp(self):
-        self.hs = yield setup_test_homeserver(handlers=None)
+        self.hs = yield setup_test_homeserver(self.addCleanup, handlers=None)
         self.hs.handlers = AuthHandlers(self.hs)
         self.auth_handler = self.hs.handlers.auth_handler
         self.server_notices_sender = self.hs.get_server_notices_sender()
@@ -37,10 +38,23 @@ def setUp(self):
         )
         self._send_notice = self._rlsn._server_notices_manager.send_notice
         self._rlsn._server_notices_manager.send_notice = Mock()
+        self._rlsn._state.get_current_state = Mock(return_value=defer.succeed(None))
+        self._rlsn._store.get_events = Mock(return_value=defer.succeed({}))
+
         self._send_notice = self._rlsn._server_notices_manager.send_notice
 
         self._rlsn._limit_usage_by_mau = True
-        self.user_id = "user_id"
+        self.user_id = "@user_id:test"
+
+        self.server_notices_mxid = "@server:test"
+        self.server_notices_mxid_display_name = None
+        self.server_notices_mxid_avatar_url = None
+        self.server_notices_room_name = "Server Notices"
+
+        self._rlsn._server_notices_manager.get_notice_room_for_user = Mock(
+            returnValue=""
+        )
+        self.hs.config.admin_uri = "mailto:user@test.com"
 
     @defer.inlineCallbacks
     def test_maybe_send_server_notice_to_user_flag_off(self):
@@ -48,13 +62,13 @@ def test_maybe_send_server_notice_to_user_flag_off(self):
         # test hs disabled case
         self._hs_disabled = True
 
-        yield self._rlsn.maybe_send_server_notice_to_user("user_id")
+        yield self._rlsn.maybe_send_server_notice_to_user(self.user_id)
 
         self._send_notice.assert_not_called()
         # Test when mau limiting disabled
         self._hs_disabled = False
         self._rlsn._limit_usage_by_mau = False
-        yield self._rlsn.maybe_send_server_notice_to_user("user_id")
+        yield self._rlsn.maybe_send_server_notice_to_user(self.user_id)
 
         self._send_notice.assert_not_called()
 
@@ -62,40 +76,40 @@ def test_maybe_send_server_notice_to_user_flag_off(self):
     def test_maybe_send_server_notice_to_user_remove_blocked_notice(self):
         """Test when user has blocked notice, but should have it removed"""
 
-        self._rlsn._notified_of_blocking.add(self.user_id)
         self._rlsn.auth.check_auth_blocking = Mock()
+        mock_event = Mock(type=EventTypes.ServerNoticeLimitReached)
+        self._rlsn._store.get_events = Mock(return_value=defer.succeed(
+            {"123": mock_event}
+        ))
 
         yield self._rlsn.maybe_send_server_notice_to_user(self.user_id)
-        # "remove warning" obviously aweful, but test will start failing when code
-        # actually sends a real event, and then it can be updated
-
-        self._send_notice.assert_called_once_with(self.user_id, "remove warning")
-        self.assertFalse(self.user_id in self._rlsn._notified_of_blocking)
+        # Would be better to check the content, but once == remove blocking event
+        self._send_notice.assert_called_once()
 
     @defer.inlineCallbacks
     def test_maybe_send_server_notice_to_user_remove_blocked_notice_noop(self):
         """Test when user has blocked notice, but notice ought to be there (NOOP)"""
-        self._rlsn._notified_of_blocking.add(self.user_id)
         self._rlsn.auth.check_auth_blocking = Mock(
             side_effect=AuthError(403, 'foo')
         )
 
-        yield self._rlsn.maybe_send_server_notice_to_user("user_id")
+        mock_event = Mock(type=EventTypes.ServerNoticeLimitReached)
+        self._rlsn._store.get_events = Mock(return_value=defer.succeed(
+            {"123": mock_event}
+        ))
+        yield self._rlsn.maybe_send_server_notice_to_user(self.user_id)
 
         self._send_notice.assert_not_called()
-        self.assertTrue(self.user_id in self._rlsn._notified_of_blocking)
 
     @defer.inlineCallbacks
     def test_maybe_send_server_notice_to_user_add_blocked_notice(self):
         """Test when user does not have blocked notice, but should have one"""
 
         self._rlsn.auth.check_auth_blocking = Mock(side_effect=AuthError(403, 'foo'))
-        yield self._rlsn.maybe_send_server_notice_to_user("user_id")
+        yield self._rlsn.maybe_send_server_notice_to_user(self.user_id)
 
-        # "add warning" obviously awful, but test will start failing when code
-        # actually sends a real event, and then it can be updated
-        self._send_notice.assert_called_once_with(self.user_id, "add warning")
-        self.assertTrue(self.user_id in self._rlsn._notified_of_blocking)
+        # Would be better to check contents, but 2 calls == set blocking event
+        self.assertTrue(self._send_notice.call_count == 2)
 
     @defer.inlineCallbacks
     def test_maybe_send_server_notice_to_user_add_blocked_notice_noop(self):
@@ -106,7 +120,6 @@ def test_maybe_send_server_notice_to_user_add_blocked_notice_noop(self):
         yield self._rlsn.maybe_send_server_notice_to_user(self.user_id)
 
         self._send_notice.assert_not_called()
-        self.assertFalse(self.user_id in self._rlsn._notified_of_blocking)
 
     @defer.inlineCallbacks
     def test_maybe_send_server_notice_to_user_not_in_mau_cohort(self):
@@ -122,4 +135,3 @@ def test_maybe_send_server_notice_to_user_not_in_mau_cohort(self):
         yield self._rlsn.maybe_send_server_notice_to_user(self.user_id)
 
         self._send_notice.assert_not_called()
-        self.assertFalse(self.user_id in self._rlsn._notified_of_blocking)

From 7d5b1a60a3aa9f10e285e74295f407d19a3a9ad5 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 16 Aug 2018 15:32:20 +0100
Subject: [PATCH 426/830] Fix inbound federation on reader worker

Inbound federation requires calculating push, which in turn relies on
having access to account data.
---
 synapse/app/federation_reader.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py
index 52522e9d3350..7d8105778d55 100644
--- a/synapse/app/federation_reader.py
+++ b/synapse/app/federation_reader.py
@@ -32,6 +32,7 @@
 from synapse.metrics import RegistryProxy
 from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.replication.slave.storage._base import BaseSlavedStore
+from synapse.replication.slave.storage.account_data import SlavedAccountDataStore
 from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore
 from synapse.replication.slave.storage.directory import DirectoryStore
 from synapse.replication.slave.storage.events import SlavedEventStore
@@ -54,6 +55,7 @@
 
 
 class FederationReaderSlavedStore(
+    SlavedAccountDataStore,
     SlavedProfileStore,
     SlavedApplicationServiceStore,
     SlavedPusherStore,

From aae86a81eff9e5099efc97078ce35f26332f0ff4 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 16 Aug 2018 15:41:48 +0100
Subject: [PATCH 427/830] Newsfile

---
 changelog.d/3705.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3705.bugfix

diff --git a/changelog.d/3705.bugfix b/changelog.d/3705.bugfix
new file mode 100644
index 000000000000..6c5422994f34
--- /dev/null
+++ b/changelog.d/3705.bugfix
@@ -0,0 +1 @@
+Support more federation endpoints on workers

From eff3ae3b9a8ab558a106ce3fe69774a2e594cf72 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Thu, 16 Aug 2018 15:48:34 +0100
Subject: [PATCH 428/830] add room tagging

---
 .../resource_limits_server_notices.py              | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/synapse/server_notices/resource_limits_server_notices.py b/synapse/server_notices/resource_limits_server_notices.py
index 43a926fc7975..570e8307cca4 100644
--- a/synapse/server_notices/resource_limits_server_notices.py
+++ b/synapse/server_notices/resource_limits_server_notices.py
@@ -66,8 +66,12 @@ def maybe_send_server_notice_to_user(self, user_id):
 
             room_id = yield self._server_notices_manager.get_notice_room_for_user(user_id)
 
+            # Need to set tag here because room may have been created prior to
+            # tagging being set on creation. Ideally would set something to read
+            # room tags first, and cache that aggressively/
+            yield self._store.add_tag_to_room(user_id, room_id, 'm.server_notice', None)
+
             currently_blocked = False
-            logger.info("GET CURRENT STATE")
             pinned_state_event = None
             try:
                 pinned_state_event = yield self._state.get_current_state(
@@ -82,15 +86,9 @@ def maybe_send_server_notice_to_user(self, user_id):
                 referenced_events = pinned_state_event.content.get('pinned')
 
             events = yield self._store.get_events(referenced_events)
-            logger.info(events)
             for event_id, event in events.items():
-                logger.info("event_id, event event.type %s %s %s" % (
-                    event_id, event, event.type)
-                )
                 if event.type == EventTypes.ServerNoticeLimitReached:
                     currently_blocked = True
-
-            logger.info("currently_blocked is %r" % currently_blocked)
             try:
                 # Normally should always pass in user_id if you have it, but in
                 # this case are checking what would happen to other users if they
@@ -109,7 +107,6 @@ def maybe_send_server_notice_to_user(self, user_id):
                     yield self._server_notices_manager.send_notice(
                         user_id, content, EventTypes.Pinned, '',
                     )
-                    logger.info('deactivate block')
 
             except AuthError as e:
                 # Need to start notifying of blocking
@@ -131,7 +128,6 @@ def maybe_send_server_notice_to_user(self, user_id):
                                 event.event_id,
                             ]
                         }
-                        logger.info("active block")
                         yield self._server_notices_manager.send_notice(
                             user_id, content, EventTypes.Pinned, '',
                         )

From 3c1080b6e4a4d7c80d67bd9bee45d845aa145088 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Thu, 16 Aug 2018 17:02:04 +0100
Subject: [PATCH 429/830] refactor for readability, and reuse caching for
 setting tags

---
 .../resource_limits_server_notices.py         | 176 +++++++++++-------
 .../server_notices/server_notices_manager.py  |   6 +-
 .../test_resource_limits_server_notices.py    |   1 +
 3 files changed, 115 insertions(+), 68 deletions(-)

diff --git a/synapse/server_notices/resource_limits_server_notices.py b/synapse/server_notices/resource_limits_server_notices.py
index 570e8307cca4..61becca75831 100644
--- a/synapse/server_notices/resource_limits_server_notices.py
+++ b/synapse/server_notices/resource_limits_server_notices.py
@@ -18,6 +18,7 @@
 
 from synapse.api.constants import EventTypes
 from synapse.api.errors import AuthError, SynapseError
+from synapse.server_notices.server_notices_manager import SERVER_NOTICE_ROOM_TAG
 
 logger = logging.getLogger(__name__)
 
@@ -57,80 +58,121 @@ def maybe_send_server_notice_to_user(self, user_id):
         if self._hs_disabled is True:
             return
 
-        if self._limit_usage_by_mau is True:
-            timestamp = yield self._store.user_last_seen_monthly_active(user_id)
-            if timestamp is None:
-                # This user will be blocked from receiving the notice anyway.
-                # In practice, not sure we can ever get here
-                return
-
-            room_id = yield self._server_notices_manager.get_notice_room_for_user(user_id)
+        if self._limit_usage_by_mau is False:
+            return
 
-            # Need to set tag here because room may have been created prior to
-            # tagging being set on creation. Ideally would set something to read
-            # room tags first, and cache that aggressively/
-            yield self._store.add_tag_to_room(user_id, room_id, 'm.server_notice', None)
+        timestamp = yield self._store.user_last_seen_monthly_active(user_id)
+        if timestamp is None:
+            # This user will be blocked from receiving the notice anyway.
+            # In practice, not sure we can ever get here
+            return
 
-            currently_blocked = False
-            pinned_state_event = None
-            try:
-                pinned_state_event = yield self._state.get_current_state(
-                    room_id, event_type=EventTypes.Pinned
+        # Determine current state of room
+
+        room_id = yield self._server_notices_manager.get_notice_room_for_user(user_id)
+
+        yield self._check_and_set_tags(user_id, room_id)
+        currently_blocked, ref_events = yield self._is_room_currently_blocked(room_id)
+
+        try:
+            # Normally should always pass in user_id if you have it, but in
+            # this case are checking what would happen to other users if they
+            # were to arrive.
+            yield self.auth.check_auth_blocking()
+            if currently_blocked:
+                # Room is notifying of a block, when it ought not to be.
+                # Remove block notification
+                content = {
+                    "pinned": ref_events
+                }
+                yield self._server_notices_manager.send_notice(
+                    user_id, content, EventTypes.Pinned, '',
                 )
-            except AuthError as e:
-                # The user has yet to join the server notices room
-                pass
-
-            referenced_events = []
-            if pinned_state_event is not None:
-                referenced_events = pinned_state_event.content.get('pinned')
-
-            events = yield self._store.get_events(referenced_events)
-            for event_id, event in events.items():
-                if event.type == EventTypes.ServerNoticeLimitReached:
-                    currently_blocked = True
+
+        except AuthError as e:
+
             try:
-                # Normally should always pass in user_id if you have it, but in
-                # this case are checking what would happen to other users if they
-                # were to arrive.
-                yield self.auth.check_auth_blocking()
-
-                # Need to start removing notices
-                # if user_id in self._notified_of_blocking:
-                if currently_blocked:
-                    # Send message to remove warning
-                    # send state event here
-                    # How do I do this? if drop the id, how to refer to it?
+                if not currently_blocked:
+                    # Room is not notifying of a block, when it ought to be.
+                    # Add block notification
                     content = {
-                        "pinned": []
+                        'body': e.msg,
+                        'admin_uri': self._admin_uri,
+                    }
+                    event = yield self._server_notices_manager.send_notice(
+                        user_id, content, EventTypes.ServerNoticeLimitReached
+                    )
+
+                    content = {
+                        "pinned": [
+                            event.event_id,
+                        ]
                     }
                     yield self._server_notices_manager.send_notice(
                         user_id, content, EventTypes.Pinned, '',
                     )
 
-            except AuthError as e:
-                # Need to start notifying of blocking
-                try:
-                    if not currently_blocked:
-                        # TODO use admin email contained in error once PR lands
-                        content = {
-                            'body': e.msg,
-                            'admin_uri': self._admin_uri,
-                        }
-                        event = yield self._server_notices_manager.send_notice(
-                            user_id, content, EventTypes.ServerNoticeLimitReached
-                        )
-
-                        # send server notices state event here
-                        # TODO Over writing pinned events
-                        content = {
-                            "pinned": [
-                                event.event_id,
-                            ]
-                        }
-                        yield self._server_notices_manager.send_notice(
-                            user_id, content, EventTypes.Pinned, '',
-                        )
-
-                except SynapseError as e:
-                    logger.error("Error sending resource limits server notice: %s", e)
+            except SynapseError as e:
+                logger.error("Error sending resource limits server notice: %s", e)
+
+    @defer.inlineCallbacks
+    def _check_and_set_tags(self, user_id, room_id):
+        """
+        Since server notices rooms were originally not with tags,
+        important to check that tags have been set correctly
+        Args:
+            user_id(str): the user in question
+            room_id(str): the server notices room for that user
+        """
+        tags = yield self._store.get_tags_for_user(user_id)
+        server_notices_tags = tags.get(room_id)
+        need_to_set_tag = True
+        if server_notices_tags:
+            if server_notice_tags.get(SERVER_NOTICE_ROOM_TAG):
+                # tag already present, nothing to do here
+                need_to_set_tag = False
+        if need_to_set_tag:
+            yield self._store.add_tag_to_room(
+                user_id, room_id, SERVER_NOTICE_ROOM_TAG, None
+            )
+
+    @defer.inlineCallbacks
+    def _is_room_currently_blocked(self, room_id):
+        """
+        Determines if the room is currently blocked
+
+        Args:
+            room_id(str): The room id of the server notices room
+
+        Returns:
+
+            bool: Is the room currently blocked
+            list: The list of pinned events that are unrelated to limit blocking
+            This list can be used as a convenience in the case where the block
+            is to be lifted and the remaining pinned event references need to be
+            preserved
+        """
+        currently_blocked = False
+        pinned_state_event = None
+        try:
+            pinned_state_event = yield self._state.get_current_state(
+                room_id, event_type=EventTypes.Pinned
+            )
+        except AuthError as e:
+            # The user has yet to join the server notices room
+            pass
+
+        referenced_events = []
+        if pinned_state_event is not None:
+            referenced_events = pinned_state_event.content.get('pinned')
+
+        events = yield self._store.get_events(referenced_events)
+        event_to_remove = None
+        for event_id, event in events.items():
+            if event.type == EventTypes.ServerNoticeLimitReached:
+                currently_blocked = True
+                # remove event in case we need to disable blocking later on.
+                if event_id in referenced_events:
+                    referenced_events.remove(event.event_id)
+
+        defer.returnValue((currently_blocked, referenced_events))
diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py
index 22c819cc5ba2..5968104a99ac 100644
--- a/synapse/server_notices/server_notices_manager.py
+++ b/synapse/server_notices/server_notices_manager.py
@@ -22,6 +22,8 @@
 
 logger = logging.getLogger(__name__)
 
+SERVER_NOTICE_ROOM_TAG = "m.server_notice"
+
 
 class ServerNoticesManager(object):
     def __init__(self, hs):
@@ -151,7 +153,9 @@ def get_notice_room_for_user(self, user_id):
             creator_join_profile=join_profile,
         )
         room_id = info['room_id']
-        yield self._store.add_tag_to_room(user_id, room_id, 'm.server_notice', None)
+        yield self._store.add_tag_to_room(
+            user_id, room_id, SERVER_NOTICE_ROOM_TAG, None
+        )
 
         logger.info("Created server notices room %s for %s", room_id, user_id)
         defer.returnValue(room_id)
diff --git a/tests/server_notices/test_resource_limits_server_notices.py b/tests/server_notices/test_resource_limits_server_notices.py
index fb8c593a75e8..ccb69097b1e8 100644
--- a/tests/server_notices/test_resource_limits_server_notices.py
+++ b/tests/server_notices/test_resource_limits_server_notices.py
@@ -54,6 +54,7 @@ def setUp(self):
         self._rlsn._server_notices_manager.get_notice_room_for_user = Mock(
             returnValue=""
         )
+        self._rlsn._store.add_tag_to_room = Mock()
         self.hs.config.admin_uri = "mailto:user@test.com"
 
     @defer.inlineCallbacks

From 51b17ec56615b710f4937e3c20208841a6591477 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Thu, 16 Aug 2018 17:32:22 +0100
Subject: [PATCH 430/830] flake8

---
 synapse/server_notices/resource_limits_server_notices.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/synapse/server_notices/resource_limits_server_notices.py b/synapse/server_notices/resource_limits_server_notices.py
index 61becca75831..33625fba723d 100644
--- a/synapse/server_notices/resource_limits_server_notices.py
+++ b/synapse/server_notices/resource_limits_server_notices.py
@@ -128,7 +128,7 @@ def _check_and_set_tags(self, user_id, room_id):
         server_notices_tags = tags.get(room_id)
         need_to_set_tag = True
         if server_notices_tags:
-            if server_notice_tags.get(SERVER_NOTICE_ROOM_TAG):
+            if server_notices_tags.get(SERVER_NOTICE_ROOM_TAG):
                 # tag already present, nothing to do here
                 need_to_set_tag = False
         if need_to_set_tag:
@@ -158,7 +158,7 @@ def _is_room_currently_blocked(self, room_id):
             pinned_state_event = yield self._state.get_current_state(
                 room_id, event_type=EventTypes.Pinned
             )
-        except AuthError as e:
+        except AuthError:
             # The user has yet to join the server notices room
             pass
 
@@ -167,7 +167,6 @@ def _is_room_currently_blocked(self, room_id):
             referenced_events = pinned_state_event.content.get('pinned')
 
         events = yield self._store.get_events(referenced_events)
-        event_to_remove = None
         for event_id, event in events.items():
             if event.type == EventTypes.ServerNoticeLimitReached:
                 currently_blocked = True

From 13ad9930c8799ea54671a6ce00533528d89e061b Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Thu, 16 Aug 2018 18:02:02 +0100
Subject: [PATCH 431/830] add new error type ResourceLimit

---
 synapse/api/auth.py             | 10 ++++++----
 synapse/api/errors.py           | 23 +++++++++++++++++++++--
 synapse/config/server.py        |  1 +
 tests/api/test_auth.py          |  6 +++---
 tests/handlers/test_auth.py     | 10 +++++-----
 tests/handlers/test_register.py | 14 +++++++-------
 tests/handlers/test_sync.py     |  6 +++---
 tests/utils.py                  |  1 +
 8 files changed, 47 insertions(+), 24 deletions(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 3b2a2ab77ade..6945c118d39d 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -25,7 +25,7 @@
 import synapse.types
 from synapse import event_auth
 from synapse.api.constants import EventTypes, JoinRules, Membership
-from synapse.api.errors import AuthError, Codes
+from synapse.api.errors import AuthError, Codes, ResourceLimitError
 from synapse.types import UserID
 from synapse.util.caches import CACHE_SIZE_FACTOR, register_cache
 from synapse.util.caches.lrucache import LruCache
@@ -784,10 +784,11 @@ def check_auth_blocking(self, user_id=None):
             MAU cohort
         """
         if self.hs.config.hs_disabled:
-            raise AuthError(
+            raise ResourceLimitError(
                 403, self.hs.config.hs_disabled_message,
                 errcode=Codes.RESOURCE_LIMIT_EXCEED,
                 admin_uri=self.hs.config.admin_uri,
+                limit_type=self.hs.config.hs_disabled_limit_type
             )
         if self.hs.config.limit_usage_by_mau is True:
             # If the user is already part of the MAU cohort
@@ -798,8 +799,9 @@ def check_auth_blocking(self, user_id=None):
             # Else if there is no room in the MAU bucket, bail
             current_mau = yield self.store.get_monthly_active_count()
             if current_mau >= self.hs.config.max_mau_value:
-                raise AuthError(
+                raise ResourceLimitError(
                     403, "Monthly Active User Limits AU Limit Exceeded",
                     admin_uri=self.hs.config.admin_uri,
-                    errcode=Codes.RESOURCE_LIMIT_EXCEED
+                    errcode=Codes.RESOURCE_LIMIT_EXCEED,
+                    limit_type="monthly_active_user"
                 )
diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index 08f0cb55548e..e26001ab12c4 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -224,15 +224,34 @@ def __init__(self, msg="Not found", errcode=Codes.NOT_FOUND):
 
 class AuthError(SynapseError):
     """An error raised when there was a problem authorising an event."""
-    def __init__(self, code, msg, errcode=Codes.FORBIDDEN, admin_uri=None):
+
+    def __init__(self, *args, **kwargs):
+        if "errcode" not in kwargs:
+            kwargs["errcode"] = Codes.FORBIDDEN
+        super(AuthError, self).__init__(*args, **kwargs)
+
+
+class ResourceLimitError(SynapseError):
+    """
+    Any error raised when there is a problem with resource usage.
+    For instance, the monthly active user limit for the server has been exceeded
+    """
+    def __init__(
+        self, code, msg,
+        errcode=Codes.RESOURCE_LIMIT_EXCEED,
+        admin_uri=None,
+        limit_type=None,
+    ):
         self.admin_uri = admin_uri
-        super(AuthError, self).__init__(code, msg, errcode=errcode)
+        self.limit_type = limit_type
+        super(ResourceLimitError, self).__init__(code, msg, errcode=errcode)
 
     def error_dict(self):
         return cs_error(
             self.msg,
             self.errcode,
             admin_uri=self.admin_uri,
+            limit_type=self.limit_type
         )
 
 
diff --git a/synapse/config/server.py b/synapse/config/server.py
index 2190f3210a73..ae72c872d934 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -81,6 +81,7 @@ def read_config(self, config):
         # Options to disable HS
         self.hs_disabled = config.get("hs_disabled", False)
         self.hs_disabled_message = config.get("hs_disabled_message", "")
+        self.hs_disabled_limit_type = config.get("hs_disabled_limit_type", "")
 
         # Admin uri to direct users at should their instance become blocked
         # due to resource constraints
diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py
index 32a2b5fc3d0f..022d81ce3e9d 100644
--- a/tests/api/test_auth.py
+++ b/tests/api/test_auth.py
@@ -21,7 +21,7 @@
 
 import synapse.handlers.auth
 from synapse.api.auth import Auth
-from synapse.api.errors import AuthError, Codes
+from synapse.api.errors import AuthError, Codes, ResourceLimitError
 from synapse.types import UserID
 
 from tests import unittest
@@ -455,7 +455,7 @@ def test_blocking_mau(self):
             return_value=defer.succeed(lots_of_users)
         )
 
-        with self.assertRaises(AuthError) as e:
+        with self.assertRaises(ResourceLimitError) as e:
             yield self.auth.check_auth_blocking()
         self.assertEquals(e.exception.admin_uri, self.hs.config.admin_uri)
         self.assertEquals(e.exception.errcode, Codes.RESOURCE_LIMIT_EXCEED)
@@ -471,7 +471,7 @@ def test_blocking_mau(self):
     def test_hs_disabled(self):
         self.hs.config.hs_disabled = True
         self.hs.config.hs_disabled_message = "Reason for being disabled"
-        with self.assertRaises(AuthError) as e:
+        with self.assertRaises(ResourceLimitError) as e:
             yield self.auth.check_auth_blocking()
         self.assertEquals(e.exception.admin_uri, self.hs.config.admin_uri)
         self.assertEquals(e.exception.errcode, Codes.RESOURCE_LIMIT_EXCEED)
diff --git a/tests/handlers/test_auth.py b/tests/handlers/test_auth.py
index 3046bd609365..1e39fe0ec201 100644
--- a/tests/handlers/test_auth.py
+++ b/tests/handlers/test_auth.py
@@ -20,7 +20,7 @@
 
 import synapse
 import synapse.api.errors
-from synapse.api.errors import AuthError
+from synapse.api.errors import ResourceLimitError
 from synapse.handlers.auth import AuthHandler
 
 from tests import unittest
@@ -130,13 +130,13 @@ def test_mau_limits_exceeded_large(self):
             return_value=defer.succeed(self.large_number_of_users)
         )
 
-        with self.assertRaises(AuthError):
+        with self.assertRaises(ResourceLimitError):
             yield self.auth_handler.get_access_token_for_user_id('user_a')
 
         self.hs.get_datastore().get_monthly_active_count = Mock(
             return_value=defer.succeed(self.large_number_of_users)
         )
-        with self.assertRaises(AuthError):
+        with self.assertRaises(ResourceLimitError):
             yield self.auth_handler.validate_short_term_login_token_and_get_user_id(
                 self._get_macaroon().serialize()
             )
@@ -149,13 +149,13 @@ def test_mau_limits_parity(self):
         self.hs.get_datastore().get_monthly_active_count = Mock(
             return_value=defer.succeed(self.hs.config.max_mau_value)
         )
-        with self.assertRaises(AuthError):
+        with self.assertRaises(ResourceLimitError):
             yield self.auth_handler.get_access_token_for_user_id('user_a')
 
         self.hs.get_datastore().get_monthly_active_count = Mock(
             return_value=defer.succeed(self.hs.config.max_mau_value)
         )
-        with self.assertRaises(AuthError):
+        with self.assertRaises(ResourceLimitError):
             yield self.auth_handler.validate_short_term_login_token_and_get_user_id(
                 self._get_macaroon().serialize()
             )
diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py
index 7154816a34a7..7b4ade3dfbf4 100644
--- a/tests/handlers/test_register.py
+++ b/tests/handlers/test_register.py
@@ -17,7 +17,7 @@
 
 from twisted.internet import defer
 
-from synapse.api.errors import AuthError
+from synapse.api.errors import ResourceLimitError
 from synapse.handlers.register import RegistrationHandler
 from synapse.types import UserID, create_requester
 
@@ -109,13 +109,13 @@ def test_get_or_create_user_mau_blocked(self):
         self.store.get_monthly_active_count = Mock(
             return_value=defer.succeed(self.lots_of_users)
         )
-        with self.assertRaises(AuthError):
+        with self.assertRaises(ResourceLimitError):
             yield self.handler.get_or_create_user("requester", 'b', "display_name")
 
         self.store.get_monthly_active_count = Mock(
             return_value=defer.succeed(self.hs.config.max_mau_value)
         )
-        with self.assertRaises(AuthError):
+        with self.assertRaises(ResourceLimitError):
             yield self.handler.get_or_create_user("requester", 'b', "display_name")
 
     @defer.inlineCallbacks
@@ -124,13 +124,13 @@ def test_register_mau_blocked(self):
         self.store.get_monthly_active_count = Mock(
             return_value=defer.succeed(self.lots_of_users)
         )
-        with self.assertRaises(AuthError):
+        with self.assertRaises(ResourceLimitError):
             yield self.handler.register(localpart="local_part")
 
         self.store.get_monthly_active_count = Mock(
             return_value=defer.succeed(self.hs.config.max_mau_value)
         )
-        with self.assertRaises(AuthError):
+        with self.assertRaises(ResourceLimitError):
             yield self.handler.register(localpart="local_part")
 
     @defer.inlineCallbacks
@@ -139,11 +139,11 @@ def test_register_saml2_mau_blocked(self):
         self.store.get_monthly_active_count = Mock(
             return_value=defer.succeed(self.lots_of_users)
         )
-        with self.assertRaises(AuthError):
+        with self.assertRaises(ResourceLimitError):
             yield self.handler.register_saml2(localpart="local_part")
 
         self.store.get_monthly_active_count = Mock(
             return_value=defer.succeed(self.hs.config.max_mau_value)
         )
-        with self.assertRaises(AuthError):
+        with self.assertRaises(ResourceLimitError):
             yield self.handler.register_saml2(localpart="local_part")
diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py
index 33d861bd64ae..a01ab471f594 100644
--- a/tests/handlers/test_sync.py
+++ b/tests/handlers/test_sync.py
@@ -14,7 +14,7 @@
 # limitations under the License.
 from twisted.internet import defer
 
-from synapse.api.errors import AuthError, Codes
+from synapse.api.errors import Codes, ResourceLimitError
 from synapse.api.filtering import DEFAULT_FILTER_COLLECTION
 from synapse.handlers.sync import SyncConfig, SyncHandler
 from synapse.types import UserID
@@ -49,7 +49,7 @@ def test_wait_for_sync_for_user_auth_blocking(self):
 
         # Test that global lock works
         self.hs.config.hs_disabled = True
-        with self.assertRaises(AuthError) as e:
+        with self.assertRaises(ResourceLimitError) as e:
             yield self.sync_handler.wait_for_sync_for_user(sync_config)
         self.assertEquals(e.exception.errcode, Codes.RESOURCE_LIMIT_EXCEED)
 
@@ -57,7 +57,7 @@ def test_wait_for_sync_for_user_auth_blocking(self):
 
         sync_config = self._generate_sync_config(user_id2)
 
-        with self.assertRaises(AuthError) as e:
+        with self.assertRaises(ResourceLimitError) as e:
             yield self.sync_handler.wait_for_sync_for_user(sync_config)
         self.assertEquals(e.exception.errcode, Codes.RESOURCE_LIMIT_EXCEED)
 
diff --git a/tests/utils.py b/tests/utils.py
index 52326d4f670b..6f8b1de3e799 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -137,6 +137,7 @@ def setup_test_homeserver(
         config.limit_usage_by_mau = False
         config.hs_disabled = False
         config.hs_disabled_message = ""
+        config.hs_disabled_limit_type = ""
         config.max_mau_value = 50
         config.mau_limits_reserved_threepids = []
         config.admin_uri = None

From 7edd11623dac8952d6af04b49cc1390cf579cb68 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Thu, 16 Aug 2018 21:04:30 +0100
Subject: [PATCH 432/830] add new error type ResourceLimit

---
 changelog.d/3707.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3707.misc

diff --git a/changelog.d/3707.misc b/changelog.d/3707.misc
new file mode 100644
index 000000000000..8123ca65430b
--- /dev/null
+++ b/changelog.d/3707.misc
@@ -0,0 +1 @@
+add new error type ResourceLimit

From 372bf073c1a5ac5e66cca717ce9aa72c43ba9404 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Thu, 16 Aug 2018 21:25:16 +0100
Subject: [PATCH 433/830] block event creation and room creation on hitting
 resource limits

---
 synapse/handlers/message.py | 6 +++++-
 synapse/handlers/room.py    | 4 ++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 893c9bcdc4db..4d006df63c62 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -276,10 +276,14 @@ def create_event(self, requester, event_dict, token_id=None, txn_id=None,
                 where *hashes* is a map from algorithm to hash.
 
                 If None, they will be requested from the database.
-
+        Raises:
+            ResourceLimitError if server is blocked to some resource being
+            exceeded
         Returns:
             Tuple of created event (FrozenEvent), Context
         """
+        yield self.auth.check_auth_blocking(requester.user.to_string())
+
         builder = self.event_builder_factory.new(event_dict)
 
         self.validator.validate_new(builder)
diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index 6a17c42238ec..c3f820b9753b 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -98,9 +98,13 @@ def create_room(self, requester, config, ratelimit=True,
         Raises:
             SynapseError if the room ID couldn't be stored, or something went
             horribly wrong.
+            ResourceLimitError if server is blocked to some resource being
+            exceeded
         """
         user_id = requester.user.to_string()
 
+        self.auth.check_auth_blocking(user_id)
+
         if not self.spam_checker.user_may_create_room(user_id):
             raise SynapseError(403, "You are not permitted to create rooms")
 

From bcfeb44afe750dadd4199e7c02302b0157bdab11 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Thu, 16 Aug 2018 22:55:32 +0100
Subject: [PATCH 434/830] call reap on start up and fix under reaping bug

---
 synapse/api/auth.py                        |  2 +-
 synapse/app/homeserver.py                  |  1 +
 synapse/storage/monthly_active_users.py    |  5 ++++-
 tests/storage/test_monthly_active_users.py | 13 +++++++++++++
 4 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 3b2a2ab77ade..ab1e3a4e353f 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -799,7 +799,7 @@ def check_auth_blocking(self, user_id=None):
             current_mau = yield self.store.get_monthly_active_count()
             if current_mau >= self.hs.config.max_mau_value:
                 raise AuthError(
-                    403, "Monthly Active User Limits AU Limit Exceeded",
+                    403, "Monthly Active User Limit Exceeded",
                     admin_uri=self.hs.config.admin_uri,
                     errcode=Codes.RESOURCE_LIMIT_EXCEED
                 )
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index a98bb506e5fb..800b9c0e3c64 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -525,6 +525,7 @@ def generate_user_daily_visit_stats():
     clock.looping_call(
         hs.get_datastore().reap_monthly_active_users, 1000 * 60 * 60
     )
+    yield hs.get_datastore().reap_monthly_active_users()
 
     @defer.inlineCallbacks
     def generate_monthly_active_users():
diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py
index 7e417f811e9e..06f9a75a9779 100644
--- a/synapse/storage/monthly_active_users.py
+++ b/synapse/storage/monthly_active_users.py
@@ -96,7 +96,10 @@ def _reap_users(txn):
             # While Postgres does not require 'LIMIT', but also does not support
             # negative LIMIT values. So there is no way to write it that both can
             # support
-            query_args = [self.hs.config.max_mau_value]
+            safe_guard = self.hs.config.max_mau_value - len(self.reserved_users)
+            # Must be greater than zero for postgres
+            safe_guard = safe_guard if safe_guard > 0 else 0
+            query_args = [safe_guard]
 
             base_sql = """
                 DELETE FROM monthly_active_users
diff --git a/tests/storage/test_monthly_active_users.py b/tests/storage/test_monthly_active_users.py
index 511acbde9bc2..f2ed866ae7fc 100644
--- a/tests/storage/test_monthly_active_users.py
+++ b/tests/storage/test_monthly_active_users.py
@@ -75,6 +75,19 @@ def test_initialise_reserved_users(self):
         active_count = yield self.store.get_monthly_active_count()
         self.assertEquals(active_count, user_num)
 
+        # Test that regalar users are removed from the db
+        ru_count = 2
+        yield self.store.upsert_monthly_active_user("@ru1:server")
+        yield self.store.upsert_monthly_active_user("@ru2:server")
+        active_count = yield self.store.get_monthly_active_count()
+
+        self.assertEqual(active_count, user_num + ru_count)
+        self.hs.config.max_mau_value = user_num
+        yield self.store.reap_monthly_active_users()
+
+        active_count = yield self.store.get_monthly_active_count()
+        self.assertEquals(active_count, user_num)
+
     @defer.inlineCallbacks
     def test_can_insert_and_count_mau(self):
         count = yield self.store.get_monthly_active_count()

From 7e513421969738428c40d9d6032c798edd9ebab6 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Thu, 16 Aug 2018 23:05:20 +0100
Subject: [PATCH 435/830] For resource limit blocked users, prevent writing
 into rooms

---
 changelog.d/3708.feature | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3708.feature

diff --git a/changelog.d/3708.feature b/changelog.d/3708.feature
new file mode 100644
index 000000000000..2f146ba62b34
--- /dev/null
+++ b/changelog.d/3708.feature
@@ -0,0 +1 @@
+For resource limit blocked users, prevent writing into rooms

From 66f7dc8c87a6c585b4d456c5b2c4999452fc82c2 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 17 Aug 2018 00:32:39 +0100
Subject: [PATCH 436/830] Fix logcontexts for running pushers

First of all, avoid resetting the logcontext before running the pushers, to fix
the "Starting db txn 'get_all_updated_receipts' from sentinel context" warning.

Instead, give them their own "background process" logcontexts.
---
 synapse/app/pusher.py          |  4 ++--
 synapse/handlers/federation.py |  3 +--
 synapse/handlers/message.py    |  7 ++-----
 synapse/handlers/receipts.py   | 18 ++++++++----------
 synapse/push/pusherpool.py     | 17 +++++++++++++++--
 5 files changed, 28 insertions(+), 21 deletions(-)

diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py
index 9295a51d5b74..0089a7c64f54 100644
--- a/synapse/app/pusher.py
+++ b/synapse/app/pusher.py
@@ -162,11 +162,11 @@ def poke_pushers(self, stream_name, token, rows):
                     else:
                         yield self.start_pusher(row.user_id, row.app_id, row.pushkey)
             elif stream_name == "events":
-                yield self.pusher_pool.on_new_notifications(
+                self.pusher_pool.on_new_notifications(
                     token, token,
                 )
             elif stream_name == "receipts":
-                yield self.pusher_pool.on_new_receipts(
+                self.pusher_pool.on_new_receipts(
                     token, token, set(row.room_id for row in rows)
                 )
         except Exception:
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index f38b393e4ab4..3dd107a2850b 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -2386,8 +2386,7 @@ def _notify_persisted_event(self, event, max_stream_id):
             extra_users=extra_users
         )
 
-        logcontext.run_in_background(
-            self.pusher_pool.on_new_notifications,
+        self.pusher_pool.on_new_notifications(
             event_stream_id, max_stream_id,
         )
 
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 893c9bcdc4db..f21d740968bf 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -774,11 +774,8 @@ def is_inviter_member_event(e):
             event, context=context
         )
 
-        # this intentionally does not yield: we don't care about the result
-        # and don't need to wait for it.
-        run_in_background(
-            self.pusher_pool.on_new_notifications,
-            event_stream_id, max_stream_id
+        self.pusher_pool.on_new_notifications(
+            event_stream_id, max_stream_id,
         )
 
         def _notify():
diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py
index cb905a390343..a6f3181f099b 100644
--- a/synapse/handlers/receipts.py
+++ b/synapse/handlers/receipts.py
@@ -18,7 +18,6 @@
 
 from synapse.types import get_domain_from_id
 from synapse.util import logcontext
-from synapse.util.logcontext import PreserveLoggingContext
 
 from ._base import BaseHandler
 
@@ -116,16 +115,15 @@ def _handle_new_receipts(self, receipts):
 
         affected_room_ids = list(set([r["room_id"] for r in receipts]))
 
-        with PreserveLoggingContext():
-            self.notifier.on_new_event(
-                "receipt_key", max_batch_id, rooms=affected_room_ids
-            )
-            # Note that the min here shouldn't be relied upon to be accurate.
-            self.hs.get_pusherpool().on_new_receipts(
-                min_batch_id, max_batch_id, affected_room_ids
-            )
+        self.notifier.on_new_event(
+            "receipt_key", max_batch_id, rooms=affected_room_ids
+        )
+        # Note that the min here shouldn't be relied upon to be accurate.
+        self.hs.get_pusherpool().on_new_receipts(
+            min_batch_id, max_batch_id, affected_room_ids,
+        )
 
-            defer.returnValue(True)
+        defer.returnValue(True)
 
     @logcontext.preserve_fn   # caller should not yield on this
     @defer.inlineCallbacks
diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py
index 36bb5bbc6543..9f7d5ef217db 100644
--- a/synapse/push/pusherpool.py
+++ b/synapse/push/pusherpool.py
@@ -18,6 +18,7 @@
 
 from twisted.internet import defer
 
+from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.push.pusher import PusherFactory
 from synapse.util.logcontext import make_deferred_yieldable, run_in_background
 
@@ -122,8 +123,14 @@ def remove_pushers_by_access_token(self, user_id, access_tokens):
                     p['app_id'], p['pushkey'], p['user_name'],
                 )
 
-    @defer.inlineCallbacks
     def on_new_notifications(self, min_stream_id, max_stream_id):
+        run_as_background_process(
+            "on_new_notifications",
+            self._on_new_notifications, min_stream_id, max_stream_id,
+        )
+
+    @defer.inlineCallbacks
+    def _on_new_notifications(self, min_stream_id, max_stream_id):
         try:
             users_affected = yield self.store.get_push_action_users_in_range(
                 min_stream_id, max_stream_id
@@ -147,8 +154,14 @@ def on_new_notifications(self, min_stream_id, max_stream_id):
         except Exception:
             logger.exception("Exception in pusher on_new_notifications")
 
-    @defer.inlineCallbacks
     def on_new_receipts(self, min_stream_id, max_stream_id, affected_room_ids):
+        run_as_background_process(
+            "on_new_receipts",
+            self._on_new_receipts, min_stream_id, max_stream_id, affected_room_ids,
+        )
+
+    @defer.inlineCallbacks
+    def _on_new_receipts(self, min_stream_id, max_stream_id, affected_room_ids):
         try:
             # Need to subtract 1 from the minimum because the lower bound here
             # is not inclusive

From 0e8d78f6aa56020ffb948a4b2c6feadba2d16712 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 17 Aug 2018 00:43:43 +0100
Subject: [PATCH 437/830] Logcontexts for replication command handlers

Run the handlers for replication commands as background processes. This should
improve the visibility in our metrics, and reduce the number of "running db
transaction from sentinel context" warnings.

Ideally it means converting the things that fire off deferreds into the night
into things that actually return a Deferred when they are done. I've made a bit
of a stab at this, but it will probably be leaky.
---
 synapse/app/appservice.py           |  3 ++-
 synapse/app/federation_sender.py    |  3 ++-
 synapse/app/pusher.py               |  3 ++-
 synapse/app/synchrotron.py          |  3 ++-
 synapse/app/user_dir.py             |  3 ++-
 synapse/replication/tcp/client.py   |  4 +--
 synapse/replication/tcp/commands.py | 12 +++++++++
 synapse/replication/tcp/protocol.py | 42 ++++++++++++++++++++---------
 8 files changed, 53 insertions(+), 20 deletions(-)

diff --git a/synapse/app/appservice.py b/synapse/app/appservice.py
index 9a37384fb7b5..3348a8ec6dbc 100644
--- a/synapse/app/appservice.py
+++ b/synapse/app/appservice.py
@@ -117,8 +117,9 @@ def __init__(self, hs):
         super(ASReplicationHandler, self).__init__(hs.get_datastore())
         self.appservice_handler = hs.get_application_service_handler()
 
+    @defer.inlineCallbacks
     def on_rdata(self, stream_name, token, rows):
-        super(ASReplicationHandler, self).on_rdata(stream_name, token, rows)
+        yield super(ASReplicationHandler, self).on_rdata(stream_name, token, rows)
 
         if stream_name == "events":
             max_stream_id = self.store.get_room_max_stream_ordering()
diff --git a/synapse/app/federation_sender.py b/synapse/app/federation_sender.py
index 7a4310ca1828..d59007099bec 100644
--- a/synapse/app/federation_sender.py
+++ b/synapse/app/federation_sender.py
@@ -144,8 +144,9 @@ def __init__(self, hs):
         super(FederationSenderReplicationHandler, self).__init__(hs.get_datastore())
         self.send_handler = FederationSenderHandler(hs, self)
 
+    @defer.inlineCallbacks
     def on_rdata(self, stream_name, token, rows):
-        super(FederationSenderReplicationHandler, self).on_rdata(
+        yield super(FederationSenderReplicationHandler, self).on_rdata(
             stream_name, token, rows
         )
         self.send_handler.process_replication_rows(stream_name, token, rows)
diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py
index 9295a51d5b74..aa0938c376d6 100644
--- a/synapse/app/pusher.py
+++ b/synapse/app/pusher.py
@@ -148,8 +148,9 @@ def __init__(self, hs):
 
         self.pusher_pool = hs.get_pusherpool()
 
+    @defer.inlineCallbacks
     def on_rdata(self, stream_name, token, rows):
-        super(PusherReplicationHandler, self).on_rdata(stream_name, token, rows)
+        yield super(PusherReplicationHandler, self).on_rdata(stream_name, token, rows)
         run_in_background(self.poke_pushers, stream_name, token, rows)
 
     @defer.inlineCallbacks
diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py
index e201f18efdc5..39c7cbc1bae3 100644
--- a/synapse/app/synchrotron.py
+++ b/synapse/app/synchrotron.py
@@ -332,8 +332,9 @@ def __init__(self, hs):
         self.presence_handler = hs.get_presence_handler()
         self.notifier = hs.get_notifier()
 
+    @defer.inlineCallbacks
     def on_rdata(self, stream_name, token, rows):
-        super(SyncReplicationHandler, self).on_rdata(stream_name, token, rows)
+        yield super(SyncReplicationHandler, self).on_rdata(stream_name, token, rows)
         run_in_background(self.process_and_notify, stream_name, token, rows)
 
     def get_streams_to_replicate(self):
diff --git a/synapse/app/user_dir.py b/synapse/app/user_dir.py
index cb78de88340a..1388a42b59aa 100644
--- a/synapse/app/user_dir.py
+++ b/synapse/app/user_dir.py
@@ -169,8 +169,9 @@ def __init__(self, hs):
         super(UserDirectoryReplicationHandler, self).__init__(hs.get_datastore())
         self.user_directory = hs.get_user_directory_handler()
 
+    @defer.inlineCallbacks
     def on_rdata(self, stream_name, token, rows):
-        super(UserDirectoryReplicationHandler, self).on_rdata(
+        yield super(UserDirectoryReplicationHandler, self).on_rdata(
             stream_name, token, rows
         )
         if stream_name == "current_state_deltas":
diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py
index 970e94313e31..cbe964581711 100644
--- a/synapse/replication/tcp/client.py
+++ b/synapse/replication/tcp/client.py
@@ -107,7 +107,7 @@ def on_rdata(self, stream_name, token, rows):
         Can be overriden in subclasses to handle more.
         """
         logger.info("Received rdata %s -> %s", stream_name, token)
-        self.store.process_replication_rows(stream_name, token, rows)
+        return self.store.process_replication_rows(stream_name, token, rows)
 
     def on_position(self, stream_name, token):
         """Called when we get new position data. By default this just pokes
@@ -115,7 +115,7 @@ def on_position(self, stream_name, token):
 
         Can be overriden in subclasses to handle more.
         """
-        self.store.process_replication_rows(stream_name, token, [])
+        return self.store.process_replication_rows(stream_name, token, [])
 
     def on_sync(self, data):
         """When we received a SYNC we wake up any deferreds that were waiting
diff --git a/synapse/replication/tcp/commands.py b/synapse/replication/tcp/commands.py
index f3908df642b5..327556f6a1e6 100644
--- a/synapse/replication/tcp/commands.py
+++ b/synapse/replication/tcp/commands.py
@@ -59,6 +59,12 @@ def to_line(self):
         """
         return self.data
 
+    def get_logcontext_id(self):
+        """Get a suitable string for the logcontext when processing this command"""
+
+        # by default, we just use the command name.
+        return self.NAME
+
 
 class ServerCommand(Command):
     """Sent by the server on new connection and includes the server_name.
@@ -116,6 +122,9 @@ def to_line(self):
             _json_encoder.encode(self.row),
         ))
 
+    def get_logcontext_id(self):
+        return "RDATA-" + self.stream_name
+
 
 class PositionCommand(Command):
     """Sent by the client to tell the client the stream postition without
@@ -190,6 +199,9 @@ def from_line(cls, line):
     def to_line(self):
         return " ".join((self.stream_name, str(self.token),))
 
+    def get_logcontext_id(self):
+        return "REPLICATE-" + self.stream_name
+
 
 class UserSyncCommand(Command):
     """Sent by the client to inform the server that a user has started or
diff --git a/synapse/replication/tcp/protocol.py b/synapse/replication/tcp/protocol.py
index dec5ac091326..74e892c10416 100644
--- a/synapse/replication/tcp/protocol.py
+++ b/synapse/replication/tcp/protocol.py
@@ -63,6 +63,8 @@
 from twisted.python.failure import Failure
 
 from synapse.metrics import LaterGauge
+from synapse.metrics.background_process_metrics import run_as_background_process
+from synapse.util.logcontext import make_deferred_yieldable, run_in_background
 from synapse.util.stringutils import random_string
 
 from .commands import (
@@ -222,7 +224,11 @@ def lineReceived(self, line):
 
         # Now lets try and call on_<CMD_NAME> function
         try:
-            getattr(self, "on_%s" % (cmd_name,))(cmd)
+            run_as_background_process(
+                "replication-" + cmd.get_logcontext_id(),
+                getattr(self, "on_%s" % (cmd_name,)),
+                cmd,
+            )
         except Exception:
             logger.exception("[%s] Failed to handle line: %r", self.id(), line)
 
@@ -387,7 +393,7 @@ def on_NAME(self, cmd):
         self.name = cmd.data
 
     def on_USER_SYNC(self, cmd):
-        self.streamer.on_user_sync(
+        return self.streamer.on_user_sync(
             self.conn_id, cmd.user_id, cmd.is_syncing, cmd.last_sync_ms,
         )
 
@@ -397,22 +403,33 @@ def on_REPLICATE(self, cmd):
 
         if stream_name == "ALL":
             # Subscribe to all streams we're publishing to.
-            for stream in iterkeys(self.streamer.streams_by_name):
-                self.subscribe_to_stream(stream, token)
+            deferreds = [
+                run_in_background(
+                    self.subscribe_to_stream,
+                    stream, token,
+                )
+                for stream in iterkeys(self.streamer.streams_by_name)
+            ]
+
+            return make_deferred_yieldable(
+                defer.gatherResults(deferreds, consumeErrors=True)
+            )
         else:
-            self.subscribe_to_stream(stream_name, token)
+            return self.subscribe_to_stream(stream_name, token)
 
     def on_FEDERATION_ACK(self, cmd):
-        self.streamer.federation_ack(cmd.token)
+        return self.streamer.federation_ack(cmd.token)
 
     def on_REMOVE_PUSHER(self, cmd):
-        self.streamer.on_remove_pusher(cmd.app_id, cmd.push_key, cmd.user_id)
+        return self.streamer.on_remove_pusher(
+            cmd.app_id, cmd.push_key, cmd.user_id,
+        )
 
     def on_INVALIDATE_CACHE(self, cmd):
-        self.streamer.on_invalidate_cache(cmd.cache_func, cmd.keys)
+        return self.streamer.on_invalidate_cache(cmd.cache_func, cmd.keys)
 
     def on_USER_IP(self, cmd):
-        self.streamer.on_user_ip(
+        return self.streamer.on_user_ip(
             cmd.user_id, cmd.access_token, cmd.ip, cmd.user_agent, cmd.device_id,
             cmd.last_seen,
         )
@@ -542,14 +559,13 @@ def on_RDATA(self, cmd):
             # Check if this is the last of a batch of updates
             rows = self.pending_batches.pop(stream_name, [])
             rows.append(row)
-
-            self.handler.on_rdata(stream_name, cmd.token, rows)
+            return self.handler.on_rdata(stream_name, cmd.token, rows)
 
     def on_POSITION(self, cmd):
-        self.handler.on_position(cmd.stream_name, cmd.token)
+        return self.handler.on_position(cmd.stream_name, cmd.token)
 
     def on_SYNC(self, cmd):
-        self.handler.on_sync(cmd.data)
+        return self.handler.on_sync(cmd.data)
 
     def replicate(self, stream_name, token):
         """Send the subscription request to the server

From d9efd87d55fb7766ba332bedbecd5efe1ca7067b Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 17 Aug 2018 00:49:22 +0100
Subject: [PATCH 438/830] changelog

---
 changelog.d/3709.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3709.misc

diff --git a/changelog.d/3709.misc b/changelog.d/3709.misc
new file mode 100644
index 000000000000..bbda357d4481
--- /dev/null
+++ b/changelog.d/3709.misc
@@ -0,0 +1 @@
+Logcontexts for replication command handlers

From 4c9da1440f9ab26b79e5efc2a0f774916f18aeba Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 17 Aug 2018 00:50:36 +0100
Subject: [PATCH 439/830] changelog

---
 changelog.d/3710.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3710.bugfix

diff --git a/changelog.d/3710.bugfix b/changelog.d/3710.bugfix
new file mode 100644
index 000000000000..c28e17766770
--- /dev/null
+++ b/changelog.d/3710.bugfix
@@ -0,0 +1 @@
+Fix "Starting db txn 'get_all_updated_receipts' from sentinel context" warning
\ No newline at end of file

From a8ffc27db76b9805d7c91e7256c278306ab54761 Mon Sep 17 00:00:00 2001
From: Travis Ralston <travpc@gmail.com>
Date: Fri, 17 Aug 2018 02:46:25 -0600
Subject: [PATCH 440/830] Update the admin register documentation to return a
 real user ID

Presumably this is the intention anyways. I've also updated the domain part to be something more along the lines of what people might expect.
---
 docs/admin_api/register_api.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/admin_api/register_api.rst b/docs/admin_api/register_api.rst
index 209cd140fdf2..16d65c86b367 100644
--- a/docs/admin_api/register_api.rst
+++ b/docs/admin_api/register_api.rst
@@ -33,7 +33,7 @@ As an example::
 
   < {
      "access_token": "token_here",
-     "user_id": "@pepper_roni@test",
+     "user_id": "@pepper_roni:localhost",
      "home_server": "test",
      "device_id": "device_id_here"
     }

From b99a0f39415e42aeebb1dbc0e860324bb8f5b37e Mon Sep 17 00:00:00 2001
From: Travis Ralston <travpc@gmail.com>
Date: Fri, 17 Aug 2018 02:47:31 -0600
Subject: [PATCH 441/830] Create 3712.misc

---
 changelog.d/3712.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3712.misc

diff --git a/changelog.d/3712.misc b/changelog.d/3712.misc
new file mode 100644
index 000000000000..30f8c2af21c0
--- /dev/null
+++ b/changelog.d/3712.misc
@@ -0,0 +1 @@
+Update admin register API documentation to reference a real user ID.

From 521d369e7a6d6497427a03f3e4f81d20bd2e5761 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Fri, 17 Aug 2018 10:12:11 +0100
Subject: [PATCH 442/830] remove errant yield

---
 synapse/app/homeserver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 800b9c0e3c64..005921dcf733 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -525,7 +525,7 @@ def generate_user_daily_visit_stats():
     clock.looping_call(
         hs.get_datastore().reap_monthly_active_users, 1000 * 60 * 60
     )
-    yield hs.get_datastore().reap_monthly_active_users()
+    hs.get_datastore().reap_monthly_active_users()
 
     @defer.inlineCallbacks
     def generate_monthly_active_users():

From 3b2dcfff78d5c330937c4b75f34f79f8e991d8e3 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 17 Aug 2018 11:11:06 +0100
Subject: [PATCH 443/830] Fix logging bug in EDU handling over replication

---
 synapse/replication/http/federation.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/replication/http/federation.py b/synapse/replication/http/federation.py
index 2ddd18f73b31..64a79da1627a 100644
--- a/synapse/replication/http/federation.py
+++ b/synapse/replication/http/federation.py
@@ -156,7 +156,7 @@ def _handle_request(self, request, edu_type):
             edu_content = content["content"]
 
         logger.info(
-            "Got %r edu from $s",
+            "Got %r edu from %s",
             edu_type, origin,
         )
 

From 73737bd0f9be0197dfe456db5b7887087c41f009 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 17 Aug 2018 11:14:05 +0100
Subject: [PATCH 444/830] Newsfile

---
 changelog.d/3713.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3713.bugfix

diff --git a/changelog.d/3713.bugfix b/changelog.d/3713.bugfix
new file mode 100644
index 000000000000..6c5422994f34
--- /dev/null
+++ b/changelog.d/3713.bugfix
@@ -0,0 +1 @@
+Support more federation endpoints on workers

From 38f708a2bbee13ebd34f1595910dd0e4fd532818 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 17 Aug 2018 11:37:42 +0100
Subject: [PATCH 445/830] Remote profile cache should remain in master worker

---
 synapse/storage/profile.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/synapse/storage/profile.py b/synapse/storage/profile.py
index 246ab836bd2e..88b50f33b579 100644
--- a/synapse/storage/profile.py
+++ b/synapse/storage/profile.py
@@ -94,6 +94,8 @@ def set_profile_avatar_url(self, user_localpart, new_avatar_url):
             desc="set_profile_avatar_url",
         )
 
+
+class ProfileStore(ProfileWorkerStore):
     def add_remote_profile_cache(self, user_id, displayname, avatar_url):
         """Ensure we are caching the remote user's profiles.
 
@@ -180,7 +182,3 @@ def is_subscribed_remote_profile_for_user(self, user_id):
 
         if res:
             defer.returnValue(True)
-
-
-class ProfileStore(ProfileWorkerStore):
-    pass

From 0195dfbf52f3cd8e93068e7b108b64f426dba2ff Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Fri, 17 Aug 2018 13:58:25 +0100
Subject: [PATCH 446/830] server limits config docs

---
 synapse/config/server.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/synapse/config/server.py b/synapse/config/server.py
index ae72c872d934..a41c48e69c70 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -341,6 +341,32 @@ def default_config(self, server_name, **kwargs):
           # - port: 9000
           #   bind_addresses: ['::1', '127.0.0.1']
           #   type: manhole
+
+
+          # Homeserver blocking
+          #
+          # How to reach the server admin, used in ResourceLimitError
+          # admin_uri: 'mailto:admin@server.com'
+          #
+          # Global block config
+          #
+          # hs_disabled: False
+          # hs_disabled_message: 'Human readable reason for why the HS is blocked'
+          # hs_disabled_limit_type: 'error code(str), to help clients decode reason'
+          #
+          # Monthly Active User Blocking
+          #
+          # Enables monthly active user checking
+          # limit_usage_by_mau: False
+          # max_mau_value: 50
+          #
+          # Sometimes the server admin will want to ensure certain accounts are
+          # never blocked by mau checking. These accounts are specified here.
+          #
+          # mau_limit_reserved_threepids:
+          # - medium: 'email'
+          #   address: 'reserved_user@example.com'
+
         """ % locals()
 
     def read_arguments(self, args):

From ca87ad1defac1082462367854cb4a656b7a96e90 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 17 Aug 2018 11:43:16 +0100
Subject: [PATCH 447/830] Split ProfileHandler into master and worker

---
 synapse/handlers/profile.py    | 21 ++++++++++++++-------
 synapse/server.py              |  7 +++++--
 tests/handlers/test_profile.py |  4 ++--
 3 files changed, 21 insertions(+), 11 deletions(-)

diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py
index 6d1fbb1a5ca7..8b349f6ad660 100644
--- a/synapse/handlers/profile.py
+++ b/synapse/handlers/profile.py
@@ -33,12 +33,12 @@
 logger = logging.getLogger(__name__)
 
 
-class ProfileHandler(BaseHandler):
+class WorkerProfileHandler(BaseHandler):
     PROFILE_UPDATE_MS = 60 * 1000
     PROFILE_UPDATE_EVERY_MS = 24 * 60 * 60 * 1000
 
     def __init__(self, hs):
-        super(ProfileHandler, self).__init__(hs)
+        super(WorkerProfileHandler, self).__init__(hs)
 
         self.federation = hs.get_federation_client()
         hs.get_federation_registry().register_query_handler(
@@ -47,11 +47,6 @@ def __init__(self, hs):
 
         self.user_directory_handler = hs.get_user_directory_handler()
 
-        if hs.config.worker_app is None:
-            self.clock.looping_call(
-                self._start_update_remote_profile_cache, self.PROFILE_UPDATE_MS,
-            )
-
         self._notify_master_profile_change = (
             ReplicationHandleProfileChangeRestServlet.make_client(hs)
         )
@@ -298,6 +293,18 @@ def _update_join_states(self, requester, target_user):
                     room_id, str(e.message)
                 )
 
+
+class MasterProfileHandler(WorkerProfileHandler):
+    PROFILE_UPDATE_MS = 60 * 1000
+    PROFILE_UPDATE_EVERY_MS = 24 * 60 * 60 * 1000
+
+    def __init__(self, hs):
+        super(MasterProfileHandler, self).__init__(hs)
+
+        self.clock.looping_call(
+            self._start_update_remote_profile_cache, self.PROFILE_UPDATE_MS,
+        )
+
     def _start_update_remote_profile_cache(self):
         return run_as_background_process(
             "Update remote profile", self._update_remote_profile_cache,
diff --git a/synapse/server.py b/synapse/server.py
index 140be9ebe863..be85aad8cf12 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -55,7 +55,7 @@
 from synapse.handlers.message import EventCreationHandler, MessageHandler
 from synapse.handlers.pagination import PaginationHandler
 from synapse.handlers.presence import PresenceHandler
-from synapse.handlers.profile import ProfileHandler
+from synapse.handlers.profile import MasterProfileHandler, WorkerProfileHandler
 from synapse.handlers.read_marker import ReadMarkerHandler
 from synapse.handlers.receipts import ReceiptsHandler
 from synapse.handlers.room import RoomContextHandler, RoomCreationHandler
@@ -307,7 +307,10 @@ def build_initial_sync_handler(self):
         return InitialSyncHandler(self)
 
     def build_profile_handler(self):
-        return ProfileHandler(self)
+        if self.config.worker_app:
+            return WorkerProfileHandler(self)
+        else:
+            return MasterProfileHandler(self)
 
     def build_event_creation_handler(self):
         return EventCreationHandler(self)
diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py
index dc17918a3d6b..07cf5f4c8ef7 100644
--- a/tests/handlers/test_profile.py
+++ b/tests/handlers/test_profile.py
@@ -20,7 +20,7 @@
 
 import synapse.types
 from synapse.api.errors import AuthError
-from synapse.handlers.profile import ProfileHandler
+from synapse.handlers.profile import MasterProfileHandler
 from synapse.types import UserID
 
 from tests import unittest
@@ -29,7 +29,7 @@
 
 class ProfileHandlers(object):
     def __init__(self, hs):
-        self.profile_handler = ProfileHandler(hs)
+        self.profile_handler = MasterProfileHandler(hs)
 
 
 class ProfileTestCase(unittest.TestCase):

From d49b77404beab498ad8c16bd55e811b423851240 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Fri, 17 Aug 2018 15:21:34 +0100
Subject: [PATCH 448/830] clean up, no functional changes

---
 .../server_notices/consent_server_notices.py  |  2 +-
 .../resource_limits_server_notices.py         | 86 ++++++++++---------
 .../test_resource_limits_server_notices.py    | 32 +++----
 3 files changed, 64 insertions(+), 56 deletions(-)

diff --git a/synapse/server_notices/consent_server_notices.py b/synapse/server_notices/consent_server_notices.py
index a9fe58d9ffdd..5e3044d164ca 100644
--- a/synapse/server_notices/consent_server_notices.py
+++ b/synapse/server_notices/consent_server_notices.py
@@ -103,7 +103,7 @@ def maybe_send_server_notice_to_user(self, user_id):
                     },
                 )
                 yield self._server_notices_manager.send_notice(
-                    user_id, content
+                    user_id, content,
                 )
                 yield self._store.user_set_consent_server_notice_sent(
                     user_id, self._current_consent_version,
diff --git a/synapse/server_notices/resource_limits_server_notices.py b/synapse/server_notices/resource_limits_server_notices.py
index 33625fba723d..5987e448b236 100644
--- a/synapse/server_notices/resource_limits_server_notices.py
+++ b/synapse/server_notices/resource_limits_server_notices.py
@@ -14,40 +14,41 @@
 # limitations under the License.
 import logging
 
+from six import iteritems
+
 from twisted.internet import defer
 
 from synapse.api.constants import EventTypes
-from synapse.api.errors import AuthError, SynapseError
+from synapse.api.errors import AuthError, ResourceLimitError, SynapseError
 from synapse.server_notices.server_notices_manager import SERVER_NOTICE_ROOM_TAG
 
 logger = logging.getLogger(__name__)
 
 
 class ResourceLimitsServerNotices(object):
-    """
+    """ Keeps track of whether the server has reached it's resource limit and
+    ensures that the client is kept up to date.
     """
     def __init__(self, hs):
         """
-
         Args:
             hs (synapse.server.HomeServer):
         """
         self._server_notices_manager = hs.get_server_notices_manager()
         self._store = hs.get_datastore()
-        self.auth = hs.get_auth()
-        self._server_notice_content = hs.config.user_consent_server_notice_content
-        self._admin_uri = hs.config.admin_uri
-        self._limit_usage_by_mau = hs.config.limit_usage_by_mau
-        self._hs_disabled = hs.config.hs_disabled
-
+        self._auth = hs.get_auth()
+        self._config = hs.config
         self._resouce_limited = False
         self._message_handler = hs.get_message_handler()
         self._state = hs.get_state_handler()
-        # Config checks?
 
     @defer.inlineCallbacks
     def maybe_send_server_notice_to_user(self, user_id):
-        """Check if we need to send a notice to this user, and does so if so
+        """Check if we need to send a notice to this user, this will be true in
+        two cases.
+        1. The server has reached its limit does not reflect this
+        2. The room state indicates that the server has reached its limit when
+        actually the server is fine
 
         Args:
             user_id (str): user to check
@@ -55,10 +56,10 @@ def maybe_send_server_notice_to_user(self, user_id):
         Returns:
             Deferred
         """
-        if self._hs_disabled is True:
+        if self._config.hs_disabled is True:
             return
 
-        if self._limit_usage_by_mau is False:
+        if self._config.limit_usage_by_mau is False:
             return
 
         timestamp = yield self._store.user_last_seen_monthly_active(user_id)
@@ -78,8 +79,15 @@ def maybe_send_server_notice_to_user(self, user_id):
             # Normally should always pass in user_id if you have it, but in
             # this case are checking what would happen to other users if they
             # were to arrive.
-            yield self.auth.check_auth_blocking()
-            if currently_blocked:
+            try:
+                yield self._auth.check_auth_blocking()
+                is_auth_blocking = False
+            except ResourceLimitError as e:
+                is_auth_blocking = True
+                event_content = e.msg
+                event_limit_type = e.limit_type
+
+            if currently_blocked and not is_auth_blocking:
                 # Room is notifying of a block, when it ought not to be.
                 # Remove block notification
                 content = {
@@ -89,31 +97,29 @@ def maybe_send_server_notice_to_user(self, user_id):
                     user_id, content, EventTypes.Pinned, '',
                 )
 
-        except AuthError as e:
+            elif not currently_blocked and is_auth_blocking:
+                # Room is not notifying of a block, when it ought to be.
+                # Add block notification
+                content = {
+                    'body': event_content,
+                    'admin_uri': self._config.admin_uri,
+                    'limit_type': event_limit_type
+                }
+                event = yield self._server_notices_manager.send_notice(
+                    user_id, content, EventTypes.ServerNoticeLimitReached
+                )
 
-            try:
-                if not currently_blocked:
-                    # Room is not notifying of a block, when it ought to be.
-                    # Add block notification
-                    content = {
-                        'body': e.msg,
-                        'admin_uri': self._admin_uri,
-                    }
-                    event = yield self._server_notices_manager.send_notice(
-                        user_id, content, EventTypes.ServerNoticeLimitReached
-                    )
-
-                    content = {
-                        "pinned": [
-                            event.event_id,
-                        ]
-                    }
-                    yield self._server_notices_manager.send_notice(
-                        user_id, content, EventTypes.Pinned, '',
-                    )
-
-            except SynapseError as e:
-                logger.error("Error sending resource limits server notice: %s", e)
+                content = {
+                    "pinned": [
+                        event.event_id,
+                    ]
+                }
+                yield self._server_notices_manager.send_notice(
+                    user_id, content, EventTypes.Pinned, '',
+                )
+
+        except SynapseError as e:
+            logger.error("Error sending resource limits server notice: %s", e)
 
     @defer.inlineCallbacks
     def _check_and_set_tags(self, user_id, room_id):
@@ -167,7 +173,7 @@ def _is_room_currently_blocked(self, room_id):
             referenced_events = pinned_state_event.content.get('pinned')
 
         events = yield self._store.get_events(referenced_events)
-        for event_id, event in events.items():
+        for event_id, event in events.iteritems():
             if event.type == EventTypes.ServerNoticeLimitReached:
                 currently_blocked = True
                 # remove event in case we need to disable blocking later on.
diff --git a/tests/server_notices/test_resource_limits_server_notices.py b/tests/server_notices/test_resource_limits_server_notices.py
index ccb69097b1e8..cc8d2f539c6e 100644
--- a/tests/server_notices/test_resource_limits_server_notices.py
+++ b/tests/server_notices/test_resource_limits_server_notices.py
@@ -3,7 +3,7 @@
 from twisted.internet import defer
 
 from synapse.api.constants import EventTypes
-from synapse.api.errors import AuthError
+from synapse.api.errors import ResourceLimitError
 from synapse.handlers.auth import AuthHandler
 from synapse.server_notices.resource_limits_server_notices import (
     ResourceLimitsServerNotices,
@@ -43,13 +43,13 @@ def setUp(self):
 
         self._send_notice = self._rlsn._server_notices_manager.send_notice
 
-        self._rlsn._limit_usage_by_mau = True
+        self.hs.config.limit_usage_by_mau = True
         self.user_id = "@user_id:test"
 
-        self.server_notices_mxid = "@server:test"
-        self.server_notices_mxid_display_name = None
-        self.server_notices_mxid_avatar_url = None
-        self.server_notices_room_name = "Server Notices"
+        # self.server_notices_mxid = "@server:test"
+        # self.server_notices_mxid_display_name = None
+        # self.server_notices_mxid_avatar_url = None
+        # self.server_notices_room_name = "Server Notices"
 
         self._rlsn._server_notices_manager.get_notice_room_for_user = Mock(
             returnValue=""
@@ -61,14 +61,14 @@ def setUp(self):
     def test_maybe_send_server_notice_to_user_flag_off(self):
         """Tests cases where the flags indicate nothing to do"""
         # test hs disabled case
-        self._hs_disabled = True
+        self.hs.config.hs_disabled = True
 
         yield self._rlsn.maybe_send_server_notice_to_user(self.user_id)
 
         self._send_notice.assert_not_called()
         # Test when mau limiting disabled
-        self._hs_disabled = False
-        self._rlsn._limit_usage_by_mau = False
+        self.hs.config.hs_disabled = False
+        self.hs.limit_usage_by_mau = False
         yield self._rlsn.maybe_send_server_notice_to_user(self.user_id)
 
         self._send_notice.assert_not_called()
@@ -77,7 +77,7 @@ def test_maybe_send_server_notice_to_user_flag_off(self):
     def test_maybe_send_server_notice_to_user_remove_blocked_notice(self):
         """Test when user has blocked notice, but should have it removed"""
 
-        self._rlsn.auth.check_auth_blocking = Mock()
+        self._rlsn._auth.check_auth_blocking = Mock()
         mock_event = Mock(type=EventTypes.ServerNoticeLimitReached)
         self._rlsn._store.get_events = Mock(return_value=defer.succeed(
             {"123": mock_event}
@@ -90,8 +90,8 @@ def test_maybe_send_server_notice_to_user_remove_blocked_notice(self):
     @defer.inlineCallbacks
     def test_maybe_send_server_notice_to_user_remove_blocked_notice_noop(self):
         """Test when user has blocked notice, but notice ought to be there (NOOP)"""
-        self._rlsn.auth.check_auth_blocking = Mock(
-            side_effect=AuthError(403, 'foo')
+        self._rlsn._auth.check_auth_blocking = Mock(
+            side_effect=ResourceLimitError(403, 'foo')
         )
 
         mock_event = Mock(type=EventTypes.ServerNoticeLimitReached)
@@ -106,7 +106,9 @@ def test_maybe_send_server_notice_to_user_remove_blocked_notice_noop(self):
     def test_maybe_send_server_notice_to_user_add_blocked_notice(self):
         """Test when user does not have blocked notice, but should have one"""
 
-        self._rlsn.auth.check_auth_blocking = Mock(side_effect=AuthError(403, 'foo'))
+        self._rlsn._auth.check_auth_blocking = Mock(
+            side_effect=ResourceLimitError(403, 'foo')
+        )
         yield self._rlsn.maybe_send_server_notice_to_user(self.user_id)
 
         # Would be better to check contents, but 2 calls == set blocking event
@@ -116,7 +118,7 @@ def test_maybe_send_server_notice_to_user_add_blocked_notice(self):
     def test_maybe_send_server_notice_to_user_add_blocked_notice_noop(self):
         """Test when user does not have blocked notice, nor should they (NOOP)"""
 
-        self._rlsn.auth.check_auth_blocking = Mock()
+        self._rlsn._auth.check_auth_blocking = Mock()
 
         yield self._rlsn.maybe_send_server_notice_to_user(self.user_id)
 
@@ -129,7 +131,7 @@ def test_maybe_send_server_notice_to_user_not_in_mau_cohort(self):
         happen - but ...
         """
 
-        self._rlsn.auth.check_auth_blocking = Mock()
+        self._rlsn._auth.check_auth_blocking = Mock()
         self._rlsn._store.user_last_seen_monthly_active = Mock(
             return_value=defer.succeed(None)
         )

From 91cdb6de08aa20f2bc7f8df906eb5b56df387309 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 17 Aug 2018 15:24:16 +0100
Subject: [PATCH 449/830] Call UserDirectoryHandler methods directly

Turns out that the user directory handling is fairly racey as a bunch
of stuff assumes that the processing happens on master, which it doesn't
when there is a synapse.app.user_dir worker. So lets just call the
function directly until we actually get round to fixing it, since it
doesn't make the situation any worse.
---
 synapse/app/event_creator.py         |  2 +
 synapse/handlers/profile.py          | 26 ++-------
 synapse/replication/http/__init__.py |  3 +-
 synapse/replication/http/profile.py  | 81 ----------------------------
 4 files changed, 8 insertions(+), 104 deletions(-)
 delete mode 100644 synapse/replication/http/profile.py

diff --git a/synapse/app/event_creator.py b/synapse/app/event_creator.py
index 2c9a73585993..a2bdaf2a2c0b 100644
--- a/synapse/app/event_creator.py
+++ b/synapse/app/event_creator.py
@@ -58,6 +58,7 @@
 )
 from synapse.server import HomeServer
 from synapse.storage.engines import create_engine
+from synapse.storage.user_directory import UserDirectoryStore
 from synapse.util.httpresourcetree import create_resource_tree
 from synapse.util.logcontext import LoggingContext
 from synapse.util.manhole import manhole
@@ -67,6 +68,7 @@
 
 
 class EventCreatorSlavedStore(
+    UserDirectoryStore,
     DirectoryStore,
     SlavedTransactionStore,
     SlavedProfileStore,
diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py
index 8b349f6ad660..3e1d95d5e5c0 100644
--- a/synapse/handlers/profile.py
+++ b/synapse/handlers/profile.py
@@ -25,7 +25,6 @@
     SynapseError,
 )
 from synapse.metrics.background_process_metrics import run_as_background_process
-from synapse.replication.http.profile import ReplicationHandleProfileChangeRestServlet
 from synapse.types import UserID, get_domain_from_id
 
 from ._base import BaseHandler
@@ -47,10 +46,6 @@ def __init__(self, hs):
 
         self.user_directory_handler = hs.get_user_directory_handler()
 
-        self._notify_master_profile_change = (
-            ReplicationHandleProfileChangeRestServlet.make_client(hs)
-        )
-
     @defer.inlineCallbacks
     def get_profile(self, user_id):
         target_user = UserID.from_string(user_id)
@@ -166,16 +161,10 @@ def set_displayname(self, target_user, requester, new_displayname, by_admin=Fals
         )
 
         if self.hs.config.user_directory_search_all_users:
-            if self.hs.config.worker_app is None:
-                profile = yield self.store.get_profileinfo(target_user.localpart)
-                yield self.user_directory_handler.handle_local_profile_change(
-                    target_user.to_string(), profile
-                )
-            else:
-                yield self._notify_master_profile_change(
-                    requester=requester,
-                    user_id=target_user.to_string(),
-                )
+            profile = yield self.store.get_profileinfo(target_user.localpart)
+            yield self.user_directory_handler.handle_local_profile_change(
+                target_user.to_string(), profile
+            )
 
         yield self._update_join_states(requester, target_user)
 
@@ -225,16 +214,11 @@ def set_avatar_url(self, target_user, requester, new_avatar_url, by_admin=False)
             target_user.localpart, new_avatar_url
         )
 
-        if self.hs.config.worker_app is None:
+        if self.hs.config.user_directory_search_all_users:
             profile = yield self.store.get_profileinfo(target_user.localpart)
             yield self.user_directory_handler.handle_local_profile_change(
                 target_user.to_string(), profile
             )
-        else:
-            yield self._notify_master_profile_change(
-                requester=requester,
-                user_id=target_user.to_string(),
-            )
 
         yield self._update_join_states(requester, target_user)
 
diff --git a/synapse/replication/http/__init__.py b/synapse/replication/http/__init__.py
index d8487df6147f..19f214281ef6 100644
--- a/synapse/replication/http/__init__.py
+++ b/synapse/replication/http/__init__.py
@@ -14,7 +14,7 @@
 # limitations under the License.
 
 from synapse.http.server import JsonResource
-from synapse.replication.http import federation, membership, profile, send_event
+from synapse.replication.http import federation, membership, send_event
 
 REPLICATION_PREFIX = "/_synapse/replication"
 
@@ -27,5 +27,4 @@ def __init__(self, hs):
     def register_servlets(self, hs):
         send_event.register_servlets(hs, self)
         membership.register_servlets(hs, self)
-        profile.register_servlets(hs, self)
         federation.register_servlets(hs, self)
diff --git a/synapse/replication/http/profile.py b/synapse/replication/http/profile.py
deleted file mode 100644
index aa700c243435..000000000000
--- a/synapse/replication/http/profile.py
+++ /dev/null
@@ -1,81 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2018 New Vector Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-
-from twisted.internet import defer
-
-from synapse.http.servlet import parse_json_object_from_request
-from synapse.replication.http._base import ReplicationEndpoint
-from synapse.types import Requester, UserID
-
-logger = logging.getLogger(__name__)
-
-
-class ReplicationHandleProfileChangeRestServlet(ReplicationEndpoint):
-    """Notifies that a users profile has changed
-
-    Request format:
-
-        POST /_synapse/replication/profile_changed/:user_id
-
-        {
-            "requester": ...
-        }
-    """
-
-    NAME = "profile_changed"
-    PATH_ARGS = ("user_id",)
-
-    def __init__(self, hs):
-        super(ReplicationHandleProfileChangeRestServlet, self).__init__(hs)
-
-        self.user_directory_handler = hs.get_user_directory_handler()
-        self.store = hs.get_datastore()
-        self.clock = hs.get_clock()
-
-    @staticmethod
-    def _serialize_payload(requester, user_id):
-        """
-        Args:
-            requester (Requester)
-            user_id (str)
-        """
-
-        return {
-            "requester": requester.serialize(),
-        }
-
-    @defer.inlineCallbacks
-    def _handle_request(self, request, user_id):
-        content = parse_json_object_from_request(request)
-
-        requester = Requester.deserialize(self.store, content["requester"])
-
-        if requester.user:
-            request.authenticated_entity = requester.user.to_string()
-
-        target_user = UserID.from_string(user_id)
-
-        profile = yield self.store.get_profileinfo(target_user.localpart)
-        yield self.user_directory_handler.handle_local_profile_change(
-            user_id, profile
-        )
-
-        defer.returnValue((200, {}))
-
-
-def register_servlets(hs, http_server):
-    ReplicationHandleProfileChangeRestServlet(hs).register(http_server)

From ab822a2d1fa30514185aa257b5c6a0af39a6f5f0 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 17 Aug 2018 15:30:31 +0100
Subject: [PATCH 450/830] Add some fixmes

---
 synapse/app/event_creator.py       | 2 ++
 synapse/handlers/user_directory.py | 4 ++++
 2 files changed, 6 insertions(+)

diff --git a/synapse/app/event_creator.py b/synapse/app/event_creator.py
index a2bdaf2a2c0b..a34c89fa9924 100644
--- a/synapse/app/event_creator.py
+++ b/synapse/app/event_creator.py
@@ -68,6 +68,8 @@
 
 
 class EventCreatorSlavedStore(
+    # FIXME(#3714): We need to add UserDirectoryStore as we write directly
+    # rather than going via the correct worker.
     UserDirectoryStore,
     DirectoryStore,
     SlavedTransactionStore,
diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py
index 37dda6458738..d8413d6aa71f 100644
--- a/synapse/handlers/user_directory.py
+++ b/synapse/handlers/user_directory.py
@@ -119,6 +119,8 @@ def handle_local_profile_change(self, user_id, profile):
         """Called to update index of our local user profiles when they change
         irrespective of any rooms the user may be in.
         """
+        # FIXME(#3714): We should probably do this in the same worker as all
+        # the other changes.
         yield self.store.update_profile_in_user_dir(
             user_id, profile.display_name, profile.avatar_url, None,
         )
@@ -127,6 +129,8 @@ def handle_local_profile_change(self, user_id, profile):
     def handle_user_deactivated(self, user_id):
         """Called when a user ID is deactivated
         """
+        # FIXME(#3714): We should probably do this in the same worker as all
+        # the other changes.
         yield self.store.remove_from_user_dir(user_id)
         yield self.store.remove_from_user_in_public_room(user_id)
 

From 04f5d2db62be5fd8ace903fb3c964bd50f6811cc Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Sat, 18 Aug 2018 00:55:01 +1000
Subject: [PATCH 451/830] Remove v1/register's broken shared secret
 functionality (#3703)

---
 changelog.d/3703.removal                |  1 +
 synapse/rest/client/v1_only/register.py | 54 -------------------------
 2 files changed, 1 insertion(+), 54 deletions(-)
 create mode 100644 changelog.d/3703.removal

diff --git a/changelog.d/3703.removal b/changelog.d/3703.removal
new file mode 100644
index 000000000000..d13ce5adb408
--- /dev/null
+++ b/changelog.d/3703.removal
@@ -0,0 +1 @@
+The Shared-Secret registration method of the legacy v1/register REST endpoint has been removed. For a replacement, please see [the admin/register API documentation](https://github.com/matrix-org/synapse/blob/master/docs/admin_api/register_api.rst).
diff --git a/synapse/rest/client/v1_only/register.py b/synapse/rest/client/v1_only/register.py
index 3439c3c6d454..5e99cffbcbd1 100644
--- a/synapse/rest/client/v1_only/register.py
+++ b/synapse/rest/client/v1_only/register.py
@@ -129,12 +129,9 @@ def on_POST(self, request):
             login_type = register_json["type"]
 
             is_application_server = login_type == LoginType.APPLICATION_SERVICE
-            is_using_shared_secret = login_type == LoginType.SHARED_SECRET
-
             can_register = (
                 self.enable_registration
                 or is_application_server
-                or is_using_shared_secret
             )
             if not can_register:
                 raise SynapseError(403, "Registration has been disabled")
@@ -144,7 +141,6 @@ def on_POST(self, request):
                 LoginType.PASSWORD: self._do_password,
                 LoginType.EMAIL_IDENTITY: self._do_email_identity,
                 LoginType.APPLICATION_SERVICE: self._do_app_service,
-                LoginType.SHARED_SECRET: self._do_shared_secret,
             }
 
             session_info = self._get_session_info(request, session)
@@ -325,56 +321,6 @@ def _do_app_service(self, request, register_json, session):
             "home_server": self.hs.hostname,
         })
 
-    @defer.inlineCallbacks
-    def _do_shared_secret(self, request, register_json, session):
-        assert_params_in_dict(register_json, ["mac", "user", "password"])
-
-        if not self.hs.config.registration_shared_secret:
-            raise SynapseError(400, "Shared secret registration is not enabled")
-
-        user = register_json["user"].encode("utf-8")
-        password = register_json["password"].encode("utf-8")
-        admin = register_json.get("admin", None)
-
-        # Its important to check as we use null bytes as HMAC field separators
-        if b"\x00" in user:
-            raise SynapseError(400, "Invalid user")
-        if b"\x00" in password:
-            raise SynapseError(400, "Invalid password")
-
-        # str() because otherwise hmac complains that 'unicode' does not
-        # have the buffer interface
-        got_mac = str(register_json["mac"])
-
-        want_mac = hmac.new(
-            key=self.hs.config.registration_shared_secret.encode(),
-            digestmod=sha1,
-        )
-        want_mac.update(user)
-        want_mac.update(b"\x00")
-        want_mac.update(password)
-        want_mac.update(b"\x00")
-        want_mac.update(b"admin" if admin else b"notadmin")
-        want_mac = want_mac.hexdigest()
-
-        if compare_digest(want_mac, got_mac):
-            handler = self.handlers.registration_handler
-            user_id, token = yield handler.register(
-                localpart=user.lower(),
-                password=password,
-                admin=bool(admin),
-            )
-            self._remove_session(session)
-            defer.returnValue({
-                "user_id": user_id,
-                "access_token": token,
-                "home_server": self.hs.hostname,
-            })
-        else:
-            raise SynapseError(
-                403, "HMAC incorrect",
-            )
-
 
 class CreateUserRestServlet(ClientV1RestServlet):
     """Handles user creation via a server-to-server interface

From c334ca67bb89039b3a00b7c9a1ce610e99859653 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Sat, 18 Aug 2018 01:08:45 +1000
Subject: [PATCH 452/830] Integrate presence from hotfixes (#3694)

---
 changelog.d/3694.feature                |  1 +
 docs/workers.rst                        |  8 +++
 synapse/app/_base.py                    |  6 +-
 synapse/app/frontend_proxy.py           | 39 ++++++++++-
 synapse/app/synchrotron.py              | 16 +++--
 synapse/config/server.py                |  6 ++
 synapse/federation/transaction_queue.py |  4 ++
 synapse/handlers/initial_sync.py        |  4 ++
 synapse/handlers/presence.py            | 26 ++++++--
 synapse/handlers/sync.py                |  3 +-
 synapse/rest/client/v1/presence.py      |  3 +-
 tests/app/__init__.py                   |  0
 tests/app/test_frontend_proxy.py        | 88 +++++++++++++++++++++++++
 tests/rest/client/v1/test_presence.py   | 72 ++++++++++++++++++++
 tests/rest/client/v2_alpha/test_sync.py | 79 ++++++++++------------
 tests/unittest.py                       |  8 ++-
 tests/utils.py                          |  7 +-
 17 files changed, 303 insertions(+), 67 deletions(-)
 create mode 100644 changelog.d/3694.feature
 create mode 100644 tests/app/__init__.py
 create mode 100644 tests/app/test_frontend_proxy.py
 create mode 100644 tests/rest/client/v1/test_presence.py

diff --git a/changelog.d/3694.feature b/changelog.d/3694.feature
new file mode 100644
index 000000000000..916a342ff48f
--- /dev/null
+++ b/changelog.d/3694.feature
@@ -0,0 +1 @@
+Synapse's presence functionality can now be disabled with the "use_presence" configuration option.
diff --git a/docs/workers.rst b/docs/workers.rst
index ac9efb621f06..aec319dd8458 100644
--- a/docs/workers.rst
+++ b/docs/workers.rst
@@ -241,6 +241,14 @@ regular expressions::
 
     ^/_matrix/client/(api/v1|r0|unstable)/keys/upload
 
+If ``use_presence`` is False in the homeserver config, it can also handle REST
+endpoints matching the following regular expressions::
+
+    ^/_matrix/client/(api/v1|r0|unstable)/presence/[^/]+/status
+
+This "stub" presence handler will pass through ``GET`` request but make the
+``PUT`` effectively a no-op.
+
 It will proxy any requests it cannot handle to the main synapse instance. It
 must therefore be configured with the location of the main instance, via
 the ``worker_main_http_uri`` setting in the frontend_proxy worker configuration
diff --git a/synapse/app/_base.py b/synapse/app/_base.py
index 391bd14c5c9c..7c866e246af8 100644
--- a/synapse/app/_base.py
+++ b/synapse/app/_base.py
@@ -140,7 +140,7 @@ def listen_metrics(bind_addresses, port):
         logger.info("Metrics now reporting on %s:%d", host, port)
 
 
-def listen_tcp(bind_addresses, port, factory, backlog=50):
+def listen_tcp(bind_addresses, port, factory, reactor=reactor, backlog=50):
     """
     Create a TCP socket for a port and several addresses
     """
@@ -156,7 +156,9 @@ def listen_tcp(bind_addresses, port, factory, backlog=50):
             check_bind_error(e, address, bind_addresses)
 
 
-def listen_ssl(bind_addresses, port, factory, context_factory, backlog=50):
+def listen_ssl(
+    bind_addresses, port, factory, context_factory, reactor=reactor, backlog=50
+):
     """
     Create an SSL socket for a port and several addresses
     """
diff --git a/synapse/app/frontend_proxy.py b/synapse/app/frontend_proxy.py
index 671fbbcb2a9d..8d484c1cd4a0 100644
--- a/synapse/app/frontend_proxy.py
+++ b/synapse/app/frontend_proxy.py
@@ -38,6 +38,7 @@
 from synapse.replication.slave.storage.devices import SlavedDeviceStore
 from synapse.replication.slave.storage.registration import SlavedRegistrationStore
 from synapse.replication.tcp.client import ReplicationClientHandler
+from synapse.rest.client.v1.base import ClientV1RestServlet, client_path_patterns
 from synapse.rest.client.v2_alpha._base import client_v2_patterns
 from synapse.server import HomeServer
 from synapse.storage.engines import create_engine
@@ -49,6 +50,35 @@
 logger = logging.getLogger("synapse.app.frontend_proxy")
 
 
+class PresenceStatusStubServlet(ClientV1RestServlet):
+    PATTERNS = client_path_patterns("/presence/(?P<user_id>[^/]*)/status")
+
+    def __init__(self, hs):
+        super(PresenceStatusStubServlet, self).__init__(hs)
+        self.http_client = hs.get_simple_http_client()
+        self.auth = hs.get_auth()
+        self.main_uri = hs.config.worker_main_http_uri
+
+    @defer.inlineCallbacks
+    def on_GET(self, request, user_id):
+        # Pass through the auth headers, if any, in case the access token
+        # is there.
+        auth_headers = request.requestHeaders.getRawHeaders("Authorization", [])
+        headers = {
+            "Authorization": auth_headers,
+        }
+        result = yield self.http_client.get_json(
+            self.main_uri + request.uri,
+            headers=headers,
+        )
+        defer.returnValue((200, result))
+
+    @defer.inlineCallbacks
+    def on_PUT(self, request, user_id):
+        yield self.auth.get_user_by_req(request)
+        defer.returnValue((200, {}))
+
+
 class KeyUploadServlet(RestServlet):
     PATTERNS = client_v2_patterns("/keys/upload(/(?P<device_id>[^/]+))?$")
 
@@ -135,6 +165,12 @@ def _listen_http(self, listener_config):
                 elif name == "client":
                     resource = JsonResource(self, canonical_json=False)
                     KeyUploadServlet(self).register(resource)
+
+                    # If presence is disabled, use the stub servlet that does
+                    # not allow sending presence
+                    if not self.config.use_presence:
+                        PresenceStatusStubServlet(self).register(resource)
+
                     resources.update({
                         "/_matrix/client/r0": resource,
                         "/_matrix/client/unstable": resource,
@@ -153,7 +189,8 @@ def _listen_http(self, listener_config):
                 listener_config,
                 root_resource,
                 self.version_string,
-            )
+            ),
+            reactor=self.get_reactor()
         )
 
         logger.info("Synapse client reader now listening on port %d", port)
diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py
index e201f18efdc5..cade09d60e37 100644
--- a/synapse/app/synchrotron.py
+++ b/synapse/app/synchrotron.py
@@ -114,7 +114,10 @@ def __init__(self, hs):
         logger.info("Presence process_id is %r", self.process_id)
 
     def send_user_sync(self, user_id, is_syncing, last_sync_ms):
-        self.hs.get_tcp_replication().send_user_sync(user_id, is_syncing, last_sync_ms)
+        if self.hs.config.use_presence:
+            self.hs.get_tcp_replication().send_user_sync(
+                user_id, is_syncing, last_sync_ms
+            )
 
     def mark_as_coming_online(self, user_id):
         """A user has started syncing. Send a UserSync to the master, unless they
@@ -211,10 +214,13 @@ def process_replication_rows(self, token, rows):
         yield self.notify_from_replication(states, stream_id)
 
     def get_currently_syncing_users(self):
-        return [
-            user_id for user_id, count in iteritems(self.user_to_num_current_syncs)
-            if count > 0
-        ]
+        if self.hs.config.use_presence:
+            return [
+                user_id for user_id, count in iteritems(self.user_to_num_current_syncs)
+                if count > 0
+            ]
+        else:
+            return set()
 
 
 class SynchrotronTyping(object):
diff --git a/synapse/config/server.py b/synapse/config/server.py
index a41c48e69c70..68a612e59482 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -49,6 +49,9 @@ def read_config(self, config):
         # "disable" federation
         self.send_federation = config.get("send_federation", True)
 
+        # Whether to enable user presence.
+        self.use_presence = config.get("use_presence", True)
+
         # Whether to update the user directory or not. This should be set to
         # false only if we are updating the user directory in a worker
         self.update_user_directory = config.get("update_user_directory", True)
@@ -250,6 +253,9 @@ def default_config(self, server_name, **kwargs):
         # hard limit.
         soft_file_limit: 0
 
+        # Set to false to disable presence tracking on this homeserver.
+        use_presence: true
+
         # The GC threshold parameters to pass to `gc.set_threshold`, if defined
         # gc_thresholds: [700, 10, 10]
 
diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py
index f603c8a368aa..94d7423d016a 100644
--- a/synapse/federation/transaction_queue.py
+++ b/synapse/federation/transaction_queue.py
@@ -58,6 +58,7 @@ class TransactionQueue(object):
     """
 
     def __init__(self, hs):
+        self.hs = hs
         self.server_name = hs.hostname
 
         self.store = hs.get_datastore()
@@ -308,6 +309,9 @@ def send_presence(self, states):
         Args:
             states (list(UserPresenceState))
         """
+        if not self.hs.config.use_presence:
+            # No-op if presence is disabled.
+            return
 
         # First we queue up the new presence by user ID, so multiple presence
         # updates in quick successtion are correctly handled
diff --git a/synapse/handlers/initial_sync.py b/synapse/handlers/initial_sync.py
index 1fb17fd9a5b1..e0093952074f 100644
--- a/synapse/handlers/initial_sync.py
+++ b/synapse/handlers/initial_sync.py
@@ -372,6 +372,10 @@ def _room_initial_sync_joined(self, user_id, room_id, pagin_config,
 
         @defer.inlineCallbacks
         def get_presence():
+            # If presence is disabled, return an empty list
+            if not self.hs.config.use_presence:
+                defer.returnValue([])
+
             states = yield presence_handler.get_states(
                 [m.user_id for m in room_members],
                 as_event=True,
diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index 3671d24f60d0..ba3856674d0e 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -395,6 +395,10 @@ def bump_presence_active_time(self, user):
         """We've seen the user do something that indicates they're interacting
         with the app.
         """
+        # If presence is disabled, no-op
+        if not self.hs.config.use_presence:
+            return
+
         user_id = user.to_string()
 
         bump_active_time_counter.inc()
@@ -424,6 +428,11 @@ def user_syncing(self, user_id, affect_presence=True):
                 Useful for streams that are not associated with an actual
                 client that is being used by a user.
         """
+        # Override if it should affect the user's presence, if presence is
+        # disabled.
+        if not self.hs.config.use_presence:
+            affect_presence = False
+
         if affect_presence:
             curr_sync = self.user_to_num_current_syncs.get(user_id, 0)
             self.user_to_num_current_syncs[user_id] = curr_sync + 1
@@ -469,13 +478,16 @@ def get_currently_syncing_users(self):
         Returns:
             set(str): A set of user_id strings.
         """
-        syncing_user_ids = {
-            user_id for user_id, count in self.user_to_num_current_syncs.items()
-            if count
-        }
-        for user_ids in self.external_process_to_current_syncs.values():
-            syncing_user_ids.update(user_ids)
-        return syncing_user_ids
+        if self.hs.config.use_presence:
+            syncing_user_ids = {
+                user_id for user_id, count in self.user_to_num_current_syncs.items()
+                if count
+            }
+            for user_ids in self.external_process_to_current_syncs.values():
+                syncing_user_ids.update(user_ids)
+            return syncing_user_ids
+        else:
+            return set()
 
     @defer.inlineCallbacks
     def update_external_syncs_row(self, process_id, user_id, is_syncing, sync_time_msec):
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index ac3edf0cc9bc..648debc8aa96 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -185,6 +185,7 @@ def __nonzero__(self):
 class SyncHandler(object):
 
     def __init__(self, hs):
+        self.hs_config = hs.config
         self.store = hs.get_datastore()
         self.notifier = hs.get_notifier()
         self.presence_handler = hs.get_presence_handler()
@@ -860,7 +861,7 @@ def generate_sync_result(self, sync_config, since_token=None, full_state=False):
             since_token is None and
             sync_config.filter_collection.blocks_all_presence()
         )
-        if not block_all_presence_data:
+        if self.hs_config.use_presence and not block_all_presence_data:
             yield self._generate_sync_entry_for_presence(
                 sync_result_builder, newly_joined_rooms, newly_joined_users
             )
diff --git a/synapse/rest/client/v1/presence.py b/synapse/rest/client/v1/presence.py
index a14f0c807e2e..b5a6d6aebfc7 100644
--- a/synapse/rest/client/v1/presence.py
+++ b/synapse/rest/client/v1/presence.py
@@ -84,7 +84,8 @@ def on_PUT(self, request, user_id):
         except Exception:
             raise SynapseError(400, "Unable to parse state")
 
-        yield self.presence_handler.set_state(user, state)
+        if self.hs.config.use_presence:
+            yield self.presence_handler.set_state(user, state)
 
         defer.returnValue((200, {}))
 
diff --git a/tests/app/__init__.py b/tests/app/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/tests/app/test_frontend_proxy.py b/tests/app/test_frontend_proxy.py
new file mode 100644
index 000000000000..76b5090fffd1
--- /dev/null
+++ b/tests/app/test_frontend_proxy.py
@@ -0,0 +1,88 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from synapse.app.frontend_proxy import FrontendProxyServer
+
+from tests.unittest import HomeserverTestCase
+
+
+class FrontendProxyTests(HomeserverTestCase):
+    def make_homeserver(self, reactor, clock):
+
+        hs = self.setup_test_homeserver(
+            http_client=None, homeserverToUse=FrontendProxyServer
+        )
+
+        return hs
+
+    def test_listen_http_with_presence_enabled(self):
+        """
+        When presence is on, the stub servlet will not register.
+        """
+        # Presence is on
+        self.hs.config.use_presence = True
+
+        config = {
+            "port": 8080,
+            "bind_addresses": ["0.0.0.0"],
+            "resources": [{"names": ["client"]}],
+        }
+
+        # Listen with the config
+        self.hs._listen_http(config)
+
+        # Grab the resource from the site that was told to listen
+        self.assertEqual(len(self.reactor.tcpServers), 1)
+        site = self.reactor.tcpServers[0][1]
+        self.resource = (
+            site.resource.children["_matrix"].children["client"].children["r0"]
+        )
+
+        request, channel = self.make_request("PUT", "presence/a/status")
+        self.render(request)
+
+        # 400 + unrecognised, because nothing is registered
+        self.assertEqual(channel.code, 400)
+        self.assertEqual(channel.json_body["errcode"], "M_UNRECOGNIZED")
+
+    def test_listen_http_with_presence_disabled(self):
+        """
+        When presence is on, the stub servlet will register.
+        """
+        # Presence is off
+        self.hs.config.use_presence = False
+
+        config = {
+            "port": 8080,
+            "bind_addresses": ["0.0.0.0"],
+            "resources": [{"names": ["client"]}],
+        }
+
+        # Listen with the config
+        self.hs._listen_http(config)
+
+        # Grab the resource from the site that was told to listen
+        self.assertEqual(len(self.reactor.tcpServers), 1)
+        site = self.reactor.tcpServers[0][1]
+        self.resource = (
+            site.resource.children["_matrix"].children["client"].children["r0"]
+        )
+
+        request, channel = self.make_request("PUT", "presence/a/status")
+        self.render(request)
+
+        # 401, because the stub servlet still checks authentication
+        self.assertEqual(channel.code, 401)
+        self.assertEqual(channel.json_body["errcode"], "M_MISSING_TOKEN")
diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py
new file mode 100644
index 000000000000..66c2b687079c
--- /dev/null
+++ b/tests/rest/client/v1/test_presence.py
@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from mock import Mock
+
+from synapse.rest.client.v1 import presence
+from synapse.types import UserID
+
+from tests import unittest
+
+
+class PresenceTestCase(unittest.HomeserverTestCase):
+    """ Tests presence REST API. """
+
+    user_id = "@sid:red"
+
+    user = UserID.from_string(user_id)
+    servlets = [presence.register_servlets]
+
+    def make_homeserver(self, reactor, clock):
+
+        hs = self.setup_test_homeserver(
+            "red", http_client=None, federation_client=Mock()
+        )
+
+        hs.presence_handler = Mock()
+
+        return hs
+
+    def test_put_presence(self):
+        """
+        PUT to the status endpoint with use_presence enabled will call
+        set_state on the presence handler.
+        """
+        self.hs.config.use_presence = True
+
+        body = {"presence": "here", "status_msg": "beep boop"}
+        request, channel = self.make_request(
+            "PUT", "/presence/%s/status" % (self.user_id,), body
+        )
+        self.render(request)
+
+        self.assertEqual(channel.code, 200)
+        self.assertEqual(self.hs.presence_handler.set_state.call_count, 1)
+
+    def test_put_presence_disabled(self):
+        """
+        PUT to the status endpoint with use_presence disbled will NOT call
+        set_state on the presence handler.
+        """
+        self.hs.config.use_presence = False
+
+        body = {"presence": "here", "status_msg": "beep boop"}
+        request, channel = self.make_request(
+            "PUT", "/presence/%s/status" % (self.user_id,), body
+        )
+        self.render(request)
+
+        self.assertEqual(channel.code, 200)
+        self.assertEqual(self.hs.presence_handler.set_state.call_count, 0)
diff --git a/tests/rest/client/v2_alpha/test_sync.py b/tests/rest/client/v2_alpha/test_sync.py
index 9f3d8bd1dbb4..560b1fba967a 100644
--- a/tests/rest/client/v2_alpha/test_sync.py
+++ b/tests/rest/client/v2_alpha/test_sync.py
@@ -13,71 +13,58 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import synapse.types
-from synapse.http.server import JsonResource
+from mock import Mock
+
 from synapse.rest.client.v2_alpha import sync
-from synapse.types import UserID
-from synapse.util import Clock
 
 from tests import unittest
-from tests.server import (
-    ThreadedMemoryReactorClock as MemoryReactorClock,
-    make_request,
-    render,
-    setup_test_homeserver,
-)
-
-PATH_PREFIX = "/_matrix/client/v2_alpha"
 
 
-class FilterTestCase(unittest.TestCase):
+class FilterTestCase(unittest.HomeserverTestCase):
 
-    USER_ID = "@apple:test"
-    TO_REGISTER = [sync]
+    user_id = "@apple:test"
+    servlets = [sync.register_servlets]
 
-    def setUp(self):
-        self.clock = MemoryReactorClock()
-        self.hs_clock = Clock(self.clock)
+    def make_homeserver(self, reactor, clock):
 
-        self.hs = setup_test_homeserver(
-            self.addCleanup, http_client=None, clock=self.hs_clock, reactor=self.clock
+        hs = self.setup_test_homeserver(
+            "red", http_client=None, federation_client=Mock()
         )
+        return hs
 
-        self.auth = self.hs.get_auth()
-
-        def get_user_by_access_token(token=None, allow_guest=False):
-            return {
-                "user": UserID.from_string(self.USER_ID),
-                "token_id": 1,
-                "is_guest": False,
-            }
-
-        def get_user_by_req(request, allow_guest=False, rights="access"):
-            return synapse.types.create_requester(
-                UserID.from_string(self.USER_ID), 1, False, None
-            )
-
-        self.auth.get_user_by_access_token = get_user_by_access_token
-        self.auth.get_user_by_req = get_user_by_req
+    def test_sync_argless(self):
+        request, channel = self.make_request("GET", "/sync")
+        self.render(request)
 
-        self.store = self.hs.get_datastore()
-        self.filtering = self.hs.get_filtering()
-        self.resource = JsonResource(self.hs)
+        self.assertEqual(channel.code, 200)
+        self.assertTrue(
+            set(
+                [
+                    "next_batch",
+                    "rooms",
+                    "presence",
+                    "account_data",
+                    "to_device",
+                    "device_lists",
+                ]
+            ).issubset(set(channel.json_body.keys()))
+        )
 
-        for r in self.TO_REGISTER:
-            r.register_servlets(self.hs, self.resource)
+    def test_sync_presence_disabled(self):
+        """
+        When presence is disabled, the key does not appear in /sync.
+        """
+        self.hs.config.use_presence = False
 
-    def test_sync_argless(self):
-        request, channel = make_request("GET", "/_matrix/client/r0/sync")
-        render(request, self.resource, self.clock)
+        request, channel = self.make_request("GET", "/sync")
+        self.render(request)
 
-        self.assertEqual(channel.result["code"], b"200")
+        self.assertEqual(channel.code, 200)
         self.assertTrue(
             set(
                 [
                     "next_batch",
                     "rooms",
-                    "presence",
                     "account_data",
                     "to_device",
                     "device_lists",
diff --git a/tests/unittest.py b/tests/unittest.py
index e6afe3b96d61..d852e2465a06 100644
--- a/tests/unittest.py
+++ b/tests/unittest.py
@@ -18,6 +18,8 @@
 
 from mock import Mock
 
+from canonicaljson import json
+
 import twisted
 import twisted.logger
 from twisted.trial import unittest
@@ -241,11 +243,15 @@ def make_request(self, method, path, content=b""):
             method (bytes/unicode): The HTTP request method ("verb").
             path (bytes/unicode): The HTTP path, suitably URL encoded (e.g.
             escaped UTF-8 & spaces and such).
-            content (bytes): The body of the request.
+            content (bytes or dict): The body of the request. JSON-encoded, if
+            a dict.
 
         Returns:
             A synapse.http.site.SynapseRequest.
         """
+        if isinstance(content, dict):
+            content = json.dumps(content).encode('utf8')
+
         return make_request(method, path, content)
 
     def render(self, request):
diff --git a/tests/utils.py b/tests/utils.py
index 6f8b1de3e799..bb0fc740540c 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -93,7 +93,8 @@ def _cleanup():
 
 @defer.inlineCallbacks
 def setup_test_homeserver(
-    cleanup_func, name="test", datastore=None, config=None, reactor=None, **kargs
+    cleanup_func, name="test", datastore=None, config=None, reactor=None,
+    homeserverToUse=HomeServer, **kargs
 ):
     """
     Setup a homeserver suitable for running tests against.  Keyword arguments
@@ -192,7 +193,7 @@ def setup_test_homeserver(
     config.database_config["args"]["cp_openfun"] = db_engine.on_new_connection
 
     if datastore is None:
-        hs = HomeServer(
+        hs = homeserverToUse(
             name,
             config=config,
             db_config=config.database_config,
@@ -235,7 +236,7 @@ def cleanup():
 
         hs.setup()
     else:
-        hs = HomeServer(
+        hs = homeserverToUse(
             name,
             db_pool=None,
             datastore=datastore,

From ba1fbf7d5bd8807e892de6de4bc82cb5e49581bd Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Sat, 18 Aug 2018 12:31:08 +0100
Subject: [PATCH 453/830] special case server_notices_mxid

---
 synapse/server_notices/resource_limits_server_notices.py | 2 +-
 tests/api/test_auth.py                                   | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/synapse/server_notices/resource_limits_server_notices.py b/synapse/server_notices/resource_limits_server_notices.py
index 5987e448b236..84b91aeb5dcb 100644
--- a/synapse/server_notices/resource_limits_server_notices.py
+++ b/synapse/server_notices/resource_limits_server_notices.py
@@ -173,7 +173,7 @@ def _is_room_currently_blocked(self, room_id):
             referenced_events = pinned_state_event.content.get('pinned')
 
         events = yield self._store.get_events(referenced_events)
-        for event_id, event in events.iteritems():
+        for event_id, event in iteritems(events):
             if event.type == EventTypes.ServerNoticeLimitReached:
                 currently_blocked = True
                 # remove event in case we need to disable blocking later on.
diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py
index 022d81ce3e9d..c4cbff4e8daa 100644
--- a/tests/api/test_auth.py
+++ b/tests/api/test_auth.py
@@ -476,3 +476,11 @@ def test_hs_disabled(self):
         self.assertEquals(e.exception.admin_uri, self.hs.config.admin_uri)
         self.assertEquals(e.exception.errcode, Codes.RESOURCE_LIMIT_EXCEED)
         self.assertEquals(e.exception.code, 403)
+
+    @defer.inlineCallbacks
+    def test_server_notices_mxid_special_cased(self):
+        self.hs.config.hs_disabled = True
+        user = "@user:server"
+        self.hs.config.server_notices_mxid = user
+        self.hs.config.hs_disabled_message = "Reason for being disabled"
+        yield self.auth.check_auth_blocking(user)

From c5171bf171ae28596c4e101e9dcc61bad7bcae63 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Sat, 18 Aug 2018 12:33:07 +0100
Subject: [PATCH 454/830] special case server_notices_mxid

---
 synapse/api/auth.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 022211e34e0f..55384d6ffec6 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -783,6 +783,12 @@ def check_auth_blocking(self, user_id=None):
             user_id(str|None): If present, checks for presence against existing
             MAU cohort
         """
+
+        # Never fail an auth check for the server notices users
+        # This can be a problem where event creation is prohibited due to blocking
+        if user_id == self.hs.config.server_notices_mxid:
+            return
+
         if self.hs.config.hs_disabled:
             raise ResourceLimitError(
                 403, self.hs.config.hs_disabled_message,

From e07970165f852ccbc4542f1aaf0fd1b2bc54b973 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Sat, 18 Aug 2018 14:39:45 +0100
Subject: [PATCH 455/830] rename error code

---
 synapse/api/auth.py         | 4 ++--
 synapse/api/errors.py       | 4 ++--
 tests/api/test_auth.py      | 4 ++--
 tests/handlers/test_sync.py | 4 ++--
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 55384d6ffec6..4207a48afd29 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -792,7 +792,7 @@ def check_auth_blocking(self, user_id=None):
         if self.hs.config.hs_disabled:
             raise ResourceLimitError(
                 403, self.hs.config.hs_disabled_message,
-                errcode=Codes.RESOURCE_LIMIT_EXCEED,
+                errcode=Codes.RESOURCE_LIMIT_EXCEEDED,
                 admin_uri=self.hs.config.admin_uri,
                 limit_type=self.hs.config.hs_disabled_limit_type
             )
@@ -809,6 +809,6 @@ def check_auth_blocking(self, user_id=None):
                     403, "Monthly Active User Limit Exceeded",
 
                     admin_uri=self.hs.config.admin_uri,
-                    errcode=Codes.RESOURCE_LIMIT_EXCEED,
+                    errcode=Codes.RESOURCE_LIMIT_EXCEEDED,
                     limit_type="monthly_active_user"
                 )
diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index e26001ab12c4..c4ddba9889dd 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -56,7 +56,7 @@ class Codes(object):
     SERVER_NOT_TRUSTED = "M_SERVER_NOT_TRUSTED"
     CONSENT_NOT_GIVEN = "M_CONSENT_NOT_GIVEN"
     CANNOT_LEAVE_SERVER_NOTICE_ROOM = "M_CANNOT_LEAVE_SERVER_NOTICE_ROOM"
-    RESOURCE_LIMIT_EXCEED = "M_RESOURCE_LIMIT_EXCEED"
+    RESOURCE_LIMIT_EXCEEDED = "M_RESOURCE_LIMIT_EXCEEDED"
     UNSUPPORTED_ROOM_VERSION = "M_UNSUPPORTED_ROOM_VERSION"
     INCOMPATIBLE_ROOM_VERSION = "M_INCOMPATIBLE_ROOM_VERSION"
 
@@ -238,7 +238,7 @@ class ResourceLimitError(SynapseError):
     """
     def __init__(
         self, code, msg,
-        errcode=Codes.RESOURCE_LIMIT_EXCEED,
+        errcode=Codes.RESOURCE_LIMIT_EXCEEDED,
         admin_uri=None,
         limit_type=None,
     ):
diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py
index c4cbff4e8daa..ed960090c426 100644
--- a/tests/api/test_auth.py
+++ b/tests/api/test_auth.py
@@ -458,7 +458,7 @@ def test_blocking_mau(self):
         with self.assertRaises(ResourceLimitError) as e:
             yield self.auth.check_auth_blocking()
         self.assertEquals(e.exception.admin_uri, self.hs.config.admin_uri)
-        self.assertEquals(e.exception.errcode, Codes.RESOURCE_LIMIT_EXCEED)
+        self.assertEquals(e.exception.errcode, Codes.RESOURCE_LIMIT_EXCEEDED)
         self.assertEquals(e.exception.code, 403)
 
         # Ensure does not throw an error
@@ -474,7 +474,7 @@ def test_hs_disabled(self):
         with self.assertRaises(ResourceLimitError) as e:
             yield self.auth.check_auth_blocking()
         self.assertEquals(e.exception.admin_uri, self.hs.config.admin_uri)
-        self.assertEquals(e.exception.errcode, Codes.RESOURCE_LIMIT_EXCEED)
+        self.assertEquals(e.exception.errcode, Codes.RESOURCE_LIMIT_EXCEEDED)
         self.assertEquals(e.exception.code, 403)
 
     @defer.inlineCallbacks
diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py
index a01ab471f594..31f54bbd7daf 100644
--- a/tests/handlers/test_sync.py
+++ b/tests/handlers/test_sync.py
@@ -51,7 +51,7 @@ def test_wait_for_sync_for_user_auth_blocking(self):
         self.hs.config.hs_disabled = True
         with self.assertRaises(ResourceLimitError) as e:
             yield self.sync_handler.wait_for_sync_for_user(sync_config)
-        self.assertEquals(e.exception.errcode, Codes.RESOURCE_LIMIT_EXCEED)
+        self.assertEquals(e.exception.errcode, Codes.RESOURCE_LIMIT_EXCEEDED)
 
         self.hs.config.hs_disabled = False
 
@@ -59,7 +59,7 @@ def test_wait_for_sync_for_user_auth_blocking(self):
 
         with self.assertRaises(ResourceLimitError) as e:
             yield self.sync_handler.wait_for_sync_for_user(sync_config)
-        self.assertEquals(e.exception.errcode, Codes.RESOURCE_LIMIT_EXCEED)
+        self.assertEquals(e.exception.errcode, Codes.RESOURCE_LIMIT_EXCEEDED)
 
     def _generate_sync_config(self, user_id):
         return SyncConfig(

From f2a48d87df3129bd32597cb7e28496ade0fa8250 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 20 Aug 2018 13:01:46 +0100
Subject: [PATCH 456/830] Use get_cache_factor_for function for `state_cache`

This allows the cache factor for `state_cache` to be individually
specified in the enviroment
---
 synapse/state.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/synapse/state.py b/synapse/state.py
index 8b92d4057a2a..0f2bedb69449 100644
--- a/synapse/state.py
+++ b/synapse/state.py
@@ -29,7 +29,7 @@
 from synapse.api.errors import AuthError
 from synapse.events.snapshot import EventContext
 from synapse.util.async_helpers import Linearizer
-from synapse.util.caches import CACHE_SIZE_FACTOR
+from synapse.util.caches import get_cache_factor_for
 from synapse.util.caches.expiringcache import ExpiringCache
 from synapse.util.logutils import log_function
 from synapse.util.metrics import Measure
@@ -40,7 +40,7 @@
 KeyStateTuple = namedtuple("KeyStateTuple", ("context", "type", "state_key"))
 
 
-SIZE_OF_CACHE = int(100000 * CACHE_SIZE_FACTOR)
+SIZE_OF_CACHE = 100000 * get_cache_factor_for("state_cache")
 EVICTION_TIMEOUT_SECONDS = 60 * 60
 
 

From 48a910e128a560a8572de403eeed19734367b8f6 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 20 Aug 2018 13:04:28 +0100
Subject: [PATCH 457/830] Newsfile

---
 changelog.d/3719.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3719.bugfix

diff --git a/changelog.d/3719.bugfix b/changelog.d/3719.bugfix
new file mode 100644
index 000000000000..095d9288e897
--- /dev/null
+++ b/changelog.d/3719.bugfix
@@ -0,0 +1 @@
+Fix bug where `state_cache` cache factor ignored environment variables

From e21c368b8b7c62b2f626ff6c739ea33c3c2e73bb Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 20 Aug 2018 13:54:51 +0100
Subject: [PATCH 458/830] Revert spurious change

---
 synapse/handlers/room_member.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index b6010bb41eca..b8e1af580bb9 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -341,9 +341,9 @@ def _update_membership(
         prev_events_and_hashes = yield self.store.get_prev_events_for_room(
             room_id,
         )
-        latest_event_ids = [
+        latest_event_ids = (
             event_id for (event_id, _, _) in prev_events_and_hashes
-        ]
+        )
 
         current_state_ids = yield self.state_handler.get_current_state_ids(
             room_id, latest_event_ids=latest_event_ids,

From 8dee601054089311b81efd6239b458401da063e3 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 20 Aug 2018 14:21:12 +0100
Subject: [PATCH 459/830] Remove redundant room_version checks

---
 synapse/handlers/federation.py |  2 +-
 synapse/state/__init__.py      | 17 +++--------------
 2 files changed, 4 insertions(+), 15 deletions(-)

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 75a819dd116f..14a10c122941 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -1814,7 +1814,7 @@ def do_auth(self, origin, event, context, auth_events):
 
                 room_version = yield self.store.get_room_version(event.room_id)
 
-                new_state = yield self.state_handler.resolve_events(
+                new_state = self.state_handler.resolve_events(
                     room_version,
                     [list(local_view.values()), list(remote_view.values())],
                     event
diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py
index f1c26d10719e..a2992fd953ec 100644
--- a/synapse/state/__init__.py
+++ b/synapse/state/__init__.py
@@ -262,14 +262,9 @@ def compute_event_context(self, event, old_state=None):
             defer.returnValue(context)
 
         logger.debug("calling resolve_state_groups from compute_event_context")
-        if event.type == EventTypes.Create:
-            room_version = event.content.get("room_version", RoomVersions.V1)
-        else:
-            room_version = None
 
         entry = yield self.resolve_state_groups_for_events(
             event.room_id, [e for e, _ in event.prev_events],
-            explicit_room_version=room_version,
         )
 
         prev_state_ids = entry.state
@@ -337,8 +332,7 @@ def compute_event_context(self, event, old_state=None):
         defer.returnValue(context)
 
     @defer.inlineCallbacks
-    def resolve_state_groups_for_events(self, room_id, event_ids,
-                                        explicit_room_version=None):
+    def resolve_state_groups_for_events(self, room_id, event_ids):
         """ Given a list of event_ids this method fetches the state at each
         event, resolves conflicts between them and returns them.
 
@@ -378,9 +372,7 @@ def resolve_state_groups_for_events(self, room_id, event_ids,
                 delta_ids=delta_ids,
             ))
 
-        room_version = explicit_room_version
-        if not room_version:
-            room_version = yield self.store.get_room_version(room_id)
+        room_version = yield self.store.get_room_version(room_id)
 
         result = yield self._state_resolution_handler.resolve_state_groups(
             room_id, room_version, state_groups_ids, None,
@@ -393,7 +385,6 @@ def _state_map_factory(self, ev_ids):
             ev_ids, get_prev_content=False, check_redacted=False,
         )
 
-    @defer.inlineCallbacks
     def resolve_events(self, room_version, state_sets, event):
         logger.info(
             "Resolving state for %s with %d groups", event.room_id, len(state_sets)
@@ -409,8 +400,6 @@ def resolve_events(self, room_version, state_sets, event):
             for ev in st
         }
 
-        room_version = yield self.store.get_room_version(event.room_id)
-
         with Measure(self.clock, "state._resolve_events"):
             new_state = resolve_events_with_state_map(
                 room_version, state_set_ids, state_map,
@@ -420,7 +409,7 @@ def resolve_events(self, room_version, state_sets, event):
             key: state_map[ev_id] for key, ev_id in iteritems(new_state)
         }
 
-        defer.returnValue(new_state)
+        return new_state
 
 
 class StateResolutionHandler(object):

From 324525f40ca4df19c43971ca82db0d3478114885 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Mon, 20 Aug 2018 23:54:49 +1000
Subject: [PATCH 460/830] Port over enough to get some sytests running on
 Python 3 (#3668)

---
 changelog.d/3668.misc                    |  1 +
 synapse/api/auth.py                      | 14 +++---
 synapse/api/ratelimiting.py              |  2 +-
 synapse/config/logger.py                 |  3 +-
 synapse/http/servlet.py                  | 56 ++++++++++++++++++------
 synapse/http/site.py                     |  4 +-
 synapse/rest/client/transactions.py      |  2 +-
 synapse/rest/media/v1/upload_resource.py | 10 ++---
 synapse/secrets.py                       |  2 +-
 synapse/util/logutils.py                 | 10 ++++-
 synapse/util/stringutils.py              | 15 +++++++
 synapse/util/versionstring.py            | 12 ++---
 12 files changed, 91 insertions(+), 40 deletions(-)
 create mode 100644 changelog.d/3668.misc

diff --git a/changelog.d/3668.misc b/changelog.d/3668.misc
new file mode 100644
index 000000000000..f9265addf1a0
--- /dev/null
+++ b/changelog.d/3668.misc
@@ -0,0 +1 @@
+Port over enough to Python 3 to allow the sytests to start.
diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 022211e34e0f..6502a6be7b9b 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -211,7 +211,7 @@ def get_user_by_req(self, request, allow_guest=False, rights="access"):
             user_agent = request.requestHeaders.getRawHeaders(
                 b"User-Agent",
                 default=[b""]
-            )[0]
+            )[0].decode('ascii', 'surrogateescape')
             if user and access_token and ip_addr:
                 yield self.store.insert_client_ip(
                     user_id=user.to_string(),
@@ -682,7 +682,7 @@ def has_access_token(request):
         Returns:
             bool: False if no access_token was given, True otherwise.
         """
-        query_params = request.args.get("access_token")
+        query_params = request.args.get(b"access_token")
         auth_headers = request.requestHeaders.getRawHeaders(b"Authorization")
         return bool(query_params) or bool(auth_headers)
 
@@ -698,7 +698,7 @@ def get_access_token_from_request(request, token_not_found_http_status=401):
                 401 since some of the old clients depended on auth errors returning
                 403.
         Returns:
-            str: The access_token
+            unicode: The access_token
         Raises:
             AuthError: If there isn't an access_token in the request.
         """
@@ -720,9 +720,9 @@ def get_access_token_from_request(request, token_not_found_http_status=401):
                     "Too many Authorization headers.",
                     errcode=Codes.MISSING_TOKEN,
                 )
-            parts = auth_headers[0].split(" ")
-            if parts[0] == "Bearer" and len(parts) == 2:
-                return parts[1]
+            parts = auth_headers[0].split(b" ")
+            if parts[0] == b"Bearer" and len(parts) == 2:
+                return parts[1].decode('ascii')
             else:
                 raise AuthError(
                     token_not_found_http_status,
@@ -738,7 +738,7 @@ def get_access_token_from_request(request, token_not_found_http_status=401):
                     errcode=Codes.MISSING_TOKEN
                 )
 
-            return query_params[0]
+            return query_params[0].decode('ascii')
 
     @defer.inlineCallbacks
     def check_in_room_or_world_readable(self, room_id, user_id):
diff --git a/synapse/api/ratelimiting.py b/synapse/api/ratelimiting.py
index 06cc8d90b87e..3bb5b3da37c5 100644
--- a/synapse/api/ratelimiting.py
+++ b/synapse/api/ratelimiting.py
@@ -72,7 +72,7 @@ def send_message(self, user_id, time_now_s, msg_rate_hz, burst_count, update=Tru
         return allowed, time_allowed
 
     def prune_message_counts(self, time_now_s):
-        for user_id in self.message_counts.keys():
+        for user_id in list(self.message_counts.keys()):
             message_count, time_start, msg_rate_hz = (
                 self.message_counts[user_id]
             )
diff --git a/synapse/config/logger.py b/synapse/config/logger.py
index cfc20dcccf31..3f187adfc8cd 100644
--- a/synapse/config/logger.py
+++ b/synapse/config/logger.py
@@ -168,7 +168,8 @@ def setup_logging(config, use_worker_options=False):
         if log_file:
             # TODO: Customisable file size / backup count
             handler = logging.handlers.RotatingFileHandler(
-                log_file, maxBytes=(1000 * 1000 * 100), backupCount=3
+                log_file, maxBytes=(1000 * 1000 * 100), backupCount=3,
+                encoding='utf8'
             )
 
             def sighup(signum, stack):
diff --git a/synapse/http/servlet.py b/synapse/http/servlet.py
index 69f70852911c..a1e4b88e6ddc 100644
--- a/synapse/http/servlet.py
+++ b/synapse/http/servlet.py
@@ -29,7 +29,7 @@ def parse_integer(request, name, default=None, required=False):
 
     Args:
         request: the twisted HTTP request.
-        name (str): the name of the query parameter.
+        name (bytes/unicode): the name of the query parameter.
         default (int|None): value to use if the parameter is absent, defaults
             to None.
         required (bool): whether to raise a 400 SynapseError if the
@@ -46,6 +46,10 @@ def parse_integer(request, name, default=None, required=False):
 
 
 def parse_integer_from_args(args, name, default=None, required=False):
+
+    if not isinstance(name, bytes):
+        name = name.encode('ascii')
+
     if name in args:
         try:
             return int(args[name][0])
@@ -65,7 +69,7 @@ def parse_boolean(request, name, default=None, required=False):
 
     Args:
         request: the twisted HTTP request.
-        name (str): the name of the query parameter.
+        name (bytes/unicode): the name of the query parameter.
         default (bool|None): value to use if the parameter is absent, defaults
             to None.
         required (bool): whether to raise a 400 SynapseError if the
@@ -83,11 +87,15 @@ def parse_boolean(request, name, default=None, required=False):
 
 
 def parse_boolean_from_args(args, name, default=None, required=False):
+
+    if not isinstance(name, bytes):
+        name = name.encode('ascii')
+
     if name in args:
         try:
             return {
-                "true": True,
-                "false": False,
+                b"true": True,
+                b"false": False,
             }[args[name][0]]
         except Exception:
             message = (
@@ -104,21 +112,29 @@ def parse_boolean_from_args(args, name, default=None, required=False):
 
 
 def parse_string(request, name, default=None, required=False,
-                 allowed_values=None, param_type="string"):
-    """Parse a string parameter from the request query string.
+                 allowed_values=None, param_type="string", encoding='ascii'):
+    """
+    Parse a string parameter from the request query string.
+
+    If encoding is not None, the content of the query param will be
+    decoded to Unicode using the encoding, otherwise it will be encoded
 
     Args:
         request: the twisted HTTP request.
-        name (str): the name of the query parameter.
-        default (str|None): value to use if the parameter is absent, defaults
-            to None.
+        name (bytes/unicode): the name of the query parameter.
+        default (bytes/unicode|None): value to use if the parameter is absent,
+            defaults to None. Must be bytes if encoding is None.
         required (bool): whether to raise a 400 SynapseError if the
             parameter is absent, defaults to False.
-        allowed_values (list[str]): List of allowed values for the string,
-            or None if any value is allowed, defaults to None
+        allowed_values (list[bytes/unicode]): List of allowed values for the
+            string, or None if any value is allowed, defaults to None. Must be
+            the same type as name, if given.
+        encoding: The encoding to decode the name to, and decode the string
+            content with.
 
     Returns:
-        str|None: A string value or the default.
+        bytes/unicode|None: A string value or the default. Unicode if encoding
+        was given, bytes otherwise.
 
     Raises:
         SynapseError if the parameter is absent and required, or if the
@@ -126,14 +142,22 @@ def parse_string(request, name, default=None, required=False,
             is not one of those allowed values.
     """
     return parse_string_from_args(
-        request.args, name, default, required, allowed_values, param_type,
+        request.args, name, default, required, allowed_values, param_type, encoding
     )
 
 
 def parse_string_from_args(args, name, default=None, required=False,
-                           allowed_values=None, param_type="string"):
+                           allowed_values=None, param_type="string", encoding='ascii'):
+
+    if not isinstance(name, bytes):
+        name = name.encode('ascii')
+
     if name in args:
         value = args[name][0]
+
+        if encoding:
+            value = value.decode(encoding)
+
         if allowed_values is not None and value not in allowed_values:
             message = "Query parameter %r must be one of [%s]" % (
                 name, ", ".join(repr(v) for v in allowed_values)
@@ -146,6 +170,10 @@ def parse_string_from_args(args, name, default=None, required=False,
             message = "Missing %s query parameter %r" % (param_type, name)
             raise SynapseError(400, message, errcode=Codes.MISSING_PARAM)
         else:
+
+            if encoding and isinstance(default, bytes):
+                return default.decode(encoding)
+
             return default
 
 
diff --git a/synapse/http/site.py b/synapse/http/site.py
index f5a8f7840654..ad2a98468e62 100644
--- a/synapse/http/site.py
+++ b/synapse/http/site.py
@@ -235,7 +235,7 @@ def _finished_processing(self):
         # need to decode as it could be raw utf-8 bytes
         # from a IDN servname in an auth header
         authenticated_entity = self.authenticated_entity
-        if authenticated_entity is not None:
+        if authenticated_entity is not None and isinstance(authenticated_entity, bytes):
             authenticated_entity = authenticated_entity.decode("utf-8", "replace")
 
         # ...or could be raw utf-8 bytes in the User-Agent header.
@@ -328,7 +328,7 @@ def __init__(self, logger_name, site_tag, config, resource,
         proxied = config.get("x_forwarded", False)
         self.requestFactory = SynapseRequestFactory(self, proxied)
         self.access_logger = logging.getLogger(logger_name)
-        self.server_version_string = server_version_string
+        self.server_version_string = server_version_string.encode('ascii')
 
     def log(self, request):
         pass
diff --git a/synapse/rest/client/transactions.py b/synapse/rest/client/transactions.py
index 511e96ab00d9..48c17f1b6d41 100644
--- a/synapse/rest/client/transactions.py
+++ b/synapse/rest/client/transactions.py
@@ -53,7 +53,7 @@ def _get_transaction_key(self, request):
             str: A transaction key
         """
         token = self.auth.get_access_token_from_request(request)
-        return request.path + "/" + token
+        return request.path.decode('utf8') + "/" + token
 
     def fetch_or_execute_request(self, request, fn, *args, **kwargs):
         """A helper function for fetch_or_execute which extracts
diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py
index 9b22d204a6d1..c1240e1963dd 100644
--- a/synapse/rest/media/v1/upload_resource.py
+++ b/synapse/rest/media/v1/upload_resource.py
@@ -55,7 +55,7 @@ def _async_render_POST(self, request):
         requester = yield self.auth.get_user_by_req(request)
         # TODO: The checks here are a bit late. The content will have
         # already been uploaded to a tmp file at this point
-        content_length = request.getHeader("Content-Length")
+        content_length = request.getHeader(b"Content-Length").decode('ascii')
         if content_length is None:
             raise SynapseError(
                 msg="Request must specify a Content-Length", code=400
@@ -66,10 +66,10 @@ def _async_render_POST(self, request):
                 code=413,
             )
 
-        upload_name = parse_string(request, "filename")
+        upload_name = parse_string(request, b"filename", encoding=None)
         if upload_name:
             try:
-                upload_name = upload_name.decode('UTF-8')
+                upload_name = upload_name.decode('utf8')
             except UnicodeDecodeError:
                 raise SynapseError(
                     msg="Invalid UTF-8 filename parameter: %r" % (upload_name),
@@ -78,8 +78,8 @@ def _async_render_POST(self, request):
 
         headers = request.requestHeaders
 
-        if headers.hasHeader("Content-Type"):
-            media_type = headers.getRawHeaders(b"Content-Type")[0]
+        if headers.hasHeader(b"Content-Type"):
+            media_type = headers.getRawHeaders(b"Content-Type")[0].decode('ascii')
         else:
             raise SynapseError(
                 msg="Upload request missing 'Content-Type'",
diff --git a/synapse/secrets.py b/synapse/secrets.py
index f05e9ea53595..f6280f951cdf 100644
--- a/synapse/secrets.py
+++ b/synapse/secrets.py
@@ -38,4 +38,4 @@ def token_bytes(self, nbytes=32):
             return os.urandom(nbytes)
 
         def token_hex(self, nbytes=32):
-            return binascii.hexlify(self.token_bytes(nbytes))
+            return binascii.hexlify(self.token_bytes(nbytes)).decode('ascii')
diff --git a/synapse/util/logutils.py b/synapse/util/logutils.py
index 62a00189ccd1..ef31458226b3 100644
--- a/synapse/util/logutils.py
+++ b/synapse/util/logutils.py
@@ -20,6 +20,8 @@
 from functools import wraps
 from inspect import getcallargs
 
+from six import PY3
+
 _TIME_FUNC_ID = 0
 
 
@@ -28,8 +30,12 @@ def _log_debug_as_f(f, msg, msg_args):
     logger = logging.getLogger(name)
 
     if logger.isEnabledFor(logging.DEBUG):
-        lineno = f.func_code.co_firstlineno
-        pathname = f.func_code.co_filename
+        if PY3:
+            lineno = f.__code__.co_firstlineno
+            pathname = f.__code__.co_filename
+        else:
+            lineno = f.func_code.co_firstlineno
+            pathname = f.func_code.co_filename
 
         record = logging.LogRecord(
             name=name,
diff --git a/synapse/util/stringutils.py b/synapse/util/stringutils.py
index 43d9db67ecdd..6f318c6a299a 100644
--- a/synapse/util/stringutils.py
+++ b/synapse/util/stringutils.py
@@ -16,6 +16,7 @@
 import random
 import string
 
+from six import PY3
 from six.moves import range
 
 _string_with_symbols = (
@@ -34,6 +35,17 @@ def random_string_with_symbols(length):
 
 
 def is_ascii(s):
+
+    if PY3:
+        if isinstance(s, bytes):
+            try:
+                s.decode('ascii').encode('ascii')
+            except UnicodeDecodeError:
+                return False
+            except UnicodeEncodeError:
+                return False
+            return True
+
     try:
         s.encode("ascii")
     except UnicodeEncodeError:
@@ -49,6 +61,9 @@ def to_ascii(s):
 
     If given None then will return None.
     """
+    if PY3:
+        return s
+
     if s is None:
         return None
 
diff --git a/synapse/util/versionstring.py b/synapse/util/versionstring.py
index 1fbcd411153e..3baba3225afa 100644
--- a/synapse/util/versionstring.py
+++ b/synapse/util/versionstring.py
@@ -30,7 +30,7 @@ def get_version_string(module):
                 ['git', 'rev-parse', '--abbrev-ref', 'HEAD'],
                 stderr=null,
                 cwd=cwd,
-            ).strip()
+            ).strip().decode('ascii')
             git_branch = "b=" + git_branch
         except subprocess.CalledProcessError:
             git_branch = ""
@@ -40,7 +40,7 @@ def get_version_string(module):
                 ['git', 'describe', '--exact-match'],
                 stderr=null,
                 cwd=cwd,
-            ).strip()
+            ).strip().decode('ascii')
             git_tag = "t=" + git_tag
         except subprocess.CalledProcessError:
             git_tag = ""
@@ -50,7 +50,7 @@ def get_version_string(module):
                 ['git', 'rev-parse', '--short', 'HEAD'],
                 stderr=null,
                 cwd=cwd,
-            ).strip()
+            ).strip().decode('ascii')
         except subprocess.CalledProcessError:
             git_commit = ""
 
@@ -60,7 +60,7 @@ def get_version_string(module):
                 ['git', 'describe', '--dirty=' + dirty_string],
                 stderr=null,
                 cwd=cwd,
-            ).strip().endswith(dirty_string)
+            ).strip().decode('ascii').endswith(dirty_string)
 
             git_dirty = "dirty" if is_dirty else ""
         except subprocess.CalledProcessError:
@@ -77,8 +77,8 @@ def get_version_string(module):
                 "%s (%s)" % (
                     module.__version__, git_version,
                 )
-            ).encode("ascii")
+            )
     except Exception as e:
         logger.info("Failed to check for git repository: %s", e)
 
-    return module.__version__.encode("ascii")
+    return module.__version__

From 9a2f960736a1d1aeb6c9c094887253b280a59a37 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Tue, 21 Aug 2018 00:00:19 +1000
Subject: [PATCH 461/830] version

---
 synapse/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/__init__.py b/synapse/__init__.py
index a14d578e36a7..5175a1161d5a 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -17,4 +17,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.33.2"
+__version__ = "0.33.3rc1"

From 80bf7d35802003cee6e495421ec54c1c08bce525 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Tue, 21 Aug 2018 00:01:14 +1000
Subject: [PATCH 462/830] changelog

---
 CHANGES.md               | 64 ++++++++++++++++++++++++++++++++++++++++
 changelog.d/1491.feature |  1 -
 changelog.d/3184.feature |  1 -
 changelog.d/3423.misc    |  1 -
 changelog.d/3568.feature |  1 -
 changelog.d/3574.feature |  1 -
 changelog.d/3589.feature |  1 -
 changelog.d/3632.misc    |  1 -
 changelog.d/3633.feature |  1 -
 changelog.d/3647.misc    |  1 -
 changelog.d/3653.feature |  1 -
 changelog.d/3654.feature |  1 -
 changelog.d/3655.feature |  1 -
 changelog.d/3658.bugfix  |  1 -
 changelog.d/3660.misc    |  1 -
 changelog.d/3661.bugfix  |  1 -
 changelog.d/3662.feature |  1 -
 changelog.d/3664.feature |  1 -
 changelog.d/3668.misc    |  1 -
 changelog.d/3669.misc    |  1 -
 changelog.d/3670.feature |  1 -
 changelog.d/3676.bugfix  |  1 -
 changelog.d/3677.bugfix  |  1 -
 changelog.d/3678.misc    |  1 -
 changelog.d/3679.misc    |  1 -
 changelog.d/3681.bugfix  |  1 -
 changelog.d/3684.misc    |  1 -
 changelog.d/3687.feature |  1 -
 changelog.d/3689.bugfix  |  1 -
 changelog.d/3690.misc    |  1 -
 changelog.d/3692.bugfix  |  1 -
 changelog.d/3694.feature |  1 -
 changelog.d/3700.bugfix  |  1 -
 changelog.d/3701.bugfix  |  1 -
 changelog.d/3703.removal |  1 -
 changelog.d/3705.bugfix  |  1 -
 changelog.d/3707.misc    |  1 -
 changelog.d/3708.feature |  1 -
 changelog.d/3709.misc    |  1 -
 changelog.d/3710.bugfix  |  1 -
 changelog.d/3712.misc    |  1 -
 changelog.d/3713.bugfix  |  1 -
 changelog.d/3719.bugfix  |  1 -
 43 files changed, 64 insertions(+), 42 deletions(-)
 delete mode 100644 changelog.d/1491.feature
 delete mode 100644 changelog.d/3184.feature
 delete mode 100644 changelog.d/3423.misc
 delete mode 100644 changelog.d/3568.feature
 delete mode 100644 changelog.d/3574.feature
 delete mode 100644 changelog.d/3589.feature
 delete mode 100644 changelog.d/3632.misc
 delete mode 100644 changelog.d/3633.feature
 delete mode 100644 changelog.d/3647.misc
 delete mode 100644 changelog.d/3653.feature
 delete mode 100644 changelog.d/3654.feature
 delete mode 100644 changelog.d/3655.feature
 delete mode 100644 changelog.d/3658.bugfix
 delete mode 100644 changelog.d/3660.misc
 delete mode 100644 changelog.d/3661.bugfix
 delete mode 100644 changelog.d/3662.feature
 delete mode 100644 changelog.d/3664.feature
 delete mode 100644 changelog.d/3668.misc
 delete mode 100644 changelog.d/3669.misc
 delete mode 100644 changelog.d/3670.feature
 delete mode 100644 changelog.d/3676.bugfix
 delete mode 100644 changelog.d/3677.bugfix
 delete mode 100644 changelog.d/3678.misc
 delete mode 100644 changelog.d/3679.misc
 delete mode 100644 changelog.d/3681.bugfix
 delete mode 100644 changelog.d/3684.misc
 delete mode 100644 changelog.d/3687.feature
 delete mode 100644 changelog.d/3689.bugfix
 delete mode 100644 changelog.d/3690.misc
 delete mode 100644 changelog.d/3692.bugfix
 delete mode 100644 changelog.d/3694.feature
 delete mode 100644 changelog.d/3700.bugfix
 delete mode 100644 changelog.d/3701.bugfix
 delete mode 100644 changelog.d/3703.removal
 delete mode 100644 changelog.d/3705.bugfix
 delete mode 100644 changelog.d/3707.misc
 delete mode 100644 changelog.d/3708.feature
 delete mode 100644 changelog.d/3709.misc
 delete mode 100644 changelog.d/3710.bugfix
 delete mode 100644 changelog.d/3712.misc
 delete mode 100644 changelog.d/3713.bugfix
 delete mode 100644 changelog.d/3719.bugfix

diff --git a/CHANGES.md b/CHANGES.md
index a299110a6b1b..68ce5d14718c 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,67 @@
+Synapse 0.33.3rc1 (2018-08-21)
+==============================
+
+Features
+--------
+
+- Add support for the SNI extension to federation TLS connections ([\#1491](https://github.com/matrix-org/synapse/issues/1491))
+- Add /_media/r0/config ([\#3184](https://github.com/matrix-org/synapse/issues/3184))
+- speed up /members API and add `at` and `membership` params as per MSC1227 ([\#3568](https://github.com/matrix-org/synapse/issues/3568))
+- implement `summary` block in /sync response as per MSC688 ([\#3574](https://github.com/matrix-org/synapse/issues/3574))
+- Add lazy-loading support to /messages as per MSC1227 ([\#3589](https://github.com/matrix-org/synapse/issues/3589))
+- Add ability to limit number of monthly active users on the server ([\#3633](https://github.com/matrix-org/synapse/issues/3633))
+- Support more federation endpoints on workers ([\#3653](https://github.com/matrix-org/synapse/issues/3653))
+- Basic support for room versioning ([\#3654](https://github.com/matrix-org/synapse/issues/3654))
+- Ability to disable client/server Synapse via conf toggle ([\#3655](https://github.com/matrix-org/synapse/issues/3655))
+- Ability to whitelist specific threepids against monthly active user limiting ([\#3662](https://github.com/matrix-org/synapse/issues/3662))
+- Add some metrics for the appservice and federation event sending loops ([\#3664](https://github.com/matrix-org/synapse/issues/3664))
+- Where server is disabled, block ability for locked out users to read new messages ([\#3670](https://github.com/matrix-org/synapse/issues/3670))
+- set admin uri via config, to be used in error messages where the user should contact the administrator ([\#3687](https://github.com/matrix-org/synapse/issues/3687))
+- Synapse's presence functionality can now be disabled with the "use_presence" configuration option. ([\#3694](https://github.com/matrix-org/synapse/issues/3694))
+- For resource limit blocked users, prevent writing into rooms ([\#3708](https://github.com/matrix-org/synapse/issues/3708))
+
+
+Bugfixes
+--------
+
+- Fix occasional glitches in the synapse_event_persisted_position metric ([\#3658](https://github.com/matrix-org/synapse/issues/3658))
+- Fix bug on deleting 3pid when using identity servers that don't support unbind API ([\#3661](https://github.com/matrix-org/synapse/issues/3661))
+- Make the tests pass on Twisted < 18.7.0 ([\#3676](https://github.com/matrix-org/synapse/issues/3676))
+- Don’t ship recaptcha_ajax.js, use it directly from Google ([\#3677](https://github.com/matrix-org/synapse/issues/3677))
+- Fixes test_reap_monthly_active_users so it passes under postgres ([\#3681](https://github.com/matrix-org/synapse/issues/3681))
+- Fix mau blocking calulation bug on login ([\#3689](https://github.com/matrix-org/synapse/issues/3689))
+- Fix missing yield in synapse.storage.monthly_active_users.initialise_reserved_users ([\#3692](https://github.com/matrix-org/synapse/issues/3692))
+- Improve HTTP request logging to include all requests ([\#3700](https://github.com/matrix-org/synapse/issues/3700))
+- Avoid timing out requests while we are streaming back the response ([\#3701](https://github.com/matrix-org/synapse/issues/3701))
+- Support more federation endpoints on workers ([\#3705](https://github.com/matrix-org/synapse/issues/3705), [\#3713](https://github.com/matrix-org/synapse/issues/3713))
+- Fix "Starting db txn 'get_all_updated_receipts' from sentinel context" warning ([\#3710](https://github.com/matrix-org/synapse/issues/3710))
+- Fix bug where `state_cache` cache factor ignored environment variables ([\#3719](https://github.com/matrix-org/synapse/issues/3719))
+
+
+Deprecations and Removals
+-------------------------
+
+- The Shared-Secret registration method of the legacy v1/register REST endpoint has been removed. For a replacement, please see [the admin/register API documentation](https://github.com/matrix-org/synapse/blob/master/docs/admin_api/register_api.rst). ([\#3703](https://github.com/matrix-org/synapse/issues/3703))
+
+
+Internal Changes
+----------------
+
+- The test suite now can run under PostgreSQL. ([\#3423](https://github.com/matrix-org/synapse/issues/3423))
+- Refactor HTTP replication endpoints to reduce code duplication ([\#3632](https://github.com/matrix-org/synapse/issues/3632))
+- Tests now correctly execute on Python 3. ([\#3647](https://github.com/matrix-org/synapse/issues/3647))
+- Sytests can now be run inside a Docker container. ([\#3660](https://github.com/matrix-org/synapse/issues/3660))
+- Port over enough to Python 3 to allow the sytests to start. ([\#3668](https://github.com/matrix-org/synapse/issues/3668))
+- Update docker base image from alpine 3.7 to 3.8. ([\#3669](https://github.com/matrix-org/synapse/issues/3669))
+- Rename synapse.util.async to synapse.util.async_helpers to mitigate async becoming a keyword on Python 3.7. ([\#3678](https://github.com/matrix-org/synapse/issues/3678))
+- Synapse's tests are now formatted with the black autoformatter. ([\#3679](https://github.com/matrix-org/synapse/issues/3679))
+- Implemented a new testing base class to reduce test boilerplate. ([\#3684](https://github.com/matrix-org/synapse/issues/3684))
+- Rename MAU prometheus metrics ([\#3690](https://github.com/matrix-org/synapse/issues/3690))
+- add new error type ResourceLimit ([\#3707](https://github.com/matrix-org/synapse/issues/3707))
+- Logcontexts for replication command handlers ([\#3709](https://github.com/matrix-org/synapse/issues/3709))
+- Update admin register API documentation to reference a real user ID. ([\#3712](https://github.com/matrix-org/synapse/issues/3712))
+
+
 Synapse 0.33.2 (2018-08-09)
 ===========================
 
diff --git a/changelog.d/1491.feature b/changelog.d/1491.feature
deleted file mode 100644
index 77b6d6ca09ea..000000000000
--- a/changelog.d/1491.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add support for the SNI extension to federation TLS connections
\ No newline at end of file
diff --git a/changelog.d/3184.feature b/changelog.d/3184.feature
deleted file mode 100644
index 9f746a57a0fa..000000000000
--- a/changelog.d/3184.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add /_media/r0/config
diff --git a/changelog.d/3423.misc b/changelog.d/3423.misc
deleted file mode 100644
index 51768c6d14ef..000000000000
--- a/changelog.d/3423.misc
+++ /dev/null
@@ -1 +0,0 @@
-The test suite now can run under PostgreSQL.
diff --git a/changelog.d/3568.feature b/changelog.d/3568.feature
deleted file mode 100644
index 247f02ba4e13..000000000000
--- a/changelog.d/3568.feature
+++ /dev/null
@@ -1 +0,0 @@
-speed up /members API and add `at` and `membership` params as per MSC1227
diff --git a/changelog.d/3574.feature b/changelog.d/3574.feature
deleted file mode 100644
index 87ac32df7249..000000000000
--- a/changelog.d/3574.feature
+++ /dev/null
@@ -1 +0,0 @@
-implement `summary` block in /sync response as per MSC688
diff --git a/changelog.d/3589.feature b/changelog.d/3589.feature
deleted file mode 100644
index a8d7124719e0..000000000000
--- a/changelog.d/3589.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add lazy-loading support to /messages as per MSC1227
diff --git a/changelog.d/3632.misc b/changelog.d/3632.misc
deleted file mode 100644
index 9d64bbe83b65..000000000000
--- a/changelog.d/3632.misc
+++ /dev/null
@@ -1 +0,0 @@
-Refactor HTTP replication endpoints to reduce code duplication
diff --git a/changelog.d/3633.feature b/changelog.d/3633.feature
deleted file mode 100644
index 8007a048405b..000000000000
--- a/changelog.d/3633.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add ability to limit number of monthly active users on the server
diff --git a/changelog.d/3647.misc b/changelog.d/3647.misc
deleted file mode 100644
index dbc66dae6087..000000000000
--- a/changelog.d/3647.misc
+++ /dev/null
@@ -1 +0,0 @@
-Tests now correctly execute on Python 3.
diff --git a/changelog.d/3653.feature b/changelog.d/3653.feature
deleted file mode 100644
index 6c5422994f34..000000000000
--- a/changelog.d/3653.feature
+++ /dev/null
@@ -1 +0,0 @@
-Support more federation endpoints on workers
diff --git a/changelog.d/3654.feature b/changelog.d/3654.feature
deleted file mode 100644
index 35c95580bc45..000000000000
--- a/changelog.d/3654.feature
+++ /dev/null
@@ -1 +0,0 @@
-Basic support for room versioning
diff --git a/changelog.d/3655.feature b/changelog.d/3655.feature
deleted file mode 100644
index 1134e549e7b3..000000000000
--- a/changelog.d/3655.feature
+++ /dev/null
@@ -1 +0,0 @@
-Ability to disable client/server Synapse via conf toggle
diff --git a/changelog.d/3658.bugfix b/changelog.d/3658.bugfix
deleted file mode 100644
index 556011a15060..000000000000
--- a/changelog.d/3658.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix occasional glitches in the synapse_event_persisted_position metric
diff --git a/changelog.d/3660.misc b/changelog.d/3660.misc
deleted file mode 100644
index acd814c27391..000000000000
--- a/changelog.d/3660.misc
+++ /dev/null
@@ -1 +0,0 @@
-Sytests can now be run inside a Docker container.
diff --git a/changelog.d/3661.bugfix b/changelog.d/3661.bugfix
deleted file mode 100644
index f2b4703d806b..000000000000
--- a/changelog.d/3661.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix bug on deleting 3pid when using identity servers that don't support unbind API
diff --git a/changelog.d/3662.feature b/changelog.d/3662.feature
deleted file mode 100644
index daacef086d03..000000000000
--- a/changelog.d/3662.feature
+++ /dev/null
@@ -1 +0,0 @@
-Ability to whitelist specific threepids against monthly active user limiting
diff --git a/changelog.d/3664.feature b/changelog.d/3664.feature
deleted file mode 100644
index 184dde99393d..000000000000
--- a/changelog.d/3664.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add some metrics for the appservice and federation event sending loops
diff --git a/changelog.d/3668.misc b/changelog.d/3668.misc
deleted file mode 100644
index f9265addf1a0..000000000000
--- a/changelog.d/3668.misc
+++ /dev/null
@@ -1 +0,0 @@
-Port over enough to Python 3 to allow the sytests to start.
diff --git a/changelog.d/3669.misc b/changelog.d/3669.misc
deleted file mode 100644
index fc579ddc607d..000000000000
--- a/changelog.d/3669.misc
+++ /dev/null
@@ -1 +0,0 @@
-Update docker base image from alpine 3.7 to 3.8.
diff --git a/changelog.d/3670.feature b/changelog.d/3670.feature
deleted file mode 100644
index ba00f2d2ecb8..000000000000
--- a/changelog.d/3670.feature
+++ /dev/null
@@ -1 +0,0 @@
-Where server is disabled, block ability for locked out users to read new messages
diff --git a/changelog.d/3676.bugfix b/changelog.d/3676.bugfix
deleted file mode 100644
index 7b23a2773a55..000000000000
--- a/changelog.d/3676.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Make the tests pass on Twisted < 18.7.0
diff --git a/changelog.d/3677.bugfix b/changelog.d/3677.bugfix
deleted file mode 100644
index caa551627beb..000000000000
--- a/changelog.d/3677.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Don’t ship recaptcha_ajax.js, use it directly from Google
diff --git a/changelog.d/3678.misc b/changelog.d/3678.misc
deleted file mode 100644
index 0d7c8da64aa7..000000000000
--- a/changelog.d/3678.misc
+++ /dev/null
@@ -1 +0,0 @@
-Rename synapse.util.async to synapse.util.async_helpers to mitigate async becoming a keyword on Python 3.7.
diff --git a/changelog.d/3679.misc b/changelog.d/3679.misc
deleted file mode 100644
index 1de0a0f2b41c..000000000000
--- a/changelog.d/3679.misc
+++ /dev/null
@@ -1 +0,0 @@
-Synapse's tests are now formatted with the black autoformatter.
diff --git a/changelog.d/3681.bugfix b/changelog.d/3681.bugfix
deleted file mode 100644
index d18a69cd0c58..000000000000
--- a/changelog.d/3681.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fixes test_reap_monthly_active_users so it passes under postgres
diff --git a/changelog.d/3684.misc b/changelog.d/3684.misc
deleted file mode 100644
index 4c013263c453..000000000000
--- a/changelog.d/3684.misc
+++ /dev/null
@@ -1 +0,0 @@
-Implemented a new testing base class to reduce test boilerplate.
diff --git a/changelog.d/3687.feature b/changelog.d/3687.feature
deleted file mode 100644
index 64b89f6411ba..000000000000
--- a/changelog.d/3687.feature
+++ /dev/null
@@ -1 +0,0 @@
-set admin uri via config, to be used in error messages where the user should contact the administrator
diff --git a/changelog.d/3689.bugfix b/changelog.d/3689.bugfix
deleted file mode 100644
index 934d039836ad..000000000000
--- a/changelog.d/3689.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix mau blocking calulation bug on login
diff --git a/changelog.d/3690.misc b/changelog.d/3690.misc
deleted file mode 100644
index 710add02433d..000000000000
--- a/changelog.d/3690.misc
+++ /dev/null
@@ -1 +0,0 @@
-Rename MAU prometheus metrics
diff --git a/changelog.d/3692.bugfix b/changelog.d/3692.bugfix
deleted file mode 100644
index f44e13dca154..000000000000
--- a/changelog.d/3692.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix missing yield in synapse.storage.monthly_active_users.initialise_reserved_users
diff --git a/changelog.d/3694.feature b/changelog.d/3694.feature
deleted file mode 100644
index 916a342ff48f..000000000000
--- a/changelog.d/3694.feature
+++ /dev/null
@@ -1 +0,0 @@
-Synapse's presence functionality can now be disabled with the "use_presence" configuration option.
diff --git a/changelog.d/3700.bugfix b/changelog.d/3700.bugfix
deleted file mode 100644
index 492cce1dfc33..000000000000
--- a/changelog.d/3700.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Improve HTTP request logging to include all requests
\ No newline at end of file
diff --git a/changelog.d/3701.bugfix b/changelog.d/3701.bugfix
deleted file mode 100644
index c22de3453747..000000000000
--- a/changelog.d/3701.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Avoid timing out requests while we are streaming back the response
diff --git a/changelog.d/3703.removal b/changelog.d/3703.removal
deleted file mode 100644
index d13ce5adb408..000000000000
--- a/changelog.d/3703.removal
+++ /dev/null
@@ -1 +0,0 @@
-The Shared-Secret registration method of the legacy v1/register REST endpoint has been removed. For a replacement, please see [the admin/register API documentation](https://github.com/matrix-org/synapse/blob/master/docs/admin_api/register_api.rst).
diff --git a/changelog.d/3705.bugfix b/changelog.d/3705.bugfix
deleted file mode 100644
index 6c5422994f34..000000000000
--- a/changelog.d/3705.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Support more federation endpoints on workers
diff --git a/changelog.d/3707.misc b/changelog.d/3707.misc
deleted file mode 100644
index 8123ca65430b..000000000000
--- a/changelog.d/3707.misc
+++ /dev/null
@@ -1 +0,0 @@
-add new error type ResourceLimit
diff --git a/changelog.d/3708.feature b/changelog.d/3708.feature
deleted file mode 100644
index 2f146ba62b34..000000000000
--- a/changelog.d/3708.feature
+++ /dev/null
@@ -1 +0,0 @@
-For resource limit blocked users, prevent writing into rooms
diff --git a/changelog.d/3709.misc b/changelog.d/3709.misc
deleted file mode 100644
index bbda357d4481..000000000000
--- a/changelog.d/3709.misc
+++ /dev/null
@@ -1 +0,0 @@
-Logcontexts for replication command handlers
diff --git a/changelog.d/3710.bugfix b/changelog.d/3710.bugfix
deleted file mode 100644
index c28e17766770..000000000000
--- a/changelog.d/3710.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix "Starting db txn 'get_all_updated_receipts' from sentinel context" warning
\ No newline at end of file
diff --git a/changelog.d/3712.misc b/changelog.d/3712.misc
deleted file mode 100644
index 30f8c2af21c0..000000000000
--- a/changelog.d/3712.misc
+++ /dev/null
@@ -1 +0,0 @@
-Update admin register API documentation to reference a real user ID.
diff --git a/changelog.d/3713.bugfix b/changelog.d/3713.bugfix
deleted file mode 100644
index 6c5422994f34..000000000000
--- a/changelog.d/3713.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Support more federation endpoints on workers
diff --git a/changelog.d/3719.bugfix b/changelog.d/3719.bugfix
deleted file mode 100644
index 095d9288e897..000000000000
--- a/changelog.d/3719.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix bug where `state_cache` cache factor ignored environment variables

From 1058d141272f17435b3548a4b12abb51a6372383 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 20 Aug 2018 17:16:58 +0100
Subject: [PATCH 463/830] Make the in flight background process metrics thread
 safe

---
 synapse/metrics/background_process_metrics.py | 25 +++++++++++++++----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/synapse/metrics/background_process_metrics.py b/synapse/metrics/background_process_metrics.py
index ce678d5f75d9..264fb93892d0 100644
--- a/synapse/metrics/background_process_metrics.py
+++ b/synapse/metrics/background_process_metrics.py
@@ -14,6 +14,7 @@
 # limitations under the License.
 
 import six
+import threading
 
 from prometheus_client.core import REGISTRY, Counter, GaugeMetricFamily
 
@@ -78,6 +79,9 @@
 # of process descriptions that no longer have any active processes.
 _background_processes = dict()  # type: dict[str, set[_BackgroundProcess]]
 
+# A lock that covers the above dicts
+_bg_metrics_lock = threading.Lock()
+
 
 class _Collector(object):
     """A custom metrics collector for the background process metrics.
@@ -92,7 +96,11 @@ def collect(self):
             labels=["name"],
         )
 
-        for desc, processes in six.iteritems(_background_processes):
+        # We copy the dict so that it doesn't change from underneath us
+        with _bg_metrics_lock:
+            _background_processes_copy = dict(_background_processes)
+
+        for desc, processes in six.iteritems(_background_processes_copy):
             background_process_in_flight_count.add_metric(
                 (desc,), len(processes),
             )
@@ -167,19 +175,26 @@ def run_as_background_process(desc, func, *args, **kwargs):
     """
     @defer.inlineCallbacks
     def run():
-        count = _background_process_counts.get(desc, 0)
-        _background_process_counts[desc] = count + 1
+        with _bg_metrics_lock:
+            count = _background_process_counts.get(desc, 0)
+            _background_process_counts[desc] = count + 1
+
         _background_process_start_count.labels(desc).inc()
 
         with LoggingContext(desc) as context:
             context.request = "%s-%i" % (desc, count)
             proc = _BackgroundProcess(desc, context)
-            _background_processes.setdefault(desc, set()).add(proc)
+
+            with _bg_metrics_lock:
+                _background_processes.setdefault(desc, set()).add(proc)
+
             try:
                 yield func(*args, **kwargs)
             finally:
                 proc.update_metrics()
-                _background_processes[desc].remove(proc)
+
+                with _bg_metrics_lock:
+                    _background_processes[desc].remove(proc)
 
     with PreserveLoggingContext():
         return run()

From b01a75549893bf4f8f986a74a499e7cb6131868c Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 20 Aug 2018 17:27:52 +0100
Subject: [PATCH 464/830] Make the in flight requests metrics thread safe

---
 synapse/http/request_metrics.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py
index 588e280571e0..284d5e05dee5 100644
--- a/synapse/http/request_metrics.py
+++ b/synapse/http/request_metrics.py
@@ -15,6 +15,7 @@
 # limitations under the License.
 
 import logging
+import threading
 
 from prometheus_client.core import Counter, Histogram
 
@@ -111,6 +112,9 @@
 # The set of all in flight requests, set[RequestMetrics]
 _in_flight_requests = set()
 
+# Protects the _in_flight_requests set from concurrent accesss
+_in_flight_reqeuests_lock = threading.Lock()
+
 
 def _get_in_flight_counts():
     """Returns a count of all in flight requests by (method, server_name)
@@ -120,7 +124,8 @@ def _get_in_flight_counts():
     """
     # Cast to a list to prevent it changing while the Prometheus
     # thread is collecting metrics
-    reqs = list(_in_flight_requests)
+    with _in_flight_reqeuests_lock:
+        reqs = list(_in_flight_requests)
 
     for rm in reqs:
         rm.update_metrics()
@@ -154,10 +159,12 @@ def start(self, time_sec, name, method):
         # to the "in flight" metrics.
         self._request_stats = self.start_context.get_resource_usage()
 
-        _in_flight_requests.add(self)
+        with _in_flight_reqeuests_lock:
+            _in_flight_requests.add(self)
 
     def stop(self, time_sec, request):
-        _in_flight_requests.discard(self)
+        with _in_flight_reqeuests_lock:
+            _in_flight_requests.discard(self)
 
         context = LoggingContext.current_context()
 

From e2c0aa2c2619d6db7f6d6b1413430133e365f4cd Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 20 Aug 2018 17:40:59 +0100
Subject: [PATCH 465/830] Newsfile

---
 changelog.d/3722.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3722.bugfix

diff --git a/changelog.d/3722.bugfix b/changelog.d/3722.bugfix
new file mode 100644
index 000000000000..16cbaf76cbf4
--- /dev/null
+++ b/changelog.d/3722.bugfix
@@ -0,0 +1 @@
+Fix error collecting prometheus metrics when run on dedicated thread due to threading concurrency issues

From 55e6bdf28798153de5c2a6cf9bf0a6618f59168a Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 20 Aug 2018 18:20:07 +0100
Subject: [PATCH 466/830] Robustness fix for logcontext filter

Make the logcontext filter not explode if it somehow ends up with a logcontext
of None, since that infinite-loops the whole logging system.
---
 synapse/util/logcontext.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py
index 07e83fadda0c..848765e53041 100644
--- a/synapse/util/logcontext.py
+++ b/synapse/util/logcontext.py
@@ -385,7 +385,13 @@ def filter(self, record):
         context = LoggingContext.current_context()
         for key, value in self.defaults.items():
             setattr(record, key, value)
-        context.copy_to(record)
+
+        # context should never be None, but if it somehow ends up being, then
+        # we end up in a death spiral of infinite loops, so let's check, for
+        # robustness' sake.
+        if context is not None:
+            context.copy_to(record)
+
         return True
 
 

From be6527325a829e7d5e4540d7a1983a6e6fda2c0f Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 20 Aug 2018 18:21:10 +0100
Subject: [PATCH 467/830] Fix exceptions when a connection is closed before we
 read the headers

This fixes bugs introduced in #3700, by making sure that we behave sanely
when an incoming connection is closed before the headers are read.
---
 synapse/http/site.py       | 8 +++++++-
 synapse/util/logcontext.py | 4 +++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/synapse/http/site.py b/synapse/http/site.py
index ad2a98468e62..88ed3714f9be 100644
--- a/synapse/http/site.py
+++ b/synapse/http/site.py
@@ -182,7 +182,7 @@ def connectionLost(self, reason):
         # the client disconnects.
         with PreserveLoggingContext(self.logcontext):
             logger.warn(
-                "Error processing request: %s %s", reason.type, reason.value,
+                "Error processing request %r: %s %s", self, reason.type, reason.value,
             )
 
             if not self._is_processing:
@@ -219,6 +219,12 @@ def _finished_processing(self):
         """Log the completion of this request and update the metrics
         """
 
+        if self.logcontext is None:
+            # this can happen if the connection closed before we read the
+            # headers (so render was never called). In that case we'll already
+            # have logged a warning, so just bail out.
+            return
+
         usage = self.logcontext.get_resource_usage()
 
         if self._processing_finished_time is None:
diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py
index 848765e53041..a0c2d37610ea 100644
--- a/synapse/util/logcontext.py
+++ b/synapse/util/logcontext.py
@@ -402,7 +402,9 @@ class PreserveLoggingContext(object):
 
     __slots__ = ["current_context", "new_context", "has_parent"]
 
-    def __init__(self, new_context=LoggingContext.sentinel):
+    def __init__(self, new_context=None):
+        if new_context is None:
+            new_context = LoggingContext.sentinel
         self.new_context = new_context
 
     def __enter__(self):

From f89f6b7c09eb8607c6744db839151452ca44c18e Mon Sep 17 00:00:00 2001
From: Will Hunt <will@half-shot.uk>
Date: Mon, 20 Aug 2018 18:25:54 +0100
Subject: [PATCH 468/830] Allow guests to access /rooms/:roomId/event/:eventId

---
 synapse/rest/client/v1/room.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py
index fcc109176014..976d98387d0a 100644
--- a/synapse/rest/client/v1/room.py
+++ b/synapse/rest/client/v1/room.py
@@ -531,7 +531,7 @@ def __init__(self, hs):
 
     @defer.inlineCallbacks
     def on_GET(self, request, room_id, event_id):
-        requester = yield self.auth.get_user_by_req(request)
+        requester = yield self.auth.get_user_by_req(request, allow_guest=True)
         event = yield self.event_handler.get_event(requester.user, room_id, event_id)
 
         time_now = self.clock.time_msec()

From 012d612f9d3c8ca58538fffe4207309bf1644cd8 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 20 Aug 2018 18:26:27 +0100
Subject: [PATCH 469/830] changelog

---
 changelog.d/3723.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3723.bugfix

diff --git a/changelog.d/3723.bugfix b/changelog.d/3723.bugfix
new file mode 100644
index 000000000000..7c21083691f3
--- /dev/null
+++ b/changelog.d/3723.bugfix
@@ -0,0 +1 @@
+Fix bug in v0.33.3rc1 which caused infinite loops and OOMs

From 8bd585b09bc049ab743cf7d2162f57164ad3b7f0 Mon Sep 17 00:00:00 2001
From: Will Hunt <will@half-shot.uk>
Date: Mon, 20 Aug 2018 18:27:42 +0100
Subject: [PATCH 470/830] 3724.feature

---
 changelog.d/3724.feature | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3724.feature

diff --git a/changelog.d/3724.feature b/changelog.d/3724.feature
new file mode 100644
index 000000000000..1b374ccf4760
--- /dev/null
+++ b/changelog.d/3724.feature
@@ -0,0 +1 @@
+Allow guests to use /rooms/:roomId/event/:eventId

From 3b5b64ac99c1a253cc24b6a1063247115ba18cd3 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Tue, 21 Aug 2018 03:48:55 +1000
Subject: [PATCH 471/830] changelog

---
 CHANGES.md              | 9 +++++++++
 changelog.d/3723.bugfix | 1 -
 synapse/__init__.py     | 2 +-
 3 files changed, 10 insertions(+), 2 deletions(-)
 delete mode 100644 changelog.d/3723.bugfix

diff --git a/CHANGES.md b/CHANGES.md
index 68ce5d14718c..df01178971ca 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,12 @@
+Synapse 0.33.3rc2 (2018-08-21)
+==============================
+
+Bugfixes
+--------
+
+- Fix bug in v0.33.3rc1 which caused infinite loops and OOMs ([\#3723](https://github.com/matrix-org/synapse/issues/3723))
+
+
 Synapse 0.33.3rc1 (2018-08-21)
 ==============================
 
diff --git a/changelog.d/3723.bugfix b/changelog.d/3723.bugfix
deleted file mode 100644
index 7c21083691f3..000000000000
--- a/changelog.d/3723.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix bug in v0.33.3rc1 which caused infinite loops and OOMs
diff --git a/synapse/__init__.py b/synapse/__init__.py
index 5175a1161d5a..252c49ca821c 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -17,4 +17,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.33.3rc1"
+__version__ = "0.33.3rc2"

From 3f6762f0bbeb611714ba6c9c4329ea7d5e46cbda Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 21 Aug 2018 09:38:38 +0100
Subject: [PATCH 472/830] isort

---
 synapse/metrics/background_process_metrics.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/synapse/metrics/background_process_metrics.py b/synapse/metrics/background_process_metrics.py
index 264fb93892d0..167167be0af3 100644
--- a/synapse/metrics/background_process_metrics.py
+++ b/synapse/metrics/background_process_metrics.py
@@ -13,9 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import six
 import threading
 
+import six
+
 from prometheus_client.core import REGISTRY, Counter, GaugeMetricFamily
 
 from twisted.internet import defer

From 808d8e06aa6a3bdd9016f82d98088111e5741f4c Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 21 Aug 2018 11:09:46 +0100
Subject: [PATCH 473/830] Don't log exceptions when failing to fetch server
 keys

Not being able to resolve or connect to remote servers is an expected
error, so we shouldn't log at ERROR with stacktraces.
---
 synapse/crypto/keyclient.py            | 8 ++++++--
 synapse/federation/transport/server.py | 4 ++--
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/synapse/crypto/keyclient.py b/synapse/crypto/keyclient.py
index c20a32096a53..e94400b8e26f 100644
--- a/synapse/crypto/keyclient.py
+++ b/synapse/crypto/keyclient.py
@@ -18,7 +18,9 @@
 from canonicaljson import json
 
 from twisted.internet import defer, reactor
+from twisted.internet.error import ConnectError
 from twisted.internet.protocol import Factory
+from twisted.names.error import DomainError
 from twisted.web.http import HTTPClient
 
 from synapse.http.endpoint import matrix_federation_endpoint
@@ -47,12 +49,14 @@ def fetch_server_key(server_name, tls_client_options_factory, path=KEY_API_V1):
                 server_response, server_certificate = yield protocol.remote_key
                 defer.returnValue((server_response, server_certificate))
         except SynapseKeyClientError as e:
-            logger.exception("Error getting key for %r" % (server_name,))
+            logger.warn("Error getting key for %r: %s", server_name, e)
             if e.status.startswith("4"):
                 # Don't retry for 4xx responses.
                 raise IOError("Cannot get key for %r" % server_name)
+        except (ConnectError, DomainError) as e:
+            logger.warn("Error getting key for %r: %s", server_name, e)
         except Exception as e:
-            logger.exception(e)
+            logger.exception("Error getting key for %r", server_name)
     raise IOError("Cannot get key for %r" % server_name)
 
 
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 77969a4f3804..33a37d4498c7 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -261,10 +261,10 @@ def new_func(request, *args, **kwargs):
             except NoAuthenticationError:
                 origin = None
                 if self.REQUIRE_AUTH:
-                    logger.exception("authenticate_request failed")
+                    logger.warn("authenticate_request failed")
                     raise
             except Exception:
-                logger.exception("authenticate_request failed")
+                logger.warn("authenticate_request failed")
                 raise
 
             if origin:

From 79d3b4689e1e9fa5304beb91c6b738f9131b6aa8 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 21 Aug 2018 11:21:48 +0100
Subject: [PATCH 474/830] Newsfile

---
 changelog.d/3727.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3727.misc

diff --git a/changelog.d/3727.misc b/changelog.d/3727.misc
new file mode 100644
index 000000000000..0b83220d900a
--- /dev/null
+++ b/changelog.d/3727.misc
@@ -0,0 +1 @@
+Log failure to authenticate remote servers as warnings (without stack traces)

From c2c153dd3b97ec98ec7ef295deeb466d72027a01 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 21 Aug 2018 11:41:07 +0100
Subject: [PATCH 475/830] Log more detail when we fail to authenticate request

---
 synapse/federation/transport/server.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 33a37d4498c7..7a993fd1cfff 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -261,10 +261,10 @@ def new_func(request, *args, **kwargs):
             except NoAuthenticationError:
                 origin = None
                 if self.REQUIRE_AUTH:
-                    logger.warn("authenticate_request failed")
+                    logger.warn("authenticate_request failed: missing authentication")
                     raise
-            except Exception:
-                logger.warn("authenticate_request failed")
+            except Exception as e:
+                logger.warn("authenticate_request failed: %s", e)
                 raise
 
             if origin:

From 8550a7e9c2fcf9e46c717127d14c79010350a998 Mon Sep 17 00:00:00 2001
From: Hubert Chathi <hubert@uhoreg.ca>
Date: Tue, 21 Aug 2018 10:38:00 -0400
Subject: [PATCH 476/830] allow auth_data to be any JSON instead of a string

---
 synapse/storage/e2e_room_keys.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py
index 089989fcfa5f..c7b1fad21e79 100644
--- a/synapse/storage/e2e_room_keys.py
+++ b/synapse/storage/e2e_room_keys.py
@@ -15,6 +15,7 @@
 
 from twisted.internet import defer
 from synapse.api.errors import StoreError
+import simplejson as json
 
 from ._base import SQLBaseStore
 
@@ -215,7 +216,7 @@ def _get_e2e_room_keys_version_info_txn(txn):
             else:
                 this_version = version
 
-            return self._simple_select_one_txn(
+            result = self._simple_select_one_txn(
                 txn,
                 table="e2e_room_keys_versions",
                 keyvalues={
@@ -228,6 +229,8 @@ def _get_e2e_room_keys_version_info_txn(txn):
                     "auth_data",
                 ),
             )
+            result["auth_data"] = json.loads(result["auth_data"])
+            return result
 
         return self.runInteraction(
             "get_e2e_room_keys_version_info",
@@ -264,7 +267,7 @@ def _create_e2e_room_keys_version_txn(txn):
                     "user_id": user_id,
                     "version": new_version,
                     "algorithm": info["algorithm"],
-                    "auth_data": info["auth_data"],
+                    "auth_data": json.dumps(info["auth_data"]),
                 },
             )
 

From cd6937fb266b72080f73a7a41da761f7f8775d96 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 21 Aug 2018 16:28:10 +0100
Subject: [PATCH 477/830] Fix typo

---
 synapse/http/request_metrics.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py
index 284d5e05dee5..72c26546788c 100644
--- a/synapse/http/request_metrics.py
+++ b/synapse/http/request_metrics.py
@@ -113,7 +113,7 @@
 _in_flight_requests = set()
 
 # Protects the _in_flight_requests set from concurrent accesss
-_in_flight_reqeuests_lock = threading.Lock()
+_in_flight_requests_lock = threading.Lock()
 
 
 def _get_in_flight_counts():
@@ -124,7 +124,7 @@ def _get_in_flight_counts():
     """
     # Cast to a list to prevent it changing while the Prometheus
     # thread is collecting metrics
-    with _in_flight_reqeuests_lock:
+    with _in_flight_requests_lock:
         reqs = list(_in_flight_requests)
 
     for rm in reqs:
@@ -159,11 +159,11 @@ def start(self, time_sec, name, method):
         # to the "in flight" metrics.
         self._request_stats = self.start_context.get_resource_usage()
 
-        with _in_flight_reqeuests_lock:
+        with _in_flight_requests_lock:
             _in_flight_requests.add(self)
 
     def stop(self, time_sec, request):
-        with _in_flight_reqeuests_lock:
+        with _in_flight_requests_lock:
             _in_flight_requests.discard(self)
 
         context = LoggingContext.current_context()

From 42a394caa2884096e5afe1f1c2c75680792769d8 Mon Sep 17 00:00:00 2001
From: Hubert Chathi <hubert@uhoreg.ca>
Date: Tue, 21 Aug 2018 14:51:34 -0400
Subject: [PATCH 478/830] allow session_data to be any JSON instead of just a
 string

---
 synapse/storage/e2e_room_keys.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py
index c7b1fad21e79..b695570a7b70 100644
--- a/synapse/storage/e2e_room_keys.py
+++ b/synapse/storage/e2e_room_keys.py
@@ -58,6 +58,8 @@ def get_e2e_room_key(self, user_id, version, room_id, session_id):
             desc="get_e2e_room_key",
         )
 
+        row["session_data"] = json.loads(row["session_data"]);
+
         defer.returnValue(row)
 
     @defer.inlineCallbacks
@@ -87,7 +89,7 @@ def set_e2e_room_key(self, user_id, version, room_id, session_id, room_key):
                 "first_message_index": room_key['first_message_index'],
                 "forwarded_count": room_key['forwarded_count'],
                 "is_verified": room_key['is_verified'],
-                "session_data": room_key['session_data'],
+                "session_data": json.dumps(room_key['session_data']),
             },
             lock=False,
         )
@@ -145,7 +147,7 @@ def get_e2e_room_keys(
                 "first_message_index": row["first_message_index"],
                 "forwarded_count": row["forwarded_count"],
                 "is_verified": row["is_verified"],
-                "session_data": row["session_data"],
+                "session_data": json.loads(row["session_data"]),
             }
 
         defer.returnValue(sessions)

From f7baff6f7b4af039254ec16e9272b90adb58dab3 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 21 Aug 2018 22:41:41 +0100
Subject: [PATCH 479/830] Fix 500 error from /consent form

Fixes #3731
---
 synapse/rest/consent/consent_resource.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py
index 147ff7d79b01..52954ada12c8 100644
--- a/synapse/rest/consent/consent_resource.py
+++ b/synapse/rest/consent/consent_resource.py
@@ -175,7 +175,7 @@ def _async_render_POST(self, request):
         """
         version = parse_string(request, "v", required=True)
         username = parse_string(request, "u", required=True)
-        userhmac = parse_string(request, "h", required=True)
+        userhmac = parse_string(request, "h", required=True, encoding=None)
 
         self._check_hash(username, userhmac)
 
@@ -210,9 +210,18 @@ def _render_template(self, request, template_name, **template_args):
         finish_request(request)
 
     def _check_hash(self, userid, userhmac):
+        """
+        Args:
+            userid (unicode):
+            userhmac (bytes):
+
+        Raises:
+              SynapseError if the hash doesn't match
+
+        """
         want_mac = hmac.new(
             key=self._hmac_secret,
-            msg=userid,
+            msg=userid.encode('utf-8'),
             digestmod=sha256,
         ).hexdigest()
 

From f7bf181a909384b649d2b615569921ae6add0505 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 21 Aug 2018 23:14:25 +0100
Subject: [PATCH 480/830] fix another consent encoding fail

---
 synapse/rest/consent/consent_resource.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py
index 52954ada12c8..7362e1858db6 100644
--- a/synapse/rest/consent/consent_resource.py
+++ b/synapse/rest/consent/consent_resource.py
@@ -140,7 +140,7 @@ def _async_render_GET(self, request):
         version = parse_string(request, "v",
                                default=self._default_consent_version)
         username = parse_string(request, "u", required=True)
-        userhmac = parse_string(request, "h", required=True)
+        userhmac = parse_string(request, "h", required=True, encoding=None)
 
         self._check_hash(username, userhmac)
 

From afb4b490a417aec9223cfd1dcf5aa36c0fc5a6d1 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 21 Aug 2018 23:19:14 +0100
Subject: [PATCH 481/830] changelog

---
 changelog.d/3732.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3732.bugfix

diff --git a/changelog.d/3732.bugfix b/changelog.d/3732.bugfix
new file mode 100644
index 000000000000..638b6334b024
--- /dev/null
+++ b/changelog.d/3732.bugfix
@@ -0,0 +1 @@
+Fix bug introduced in v0.33.3rc1 which made the ToS give a 500 error
\ No newline at end of file

From bb81e78ec6c05edc95b25a954bdf1cf688d4d652 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@arasphere.net>
Date: Wed, 22 Aug 2018 00:56:37 +0200
Subject: [PATCH 482/830] Split the state_group_cache in two (#3726)

Splits the state_group_cache in two.

One half contains normal state events; the other contains member events.

The idea is that the lazyloading common case of: "I want a subset of member events plus all of the other state" can be accomplished efficiently by splitting the cache into two, and asking for "all events" from the non-members cache, and "just these keys" from the members cache.  This means we can avoid having to make DictionaryCache aware of these sort of complicated queries, whilst letting LL requests benefit from the caching.

Previously we were unable to sensibly use the caching and had to pull all state from the DB irrespective of the filtering, which made things slow.  Hopefully fixes https://github.com/matrix-org/synapse/issues/3720.
---
 changelog.d/3726.misc       |   1 +
 synapse/storage/state.py    | 158 ++++++++++++++++++++++++++++++++----
 tests/storage/test_state.py | 105 +++++++++++++++++++++---
 3 files changed, 236 insertions(+), 28 deletions(-)
 create mode 100644 changelog.d/3726.misc

diff --git a/changelog.d/3726.misc b/changelog.d/3726.misc
new file mode 100644
index 000000000000..c4f66ec998af
--- /dev/null
+++ b/changelog.d/3726.misc
@@ -0,0 +1 @@
+Split the state_group_cache into member and non-member state events (and so speed up LL /sync)
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index dd03c4168b6c..4b971efdbaf8 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -60,8 +60,43 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore):
     def __init__(self, db_conn, hs):
         super(StateGroupWorkerStore, self).__init__(db_conn, hs)
 
+        # Originally the state store used a single DictionaryCache to cache the
+        # event IDs for the state types in a given state group to avoid hammering
+        # on the state_group* tables.
+        #
+        # The point of using a DictionaryCache is that it can cache a subset
+        # of the state events for a given state group (i.e. a subset of the keys for a
+        # given dict which is an entry in the cache for a given state group ID).
+        #
+        # However, this poses problems when performing complicated queries
+        # on the store - for instance: "give me all the state for this group, but
+        # limit members to this subset of users", as DictionaryCache's API isn't
+        # rich enough to say "please cache any of these fields, apart from this subset".
+        # This is problematic when lazy loading members, which requires this behaviour,
+        # as without it the cache has no choice but to speculatively load all
+        # state events for the group, which negates the efficiency being sought.
+        #
+        # Rather than overcomplicating DictionaryCache's API, we instead split the
+        # state_group_cache into two halves - one for tracking non-member events,
+        # and the other for tracking member_events.  This means that lazy loading
+        # queries can be made in a cache-friendly manner by querying both caches
+        # separately and then merging the result.  So for the example above, you
+        # would query the members cache for a specific subset of state keys
+        # (which DictionaryCache will handle efficiently and fine) and the non-members
+        # cache for all state (which DictionaryCache will similarly handle fine)
+        # and then just merge the results together.
+        #
+        # We size the non-members cache to be smaller than the members cache as the
+        # vast majority of state in Matrix (today) is member events.
+
         self._state_group_cache = DictionaryCache(
-            "*stateGroupCache*", 500000 * get_cache_factor_for("stateGroupCache")
+            "*stateGroupCache*",
+            # TODO: this hasn't been tuned yet
+            50000 * get_cache_factor_for("stateGroupCache")
+        )
+        self._state_group_members_cache = DictionaryCache(
+            "*stateGroupMembersCache*",
+            500000 * get_cache_factor_for("stateGroupMembersCache")
         )
 
     @defer.inlineCallbacks
@@ -275,7 +310,7 @@ def get_state_groups(self, room_id, event_ids):
         })
 
     @defer.inlineCallbacks
-    def _get_state_groups_from_groups(self, groups, types):
+    def _get_state_groups_from_groups(self, groups, types, members=None):
         """Returns the state groups for a given set of groups, filtering on
         types of state events.
 
@@ -284,6 +319,9 @@ def _get_state_groups_from_groups(self, groups, types):
             types (Iterable[str, str|None]|None): list of 2-tuples of the form
                 (`type`, `state_key`), where a `state_key` of `None` matches all
                 state_keys for the `type`. If None, all types are returned.
+            members (bool|None): If not None, then, in addition to any filtering
+                implied by types, the results are also filtered to only include
+                member events (if True), or to exclude member events (if False)
 
         Returns:
             dictionary state_group -> (dict of (type, state_key) -> event id)
@@ -294,14 +332,14 @@ def _get_state_groups_from_groups(self, groups, types):
         for chunk in chunks:
             res = yield self.runInteraction(
                 "_get_state_groups_from_groups",
-                self._get_state_groups_from_groups_txn, chunk, types,
+                self._get_state_groups_from_groups_txn, chunk, types, members,
             )
             results.update(res)
 
         defer.returnValue(results)
 
     def _get_state_groups_from_groups_txn(
-        self, txn, groups, types=None,
+        self, txn, groups, types=None, members=None,
     ):
         results = {group: {} for group in groups}
 
@@ -339,6 +377,11 @@ def _get_state_groups_from_groups_txn(
                 %s
             """)
 
+            if members is True:
+                sql += " AND type = '%s'" % (EventTypes.Member,)
+            elif members is False:
+                sql += " AND type <> '%s'" % (EventTypes.Member,)
+
             # Turns out that postgres doesn't like doing a list of OR's and
             # is about 1000x slower, so we just issue a query for each specific
             # type seperately.
@@ -386,6 +429,11 @@ def _get_state_groups_from_groups_txn(
             else:
                 where_clause = ""
 
+            if members is True:
+                where_clause += " AND type = '%s'" % EventTypes.Member
+            elif members is False:
+                where_clause += " AND type <> '%s'" % EventTypes.Member
+
             # We don't use WITH RECURSIVE on sqlite3 as there are distributions
             # that ship with an sqlite3 version that doesn't support it (e.g. wheezy)
             for group in groups:
@@ -580,10 +628,11 @@ def _get_state_group_for_events(self, event_ids):
 
         defer.returnValue({row["event_id"]: row["state_group"] for row in rows})
 
-    def _get_some_state_from_cache(self, group, types, filtered_types=None):
+    def _get_some_state_from_cache(self, cache, group, types, filtered_types=None):
         """Checks if group is in cache. See `_get_state_for_groups`
 
         Args:
+            cache(DictionaryCache): the state group cache to use
             group(int): The state group to lookup
             types(list[str, str|None]): List of 2-tuples of the form
                 (`type`, `state_key`), where a `state_key` of `None` matches all
@@ -597,11 +646,11 @@ def _get_some_state_from_cache(self, group, types, filtered_types=None):
         requests state from the cache, if False we need to query the DB for the
         missing state.
         """
-        is_all, known_absent, state_dict_ids = self._state_group_cache.get(group)
+        is_all, known_absent, state_dict_ids = cache.get(group)
 
         type_to_key = {}
 
-        # tracks whether any of ourrequested types are missing from the cache
+        # tracks whether any of our requested types are missing from the cache
         missing_types = False
 
         for typ, state_key in types:
@@ -648,7 +697,7 @@ def include(typ, state_key):
             if include(k[0], k[1])
         }, got_all
 
-    def _get_all_state_from_cache(self, group):
+    def _get_all_state_from_cache(self, cache, group):
         """Checks if group is in cache. See `_get_state_for_groups`
 
         Returns 2-tuple (`state_dict`, `got_all`). `got_all` is a bool
@@ -656,9 +705,10 @@ def _get_all_state_from_cache(self, group):
         cache, if False we need to query the DB for the missing state.
 
         Args:
+            cache(DictionaryCache): the state group cache to use
             group: The state group to lookup
         """
-        is_all, _, state_dict_ids = self._state_group_cache.get(group)
+        is_all, _, state_dict_ids = cache.get(group)
 
         return state_dict_ids, is_all
 
@@ -681,6 +731,62 @@ def _get_state_for_groups(self, groups, types=None, filtered_types=None):
                 list of event types.  Other types of events are returned unfiltered.
                 If None, `types` filtering is applied to all events.
 
+        Returns:
+            Deferred[dict[int, dict[(type, state_key), EventBase]]]
+                a dictionary mapping from state group to state dictionary.
+        """
+        if types is not None:
+            non_member_types = [t for t in types if t[0] != EventTypes.Member]
+
+            if filtered_types is not None and EventTypes.Member not in filtered_types:
+                # we want all of the membership events
+                member_types = None
+            else:
+                member_types = [t for t in types if t[0] == EventTypes.Member]
+
+        else:
+            non_member_types = None
+            member_types = None
+
+        non_member_state = yield self._get_state_for_groups_using_cache(
+            groups, self._state_group_cache, non_member_types, filtered_types,
+        )
+        # XXX: we could skip this entirely if member_types is []
+        member_state = yield self._get_state_for_groups_using_cache(
+            # we set filtered_types=None as member_state only ever contain members.
+            groups, self._state_group_members_cache, member_types, None,
+        )
+
+        state = non_member_state
+        for group in groups:
+            state[group].update(member_state[group])
+
+        defer.returnValue(state)
+
+    @defer.inlineCallbacks
+    def _get_state_for_groups_using_cache(
+        self, groups, cache, types=None, filtered_types=None
+    ):
+        """Gets the state at each of a list of state groups, optionally
+        filtering by type/state_key, querying from a specific cache.
+
+        Args:
+            groups (iterable[int]): list of state groups for which we want
+                to get the state.
+            cache (DictionaryCache): the cache of group ids to state dicts which
+                we will pass through - either the normal state cache or the specific
+                members state cache.
+            types (None|iterable[(str, None|str)]):
+                indicates the state type/keys required. If None, the whole
+                state is fetched and returned.
+
+                Otherwise, each entry should be a `(type, state_key)` tuple to
+                include in the response. A `state_key` of None is a wildcard
+                meaning that we require all state with that type.
+            filtered_types(list[str]|None): Only apply filtering via `types` to this
+                list of event types.  Other types of events are returned unfiltered.
+                If None, `types` filtering is applied to all events.
+
         Returns:
             Deferred[dict[int, dict[(type, state_key), EventBase]]]
                 a dictionary mapping from state group to state dictionary.
@@ -692,7 +798,7 @@ def _get_state_for_groups(self, groups, types=None, filtered_types=None):
         if types is not None:
             for group in set(groups):
                 state_dict_ids, got_all = self._get_some_state_from_cache(
-                    group, types, filtered_types
+                    cache, group, types, filtered_types
                 )
                 results[group] = state_dict_ids
 
@@ -701,7 +807,7 @@ def _get_state_for_groups(self, groups, types=None, filtered_types=None):
         else:
             for group in set(groups):
                 state_dict_ids, got_all = self._get_all_state_from_cache(
-                    group
+                    cache, group
                 )
 
                 results[group] = state_dict_ids
@@ -710,8 +816,8 @@ def _get_state_for_groups(self, groups, types=None, filtered_types=None):
                     missing_groups.append(group)
 
         if missing_groups:
-            # Okay, so we have some missing_types, lets fetch them.
-            cache_seq_num = self._state_group_cache.sequence
+            # Okay, so we have some missing_types, let's fetch them.
+            cache_seq_num = cache.sequence
 
             # the DictionaryCache knows if it has *all* the state, but
             # does not know if it has all of the keys of a particular type,
@@ -725,7 +831,7 @@ def _get_state_for_groups(self, groups, types=None, filtered_types=None):
                 types_to_fetch = types
 
             group_to_state_dict = yield self._get_state_groups_from_groups(
-                missing_groups, types_to_fetch
+                missing_groups, types_to_fetch, cache == self._state_group_members_cache,
             )
 
             for group, group_state_dict in iteritems(group_to_state_dict):
@@ -745,7 +851,7 @@ def _get_state_for_groups(self, groups, types=None, filtered_types=None):
 
                 # update the cache with all the things we fetched from the
                 # database.
-                self._state_group_cache.update(
+                cache.update(
                     cache_seq_num,
                     key=group,
                     value=group_state_dict,
@@ -847,15 +953,33 @@ def _store_state_group_txn(txn):
                     ],
                 )
 
-            # Prefill the state group cache with this group.
+            # Prefill the state group caches with this group.
             # It's fine to use the sequence like this as the state group map
             # is immutable. (If the map wasn't immutable then this prefill could
             # race with another update)
+
+            current_member_state_ids = {
+                s: ev
+                for (s, ev) in iteritems(current_state_ids)
+                if s[0] == EventTypes.Member
+            }
+            txn.call_after(
+                self._state_group_members_cache.update,
+                self._state_group_members_cache.sequence,
+                key=state_group,
+                value=dict(current_member_state_ids),
+            )
+
+            current_non_member_state_ids = {
+                s: ev
+                for (s, ev) in iteritems(current_state_ids)
+                if s[0] != EventTypes.Member
+            }
             txn.call_after(
                 self._state_group_cache.update,
                 self._state_group_cache.sequence,
                 key=state_group,
-                value=dict(current_state_ids),
+                value=dict(current_non_member_state_ids),
             )
 
             return state_group
diff --git a/tests/storage/test_state.py b/tests/storage/test_state.py
index ebfd969b36bb..d717b9f94e27 100644
--- a/tests/storage/test_state.py
+++ b/tests/storage/test_state.py
@@ -185,6 +185,7 @@ def test_get_state_for_event(self):
 
         # test _get_some_state_from_cache correctly filters out members with types=[]
         (state_dict, is_all) = yield self.store._get_some_state_from_cache(
+            self.store._state_group_cache,
             group, [], filtered_types=[EventTypes.Member]
         )
 
@@ -197,8 +198,20 @@ def test_get_state_for_event(self):
             state_dict,
         )
 
+        (state_dict, is_all) = yield self.store._get_some_state_from_cache(
+            self.store._state_group_members_cache,
+            group, [], filtered_types=[EventTypes.Member]
+        )
+
+        self.assertEqual(is_all, True)
+        self.assertDictEqual(
+            {},
+            state_dict,
+        )
+
         # test _get_some_state_from_cache correctly filters in members with wildcard types
         (state_dict, is_all) = yield self.store._get_some_state_from_cache(
+            self.store._state_group_cache,
             group, [(EventTypes.Member, None)], filtered_types=[EventTypes.Member]
         )
 
@@ -207,6 +220,18 @@ def test_get_state_for_event(self):
             {
                 (e1.type, e1.state_key): e1.event_id,
                 (e2.type, e2.state_key): e2.event_id,
+            },
+            state_dict,
+        )
+
+        (state_dict, is_all) = yield self.store._get_some_state_from_cache(
+            self.store._state_group_members_cache,
+            group, [(EventTypes.Member, None)], filtered_types=[EventTypes.Member]
+        )
+
+        self.assertEqual(is_all, True)
+        self.assertDictEqual(
+            {
                 (e3.type, e3.state_key): e3.event_id,
                 # e4 is overwritten by e5
                 (e5.type, e5.state_key): e5.event_id,
@@ -216,6 +241,7 @@ def test_get_state_for_event(self):
 
         # test _get_some_state_from_cache correctly filters in members with specific types
         (state_dict, is_all) = yield self.store._get_some_state_from_cache(
+            self.store._state_group_cache,
             group,
             [(EventTypes.Member, e5.state_key)],
             filtered_types=[EventTypes.Member],
@@ -226,6 +252,20 @@ def test_get_state_for_event(self):
             {
                 (e1.type, e1.state_key): e1.event_id,
                 (e2.type, e2.state_key): e2.event_id,
+            },
+            state_dict,
+        )
+
+        (state_dict, is_all) = yield self.store._get_some_state_from_cache(
+            self.store._state_group_members_cache,
+            group,
+            [(EventTypes.Member, e5.state_key)],
+            filtered_types=[EventTypes.Member],
+        )
+
+        self.assertEqual(is_all, True)
+        self.assertDictEqual(
+            {
                 (e5.type, e5.state_key): e5.event_id,
             },
             state_dict,
@@ -234,6 +274,7 @@ def test_get_state_for_event(self):
         # test _get_some_state_from_cache correctly filters in members with specific types
         # and no filtered_types
         (state_dict, is_all) = yield self.store._get_some_state_from_cache(
+            self.store._state_group_members_cache,
             group, [(EventTypes.Member, e5.state_key)], filtered_types=None
         )
 
@@ -254,9 +295,6 @@ def test_get_state_for_event(self):
             {
                 (e1.type, e1.state_key): e1.event_id,
                 (e2.type, e2.state_key): e2.event_id,
-                (e3.type, e3.state_key): e3.event_id,
-                # e4 is overwritten by e5
-                (e5.type, e5.state_key): e5.event_id,
             },
         )
 
@@ -269,8 +307,6 @@ def test_get_state_for_event(self):
             # list fetched keys so it knows it's partial
             fetched_keys=(
                 (e1.type, e1.state_key),
-                (e3.type, e3.state_key),
-                (e5.type, e5.state_key),
             ),
         )
 
@@ -284,8 +320,6 @@ def test_get_state_for_event(self):
             set(
                 [
                     (e1.type, e1.state_key),
-                    (e3.type, e3.state_key),
-                    (e5.type, e5.state_key),
                 ]
             ),
         )
@@ -293,8 +327,6 @@ def test_get_state_for_event(self):
             state_dict_ids,
             {
                 (e1.type, e1.state_key): e1.event_id,
-                (e3.type, e3.state_key): e3.event_id,
-                (e5.type, e5.state_key): e5.event_id,
             },
         )
 
@@ -304,14 +336,25 @@ def test_get_state_for_event(self):
         # test _get_some_state_from_cache correctly filters out members with types=[]
         room_id = self.room.to_string()
         (state_dict, is_all) = yield self.store._get_some_state_from_cache(
+            self.store._state_group_cache,
             group, [], filtered_types=[EventTypes.Member]
         )
 
         self.assertEqual(is_all, False)
         self.assertDictEqual({(e1.type, e1.state_key): e1.event_id}, state_dict)
 
+        room_id = self.room.to_string()
+        (state_dict, is_all) = yield self.store._get_some_state_from_cache(
+            self.store._state_group_members_cache,
+            group, [], filtered_types=[EventTypes.Member]
+        )
+
+        self.assertEqual(is_all, True)
+        self.assertDictEqual({}, state_dict)
+
         # test _get_some_state_from_cache correctly filters in members wildcard types
         (state_dict, is_all) = yield self.store._get_some_state_from_cache(
+            self.store._state_group_cache,
             group, [(EventTypes.Member, None)], filtered_types=[EventTypes.Member]
         )
 
@@ -319,8 +362,19 @@ def test_get_state_for_event(self):
         self.assertDictEqual(
             {
                 (e1.type, e1.state_key): e1.event_id,
+            },
+            state_dict,
+        )
+
+        (state_dict, is_all) = yield self.store._get_some_state_from_cache(
+            self.store._state_group_members_cache,
+            group, [(EventTypes.Member, None)], filtered_types=[EventTypes.Member]
+        )
+
+        self.assertEqual(is_all, True)
+        self.assertDictEqual(
+            {
                 (e3.type, e3.state_key): e3.event_id,
-                # e4 is overwritten by e5
                 (e5.type, e5.state_key): e5.event_id,
             },
             state_dict,
@@ -328,6 +382,7 @@ def test_get_state_for_event(self):
 
         # test _get_some_state_from_cache correctly filters in members with specific types
         (state_dict, is_all) = yield self.store._get_some_state_from_cache(
+            self.store._state_group_cache,
             group,
             [(EventTypes.Member, e5.state_key)],
             filtered_types=[EventTypes.Member],
@@ -337,6 +392,20 @@ def test_get_state_for_event(self):
         self.assertDictEqual(
             {
                 (e1.type, e1.state_key): e1.event_id,
+            },
+            state_dict,
+        )
+
+        (state_dict, is_all) = yield self.store._get_some_state_from_cache(
+            self.store._state_group_members_cache,
+            group,
+            [(EventTypes.Member, e5.state_key)],
+            filtered_types=[EventTypes.Member],
+        )
+
+        self.assertEqual(is_all, True)
+        self.assertDictEqual(
+            {
                 (e5.type, e5.state_key): e5.event_id,
             },
             state_dict,
@@ -345,8 +414,22 @@ def test_get_state_for_event(self):
         # test _get_some_state_from_cache correctly filters in members with specific types
         # and no filtered_types
         (state_dict, is_all) = yield self.store._get_some_state_from_cache(
+            self.store._state_group_cache,
+            group, [(EventTypes.Member, e5.state_key)], filtered_types=None
+        )
+
+        self.assertEqual(is_all, False)
+        self.assertDictEqual({}, state_dict)
+
+        (state_dict, is_all) = yield self.store._get_some_state_from_cache(
+            self.store._state_group_members_cache,
             group, [(EventTypes.Member, e5.state_key)], filtered_types=None
         )
 
         self.assertEqual(is_all, True)
-        self.assertDictEqual({(e5.type, e5.state_key): e5.event_id}, state_dict)
+        self.assertDictEqual(
+            {
+                (e5.type, e5.state_key): e5.event_id,
+            },
+            state_dict,
+        )

From dd0ac1614cc9649936830bc1efd1f6208528d504 Mon Sep 17 00:00:00 2001
From: Travis Ralston <travpc@gmail.com>
Date: Tue, 21 Aug 2018 23:35:50 -0600
Subject: [PATCH 483/830] Reference that the federation_reader needs the HTTP
 replication port set

---
 docs/workers.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/workers.rst b/docs/workers.rst
index aec319dd8458..5435f3633125 100644
--- a/docs/workers.rst
+++ b/docs/workers.rst
@@ -74,7 +74,7 @@ replication endpoints that it's talking to on the main synapse process.
 ``worker_replication_port`` should point to the TCP replication listener port and
 ``worker_replication_http_port`` should point to the HTTP replication port.
 
-Currently, only the ``event_creator`` worker requires specifying
+Currently, the ``event_creator`` and ``federation_reader`` workers require specifying
 ``worker_replication_http_port``.
 
 For instance::

From eb7be75a108e002d547dc0f0566386ddc0239a5f Mon Sep 17 00:00:00 2001
From: Travis Ralston <travpc@gmail.com>
Date: Tue, 21 Aug 2018 23:38:06 -0600
Subject: [PATCH 484/830] Create 3735.misc

---
 changelog.d/3735.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3735.misc

diff --git a/changelog.d/3735.misc b/changelog.d/3735.misc
new file mode 100644
index 000000000000..f17004be71e3
--- /dev/null
+++ b/changelog.d/3735.misc
@@ -0,0 +1 @@
+Fix minor spelling error in federation client documentation.

From 365f588d987d531bac9506216f12c44b28c8c5b9 Mon Sep 17 00:00:00 2001
From: Travis Ralston <travpc@gmail.com>
Date: Tue, 21 Aug 2018 23:38:38 -0600
Subject: [PATCH 485/830] Create 3734.misc

---
 changelog.d/3734.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3734.misc

diff --git a/changelog.d/3734.misc b/changelog.d/3734.misc
new file mode 100644
index 000000000000..4f6e4b384808
--- /dev/null
+++ b/changelog.d/3734.misc
@@ -0,0 +1 @@
+Reference the need for an HTTP replication port when using the federation_reader worker

From 47b25ba5f3040d07f8be80584b292ca0386fdb30 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 22 Aug 2018 10:09:05 +0100
Subject: [PATCH 486/830] Remove redundant vars

---
 synapse/handlers/profile.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py
index 3e1d95d5e5c0..ec9cb1f50c83 100644
--- a/synapse/handlers/profile.py
+++ b/synapse/handlers/profile.py
@@ -33,9 +33,6 @@
 
 
 class WorkerProfileHandler(BaseHandler):
-    PROFILE_UPDATE_MS = 60 * 1000
-    PROFILE_UPDATE_EVERY_MS = 24 * 60 * 60 * 1000
-
     def __init__(self, hs):
         super(WorkerProfileHandler, self).__init__(hs)
 

From a81f1408807683d212b7bf3c4f5f36df4b0e0d33 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 22 Aug 2018 10:11:21 +0100
Subject: [PATCH 487/830] Add assert to ensure handler is only run on master

---
 synapse/handlers/profile.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py
index ec9cb1f50c83..278f131d5e2d 100644
--- a/synapse/handlers/profile.py
+++ b/synapse/handlers/profile.py
@@ -282,6 +282,8 @@ class MasterProfileHandler(WorkerProfileHandler):
     def __init__(self, hs):
         super(MasterProfileHandler, self).__init__(hs)
 
+        assert hs.config.worker_app is None
+
         self.clock.looping_call(
             self._start_update_remote_profile_cache, self.PROFILE_UPDATE_MS,
         )

From 8432e2ebd76d0462d39a329967671e845da7e404 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 22 Aug 2018 10:13:40 +0100
Subject: [PATCH 488/830] Rename WorkerProfileHandler to BaseProfileHandler

---
 synapse/handlers/profile.py | 13 ++++++++++---
 synapse/server.py           |  4 ++--
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py
index 278f131d5e2d..75b8b7ce6a76 100644
--- a/synapse/handlers/profile.py
+++ b/synapse/handlers/profile.py
@@ -32,9 +32,16 @@
 logger = logging.getLogger(__name__)
 
 
-class WorkerProfileHandler(BaseHandler):
+class BaseProfileHandler(BaseHandler):
+    """Handles fetching and updating user profile information.
+
+    BaseProfileHandler can be instantiated directly on workers and will
+    delegate to master when necessary. The master process should use the
+    subclass MasterProfileHandler
+    """
+
     def __init__(self, hs):
-        super(WorkerProfileHandler, self).__init__(hs)
+        super(BaseProfileHandler, self).__init__(hs)
 
         self.federation = hs.get_federation_client()
         hs.get_federation_registry().register_query_handler(
@@ -275,7 +282,7 @@ def _update_join_states(self, requester, target_user):
                 )
 
 
-class MasterProfileHandler(WorkerProfileHandler):
+class MasterProfileHandler(BaseProfileHandler):
     PROFILE_UPDATE_MS = 60 * 1000
     PROFILE_UPDATE_EVERY_MS = 24 * 60 * 60 * 1000
 
diff --git a/synapse/server.py b/synapse/server.py
index fcfa7e65576a..a6fbc6ec0c64 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -56,7 +56,7 @@
 from synapse.handlers.message import EventCreationHandler, MessageHandler
 from synapse.handlers.pagination import PaginationHandler
 from synapse.handlers.presence import PresenceHandler
-from synapse.handlers.profile import MasterProfileHandler, WorkerProfileHandler
+from synapse.handlers.profile import BaseProfileHandler, MasterProfileHandler
 from synapse.handlers.read_marker import ReadMarkerHandler
 from synapse.handlers.receipts import ReceiptsHandler
 from synapse.handlers.room import RoomContextHandler, RoomCreationHandler
@@ -309,7 +309,7 @@ def build_initial_sync_handler(self):
 
     def build_profile_handler(self):
         if self.config.worker_app:
-            return WorkerProfileHandler(self)
+            return BaseProfileHandler(self)
         else:
             return MasterProfileHandler(self)
 

From 9b7d9d8ba056393d73e924785f479c7fc601e868 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 22 Aug 2018 14:06:20 +0100
Subject: [PATCH 489/830] Update attributions and PR links in changelog

---
 CHANGES.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index df01178971ca..57f9d9a826e9 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -13,7 +13,7 @@ Synapse 0.33.3rc1 (2018-08-21)
 Features
 --------
 
-- Add support for the SNI extension to federation TLS connections ([\#1491](https://github.com/matrix-org/synapse/issues/1491))
+- Add support for the SNI extension to federation TLS connections. Thanks to @vojeroen! ([\#3439](https://github.com/matrix-org/synapse/issues/3439))
 - Add /_media/r0/config ([\#3184](https://github.com/matrix-org/synapse/issues/3184))
 - speed up /members API and add `at` and `membership` params as per MSC1227 ([\#3568](https://github.com/matrix-org/synapse/issues/3568))
 - implement `summary` block in /sync response as per MSC688 ([\#3574](https://github.com/matrix-org/synapse/issues/3574))
@@ -97,7 +97,7 @@ Features
 Bugfixes
 --------
 
-- Make /directory/list API return 404 for room not found instead of 400 ([\#2952](https://github.com/matrix-org/synapse/issues/2952))
+- Make /directory/list API return 404 for room not found instead of 400. Thanks to @fuzzmz! ([\#3620](https://github.com/matrix-org/synapse/issues/3620))
 - Default inviter_display_name to mxid for email invites ([\#3391](https://github.com/matrix-org/synapse/issues/3391))
 - Don't generate TURN credentials if no TURN config options are set ([\#3514](https://github.com/matrix-org/synapse/issues/3514))
 - Correctly announce deleted devices over federation ([\#3520](https://github.com/matrix-org/synapse/issues/3520))

From 4e5a4549b68d3cbc8e74dda9025321c4b40b0222 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 22 Aug 2018 14:07:10 +0100
Subject: [PATCH 490/830] bump version to 0.33.3

---
 synapse/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/__init__.py b/synapse/__init__.py
index 252c49ca821c..e62901b7614b 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -17,4 +17,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.33.3rc2"
+__version__ = "0.33.3"

From 3504982cb7918a42881618cb20e40257d3f9e4f9 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 22 Aug 2018 14:07:44 +0100
Subject: [PATCH 491/830] changelog for 0.33.3

---
 CHANGES.md              | 9 +++++++++
 changelog.d/3732.bugfix | 1 -
 2 files changed, 9 insertions(+), 1 deletion(-)
 delete mode 100644 changelog.d/3732.bugfix

diff --git a/CHANGES.md b/CHANGES.md
index 57f9d9a826e9..a35f5aebc795 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,12 @@
+Synapse 0.33.3 (2018-08-22)
+===========================
+
+Bugfixes
+--------
+
+- Fix bug introduced in v0.33.3rc1 which made the ToS give a 500 error ([\#3732](https://github.com/matrix-org/synapse/issues/3732))
+
+
 Synapse 0.33.3rc2 (2018-08-21)
 ==============================
 
diff --git a/changelog.d/3732.bugfix b/changelog.d/3732.bugfix
deleted file mode 100644
index 638b6334b024..000000000000
--- a/changelog.d/3732.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix bug introduced in v0.33.3rc1 which made the ToS give a 500 error
\ No newline at end of file

From 74854a97191191b08101821753c2672efc2a65fd Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 22 Aug 2018 14:30:49 +0100
Subject: [PATCH 492/830] Use recaptcha_ajax.js directly from Google

This was originally done in commit c75b71a397d91c013a65a7f57d2c13199cf4c204,
but got reverted on this branch due to the PR (#3677) being based on the wrong
branch.

We're ready to merge this to master now, so let's make it match
release-v0.33.3.
---
 synapse/static/client/register/index.html     |   2 +-
 .../client/register/js/recaptcha_ajax.js      | 195 ------------------
 2 files changed, 1 insertion(+), 196 deletions(-)
 delete mode 100644 synapse/static/client/register/js/recaptcha_ajax.js

diff --git a/synapse/static/client/register/index.html b/synapse/static/client/register/index.html
index 600b3ee41eab..886f2edd1fad 100644
--- a/synapse/static/client/register/index.html
+++ b/synapse/static/client/register/index.html
@@ -4,7 +4,7 @@
 <meta name='viewport' content='width=device-width, initial-scale=1, user-scalable=no, minimum-scale=1.0, maximum-scale=1.0'> 
 <link rel="stylesheet" href="style.css">
 <script src="js/jquery-2.1.3.min.js"></script>
-<script src="js/recaptcha_ajax.js"></script>
+<script src="https://www.google.com/recaptcha/api/js/recaptcha_ajax.js"></script>
 <script src="register_config.js"></script>
 <script src="js/register.js"></script>
 </head>
diff --git a/synapse/static/client/register/js/recaptcha_ajax.js b/synapse/static/client/register/js/recaptcha_ajax.js
deleted file mode 100644
index d0e71e5b8824..000000000000
--- a/synapse/static/client/register/js/recaptcha_ajax.js
+++ /dev/null
@@ -1,195 +0,0 @@
-(function(){var h,k=this,l=function(a){return void 0!==a},ba=function(){},n=function(a){var b=typeof a;if("object"==b)if(a){if(a instanceof Array)return"array";if(a instanceof Object)return b;var c=Object.prototype.toString.call(a);if("[object Window]"==c)return"object";if("[object Array]"==c||"number"==typeof a.length&&"undefined"!=typeof a.splice&&"undefined"!=typeof a.propertyIsEnumerable&&!a.propertyIsEnumerable("splice"))return"array";if("[object Function]"==c||"undefined"!=typeof a.call&&"undefined"!=typeof a.propertyIsEnumerable&&
-!a.propertyIsEnumerable("call"))return"function"}else return"null";else if("function"==b&&"undefined"==typeof a.call)return"object";return b},p=function(a){return"array"==n(a)},ca=function(a){var b=n(a);return"array"==b||"object"==b&&"number"==typeof a.length},q=function(a){return"string"==typeof a},r=function(a){return"function"==n(a)},da=function(a){var b=typeof a;return"object"==b&&null!=a||"function"==b},ea=function(a,b,c){return a.call.apply(a.bind,arguments)},fa=function(a,b,c){if(!a)throw Error();
-if(2<arguments.length){var d=Array.prototype.slice.call(arguments,2);return function(){var c=Array.prototype.slice.call(arguments);Array.prototype.unshift.apply(c,d);return a.apply(b,c)}}return function(){return a.apply(b,arguments)}},s=function(a,b,c){s=Function.prototype.bind&&-1!=Function.prototype.bind.toString().indexOf("native code")?ea:fa;return s.apply(null,arguments)},ga=function(a,b){var c=Array.prototype.slice.call(arguments,1);return function(){var b=c.slice();b.push.apply(b,arguments);
-return a.apply(this,b)}},ha=Date.now||function(){return+new Date},ia=null,t=function(a,b){var c=a.split("."),d=k;c[0]in d||!d.execScript||d.execScript("var "+c[0]);for(var e;c.length&&(e=c.shift());)!c.length&&l(b)?d[e]=b:d=d[e]?d[e]:d[e]={}},u=function(a,b){function c(){}c.prototype=b.prototype;a.superClass_=b.prototype;a.prototype=new c;a.base=function(a,c,g){return b.prototype[c].apply(a,Array.prototype.slice.call(arguments,2))}};
-Function.prototype.bind=Function.prototype.bind||function(a,b){if(1<arguments.length){var c=Array.prototype.slice.call(arguments,1);c.unshift(this,a);return s.apply(null,c)}return s(this,a)};var v={};t("RecaptchaTemplates",v);v.VertHtml='<table id="recaptcha_table" class="recaptchatable" > <tr> <td colspan="6" class=\'recaptcha_r1_c1\'></td> </tr> <tr> <td class=\'recaptcha_r2_c1\'></td> <td colspan="4" class=\'recaptcha_image_cell\'><center><div id="recaptcha_image"></div></center></td> <td class=\'recaptcha_r2_c2\'></td> </tr> <tr> <td rowspan="6" class=\'recaptcha_r3_c1\'></td> <td colspan="4" class=\'recaptcha_r3_c2\'></td> <td rowspan="6" class=\'recaptcha_r3_c3\'></td> </tr> <tr> <td rowspan="3" class=\'recaptcha_r4_c1\' height="49"> <div class="recaptcha_input_area"> <input name="recaptcha_response_field" id="recaptcha_response_field" type="text" autocorrect="off" autocapitalize="off" placeholder="" /> <span id="recaptcha_privacy" class="recaptcha_only_if_privacy"></span> </div> </td> <td rowspan="4" class=\'recaptcha_r4_c2\'></td> <td><a id=\'recaptcha_reload_btn\'><img id=\'recaptcha_reload\' width="25" height="17" /></a></td> <td rowspan="4" class=\'recaptcha_r4_c4\'></td> </tr> <tr> <td><a id=\'recaptcha_switch_audio_btn\' class="recaptcha_only_if_image"><img id=\'recaptcha_switch_audio\' width="25" height="16" alt="" /></a><a id=\'recaptcha_switch_img_btn\' class="recaptcha_only_if_audio"><img id=\'recaptcha_switch_img\' width="25" height="16" alt=""/></a></td> </tr> <tr> <td><a id=\'recaptcha_whatsthis_btn\'><img id=\'recaptcha_whatsthis\' width="25" height="16" /></a></td> </tr> <tr> <td class=\'recaptcha_r7_c1\'></td> <td class=\'recaptcha_r8_c1\'></td> </tr> </table> ';v.CleanCss=".recaptchatable td img{display:block}.recaptchatable .recaptcha_image_cell center img{height:57px}.recaptchatable .recaptcha_image_cell center{height:57px}.recaptchatable .recaptcha_image_cell{background-color:white;height:57px;padding:7px!important}.recaptchatable,#recaptcha_area tr,#recaptcha_area td,#recaptcha_area th{margin:0!important;border:0!important;border-collapse:collapse!important;vertical-align:middle!important}.recaptchatable *{margin:0;padding:0;border:0;color:black;position:static;top:auto;left:auto;right:auto;bottom:auto}.recaptchatable #recaptcha_image{position:relative;margin:auto;border:1px solid #dfdfdf!important}.recaptchatable #recaptcha_image #recaptcha_challenge_image{display:block}.recaptchatable #recaptcha_image #recaptcha_ad_image{display:block;position:absolute;top:0}.recaptchatable a img{border:0}.recaptchatable a,.recaptchatable a:hover{cursor:pointer;outline:none;border:0!important;padding:0!important;text-decoration:none;color:blue;background:none!important;font-weight:normal}.recaptcha_input_area{position:relative!important;background:none!important}.recaptchatable label.recaptcha_input_area_text{border:1px solid #dfdfdf!important;margin:0!important;padding:0!important;position:static!important;top:auto!important;left:auto!important;right:auto!important;bottom:auto!important}.recaptcha_theme_red label.recaptcha_input_area_text,.recaptcha_theme_white label.recaptcha_input_area_text{color:black!important}.recaptcha_theme_blackglass label.recaptcha_input_area_text{color:white!important}.recaptchatable #recaptcha_response_field{font-size:11pt}.recaptcha_theme_blackglass #recaptcha_response_field,.recaptcha_theme_white #recaptcha_response_field{border:1px solid gray}.recaptcha_theme_red #recaptcha_response_field{border:1px solid #cca940}.recaptcha_audio_cant_hear_link{font-size:7pt;color:black}.recaptchatable{line-height:1em;border:1px solid #dfdfdf!important}.recaptcha_error_text{color:red}.recaptcha_only_if_privacy{float:right;text-align:right;margin-right:7px}#recaptcha-ad-choices{position:absolute;height:15px;top:0;right:0}#recaptcha-ad-choices img{height:15px}.recaptcha-ad-choices-collapsed{width:30px;height:15px;display:block}.recaptcha-ad-choices-expanded{width:75px;height:15px;display:none}#recaptcha-ad-choices:hover .recaptcha-ad-choices-collapsed{display:none}#recaptcha-ad-choices:hover .recaptcha-ad-choices-expanded{display:block}";v.CleanHtml='<table id="recaptcha_table" class="recaptchatable"> <tr height="73"> <td class=\'recaptcha_image_cell\' width="302"><center><div id="recaptcha_image"></div></center></td> <td style="padding: 10px 7px 7px 7px;"> <a id=\'recaptcha_reload_btn\'><img id=\'recaptcha_reload\' width="25" height="18" alt="" /></a> <a id=\'recaptcha_switch_audio_btn\' class="recaptcha_only_if_image"><img id=\'recaptcha_switch_audio\' width="25" height="15" alt="" /></a><a id=\'recaptcha_switch_img_btn\' class="recaptcha_only_if_audio"><img id=\'recaptcha_switch_img\' width="25" height="15" alt=""/></a> <a id=\'recaptcha_whatsthis_btn\'><img id=\'recaptcha_whatsthis\' width="25" height="16" /></a> </td> <td style="padding: 18px 7px 18px 7px;"> <img id=\'recaptcha_logo\' alt="" width="71" height="36" /> </td> </tr> <tr> <td style="padding-left: 7px;"> <div class="recaptcha_input_area" style="padding-top: 2px; padding-bottom: 7px;"> <input style="border: 1px solid #3c3c3c; width: 302px;" name="recaptcha_response_field" id="recaptcha_response_field" type="text" /> </div> </td> <td colspan=2><span id="recaptcha_privacy" class="recaptcha_only_if_privacy"></span></td> </tr> </table> ';v.VertCss=".recaptchatable td img{display:block}.recaptchatable .recaptcha_r1_c1{background:url('IMGROOT/sprite.png') 0 -63px no-repeat;width:318px;height:9px}.recaptchatable .recaptcha_r2_c1{background:url('IMGROOT/sprite.png') -18px 0 no-repeat;width:9px;height:57px}.recaptchatable .recaptcha_r2_c2{background:url('IMGROOT/sprite.png') -27px 0 no-repeat;width:9px;height:57px}.recaptchatable .recaptcha_r3_c1{background:url('IMGROOT/sprite.png') 0 0 no-repeat;width:9px;height:63px}.recaptchatable .recaptcha_r3_c2{background:url('IMGROOT/sprite.png') -18px -57px no-repeat;width:300px;height:6px}.recaptchatable .recaptcha_r3_c3{background:url('IMGROOT/sprite.png') -9px 0 no-repeat;width:9px;height:63px}.recaptchatable .recaptcha_r4_c1{background:url('IMGROOT/sprite.png') -43px 0 no-repeat;width:171px;height:49px}.recaptchatable .recaptcha_r4_c2{background:url('IMGROOT/sprite.png') -36px 0 no-repeat;width:7px;height:57px}.recaptchatable .recaptcha_r4_c4{background:url('IMGROOT/sprite.png') -214px 0 no-repeat;width:97px;height:57px}.recaptchatable .recaptcha_r7_c1{background:url('IMGROOT/sprite.png') -43px -49px no-repeat;width:171px;height:8px}.recaptchatable .recaptcha_r8_c1{background:url('IMGROOT/sprite.png') -43px -49px no-repeat;width:25px;height:8px}.recaptchatable .recaptcha_image_cell center img{height:57px}.recaptchatable .recaptcha_image_cell center{height:57px}.recaptchatable .recaptcha_image_cell{background-color:white;height:57px}#recaptcha_area,#recaptcha_table{width:318px!important}.recaptchatable,#recaptcha_area tr,#recaptcha_area td,#recaptcha_area th{margin:0!important;border:0!important;padding:0!important;border-collapse:collapse!important;vertical-align:middle!important}.recaptchatable *{margin:0;padding:0;border:0;font-family:helvetica,sans-serif;font-size:8pt;color:black;position:static;top:auto;left:auto;right:auto;bottom:auto}.recaptchatable #recaptcha_image{position:relative;margin:auto}.recaptchatable #recaptcha_image #recaptcha_challenge_image{display:block}.recaptchatable #recaptcha_image #recaptcha_ad_image{display:block;position:absolute;top:0}.recaptchatable img{border:0!important;margin:0!important;padding:0!important}.recaptchatable a,.recaptchatable a:hover{cursor:pointer;outline:none;border:0!important;padding:0!important;text-decoration:none;color:blue;background:none!important;font-weight:normal}.recaptcha_input_area{position:relative!important;width:153px!important;height:45px!important;margin-left:7px!important;margin-right:7px!important;background:none!important}.recaptchatable label.recaptcha_input_area_text{margin:0!important;padding:0!important;position:static!important;top:auto!important;left:auto!important;right:auto!important;bottom:auto!important;background:none!important;height:auto!important;width:auto!important}.recaptcha_theme_red label.recaptcha_input_area_text,.recaptcha_theme_white label.recaptcha_input_area_text{color:black!important}.recaptcha_theme_blackglass label.recaptcha_input_area_text{color:white!important}.recaptchatable #recaptcha_response_field{width:153px!important;position:relative!important;bottom:7px!important;padding:0!important;margin:15px 0 0 0!important;font-size:10pt}.recaptcha_theme_blackglass #recaptcha_response_field,.recaptcha_theme_white #recaptcha_response_field{border:1px solid gray}.recaptcha_theme_red #recaptcha_response_field{border:1px solid #cca940}.recaptcha_audio_cant_hear_link{font-size:7pt;color:black}.recaptchatable{line-height:1!important}#recaptcha_instructions_error{color:red!important}.recaptcha_only_if_privacy{float:right;text-align:right}#recaptcha-ad-choices{position:absolute;height:15px;top:0;right:0}#recaptcha-ad-choices img{height:15px}.recaptcha-ad-choices-collapsed{width:30px;height:15px;display:block}.recaptcha-ad-choices-expanded{width:75px;height:15px;display:none}#recaptcha-ad-choices:hover .recaptcha-ad-choices-collapsed{display:none}#recaptcha-ad-choices:hover .recaptcha-ad-choices-expanded{display:block}";var w={visual_challenge:"Get a visual challenge",audio_challenge:"Get an audio challenge",refresh_btn:"Get a new challenge",instructions_visual:"Type the text:",instructions_audio:"Type what you hear:",help_btn:"Help",play_again:"Play sound again",cant_hear_this:"Download sound as MP3",incorrect_try_again:"Incorrect. Try again.",image_alt_text:"reCAPTCHA challenge image",privacy_and_terms:"Privacy & Terms"},ja={visual_challenge:"\u0627\u0644\u062d\u0635\u0648\u0644 \u0639\u0644\u0649 \u062a\u062d\u062f\u064d \u0645\u0631\u0626\u064a",
-audio_challenge:"\u0627\u0644\u062d\u0635\u0648\u0644 \u0639\u0644\u0649 \u062a\u062d\u062f\u064d \u0635\u0648\u062a\u064a",refresh_btn:"\u0627\u0644\u062d\u0635\u0648\u0644 \u0639\u0644\u0649 \u062a\u062d\u062f\u064d \u062c\u062f\u064a\u062f",instructions_visual:"\u064a\u0631\u062c\u0649 \u0643\u062a\u0627\u0628\u0629 \u0627\u0644\u0646\u0635:",instructions_audio:"\u0627\u0643\u062a\u0628 \u0645\u0627 \u062a\u0633\u0645\u0639\u0647:",help_btn:"\u0645\u0633\u0627\u0639\u062f\u0629",play_again:"\u062a\u0634\u063a\u064a\u0644 \u0627\u0644\u0635\u0648\u062a \u0645\u0631\u0629 \u0623\u062e\u0631\u0649",
-cant_hear_this:"\u062a\u0646\u0632\u064a\u0644 \u0627\u0644\u0635\u0648\u062a \u0628\u062a\u0646\u0633\u064a\u0642 MP3",incorrect_try_again:"\u063a\u064a\u0631 \u0635\u062d\u064a\u062d. \u0623\u0639\u062f \u0627\u0644\u0645\u062d\u0627\u0648\u0644\u0629.",image_alt_text:"\u0635\u0648\u0631\u0629 \u0627\u0644\u062a\u062d\u062f\u064a \u0645\u0646 reCAPTCHA",privacy_and_terms:"\u0627\u0644\u062e\u0635\u0648\u0635\u064a\u0629 \u0648\u0627\u0644\u0628\u0646\u0648\u062f"},ka={visual_challenge:"Obtener una pista visual",
-audio_challenge:"Obtener una pista sonora",refresh_btn:"Obtener una pista nueva",instructions_visual:"Introduzca el texto:",instructions_audio:"Escribe lo que oigas:",help_btn:"Ayuda",play_again:"Volver a reproducir el sonido",cant_hear_this:"Descargar el sonido en MP3",incorrect_try_again:"Incorrecto. Vu\u00e9lvelo a intentar.",image_alt_text:"Pista de imagen reCAPTCHA",privacy_and_terms:"Privacidad y condiciones"},la={visual_challenge:"Kumuha ng pagsubok na visual",audio_challenge:"Kumuha ng pagsubok na audio",
-refresh_btn:"Kumuha ng bagong pagsubok",instructions_visual:"I-type ang teksto:",instructions_audio:"I-type ang iyong narinig",help_btn:"Tulong",play_again:"I-play muli ang tunog",cant_hear_this:"I-download ang tunog bilang MP3",incorrect_try_again:"Hindi wasto. Muling subukan.",image_alt_text:"larawang panghamon ng reCAPTCHA",privacy_and_terms:"Privacy at Mga Tuntunin"},ma={visual_challenge:"Test visuel",audio_challenge:"Test audio",refresh_btn:"Nouveau test",instructions_visual:"Saisissez le texte\u00a0:",
-instructions_audio:"Qu'entendez-vous ?",help_btn:"Aide",play_again:"R\u00e9\u00e9couter",cant_hear_this:"T\u00e9l\u00e9charger l'audio au format MP3",incorrect_try_again:"Incorrect. Veuillez r\u00e9essayer.",image_alt_text:"Image reCAPTCHA",privacy_and_terms:"Confidentialit\u00e9 et conditions d'utilisation"},na={visual_challenge:"Dapatkan kata pengujian berbentuk visual",audio_challenge:"Dapatkan kata pengujian berbentuk audio",refresh_btn:"Dapatkan kata pengujian baru",instructions_visual:"Ketik teks:",
-instructions_audio:"Ketik yang Anda dengar:",help_btn:"Bantuan",play_again:"Putar suara sekali lagi",cant_hear_this:"Unduh suara sebagai MP3",incorrect_try_again:"Salah. Coba lagi.",image_alt_text:"Gambar tantangan reCAPTCHA",privacy_and_terms:"Privasi & Persyaratan"},oa={visual_challenge:"\u05e7\u05d1\u05dc \u05d0\u05ea\u05d2\u05e8 \u05d7\u05d6\u05d5\u05ea\u05d9",audio_challenge:"\u05e7\u05d1\u05dc \u05d0\u05ea\u05d2\u05e8 \u05e9\u05de\u05e2",refresh_btn:"\u05e7\u05d1\u05dc \u05d0\u05ea\u05d2\u05e8 \u05d7\u05d3\u05e9",
-instructions_visual:"\u05d4\u05e7\u05dc\u05d3 \u05d0\u05ea \u05d4\u05d8\u05e7\u05e1\u05d8:",instructions_audio:"\u05d4\u05e7\u05dc\u05d3 \u05d0\u05ea \u05de\u05d4 \u05e9\u05d0\u05ea\u05d4 \u05e9\u05d5\u05de\u05e2:",help_btn:"\u05e2\u05d6\u05e8\u05d4",play_again:"\u05d4\u05e4\u05e2\u05dc \u05e9\u05d5\u05d1 \u05d0\u05ea \u05d4\u05e9\u05de\u05e2",cant_hear_this:"\u05d4\u05d5\u05e8\u05d3 \u05e9\u05de\u05e2 \u05db-3MP",incorrect_try_again:"\u05e9\u05d2\u05d5\u05d9. \u05e0\u05e1\u05d4 \u05e9\u05d5\u05d1.",
-image_alt_text:"\u05ea\u05de\u05d5\u05e0\u05ea \u05d0\u05ea\u05d2\u05e8 \u05e9\u05dc reCAPTCHA",privacy_and_terms:"\u05e4\u05e8\u05d8\u05d9\u05d5\u05ea \u05d5\u05ea\u05e0\u05d0\u05d9\u05dd"},pa={visual_challenge:"Obter um desafio visual",audio_challenge:"Obter um desafio de \u00e1udio",refresh_btn:"Obter um novo desafio",instructions_visual:"Digite o texto:",instructions_audio:"Digite o que voc\u00ea ouve:",help_btn:"Ajuda",play_again:"Reproduzir som novamente",cant_hear_this:"Fazer download do som no formato MP3",
-incorrect_try_again:"Incorreto. Tente novamente.",image_alt_text:"Imagem de desafio reCAPTCHA",privacy_and_terms:"Privacidade e Termos"},qa={visual_challenge:"Ob\u0163ine\u0163i un cod captcha vizual",audio_challenge:"Ob\u0163ine\u0163i un cod captcha audio",refresh_btn:"Ob\u0163ine\u0163i un nou cod captcha",instructions_visual:"Introduce\u021bi textul:",instructions_audio:"Introduce\u0163i ceea ce auzi\u0163i:",help_btn:"Ajutor",play_again:"Reda\u0163i sunetul din nou",cant_hear_this:"Desc\u0103rca\u0163i fi\u015fierul audio ca MP3",
-incorrect_try_again:"Incorect. \u00cencerca\u0163i din nou.",image_alt_text:"Imagine de verificare reCAPTCHA",privacy_and_terms:"Confiden\u0163ialitate \u015fi termeni"},ra={visual_challenge:"\u6536\u5230\u4e00\u4e2a\u89c6\u9891\u9080\u8bf7",audio_challenge:"\u6362\u4e00\u7ec4\u97f3\u9891\u9a8c\u8bc1\u7801",refresh_btn:"\u6362\u4e00\u7ec4\u9a8c\u8bc1\u7801",instructions_visual:"\u8f93\u5165\u6587\u5b57\uff1a",instructions_audio:"\u8bf7\u952e\u5165\u60a8\u542c\u5230\u7684\u5185\u5bb9\uff1a",help_btn:"\u5e2e\u52a9",
-play_again:"\u91cd\u65b0\u64ad\u653e",cant_hear_this:"\u4ee5 MP3 \u683c\u5f0f\u4e0b\u8f7d\u58f0\u97f3",incorrect_try_again:"\u4e0d\u6b63\u786e\uff0c\u8bf7\u91cd\u8bd5\u3002",image_alt_text:"reCAPTCHA \u9a8c\u8bc1\u56fe\u7247",privacy_and_terms:"\u9690\u79c1\u6743\u548c\u4f7f\u7528\u6761\u6b3e"},sa={en:w,af:{visual_challenge:"Kry 'n visuele verifi\u00ebring",audio_challenge:"Kry 'n klankverifi\u00ebring",refresh_btn:"Kry 'n nuwe verifi\u00ebring",instructions_visual:"",instructions_audio:"Tik wat jy hoor:",
-help_btn:"Hulp",play_again:"Speel geluid weer",cant_hear_this:"Laai die klank af as MP3",incorrect_try_again:"Verkeerd. Probeer weer.",image_alt_text:"reCAPTCHA-uitdagingprent",privacy_and_terms:"Privaatheid en bepalings"},am:{visual_challenge:"\u12e8\u12a5\u12ed\u1273 \u1270\u130b\u1323\u121a \u12a0\u130d\u129d",audio_challenge:"\u120c\u120b \u12a0\u12f2\u1235 \u12e8\u12f5\u121d\u133d \u1325\u12eb\u1244 \u12ed\u1245\u1228\u1265",refresh_btn:"\u120c\u120b \u12a0\u12f2\u1235 \u1325\u12eb\u1244 \u12ed\u1245\u1228\u1265",
-instructions_visual:"",instructions_audio:"\u12e8\u121d\u1275\u1230\u121b\u12cd\u1295 \u1270\u12ed\u1265\u1361-",help_btn:"\u12a5\u1308\u12db",play_again:"\u12f5\u121d\u1339\u1295 \u12a5\u1295\u12f0\u1308\u1293 \u12a0\u132b\u12cd\u1275",cant_hear_this:"\u12f5\u121d\u1339\u1295 \u1260MP3 \u1245\u122d\u133d \u12a0\u12cd\u122d\u12f5",incorrect_try_again:"\u1275\u12ad\u12ad\u120d \u12a0\u12ed\u12f0\u1208\u121d\u1362 \u12a5\u1295\u12f0\u1308\u1293 \u121e\u12ad\u122d\u1362",image_alt_text:"reCAPTCHA \u121d\u1235\u120d \u130d\u1320\u121d",
-privacy_and_terms:"\u130d\u120b\u12ca\u1290\u1275 \u12a5\u1293 \u12cd\u120d"},ar:ja,"ar-EG":ja,bg:{visual_challenge:"\u041f\u043e\u043b\u0443\u0447\u0430\u0432\u0430\u043d\u0435 \u043d\u0430 \u0432\u0438\u0437\u0443\u0430\u043b\u043d\u0430 \u043f\u0440\u043e\u0432\u0435\u0440\u043a\u0430",audio_challenge:"\u0417\u0430\u0440\u0435\u0436\u0434\u0430\u043d\u0435 \u043d\u0430 \u0430\u0443\u0434\u0438\u043e\u0442\u0435\u0441\u0442",refresh_btn:"\u0417\u0430\u0440\u0435\u0436\u0434\u0430\u043d\u0435 \u043d\u0430 \u043d\u043e\u0432 \u0442\u0435\u0441\u0442",
-instructions_visual:"\u0412\u044a\u0432\u0435\u0434\u0435\u0442\u0435 \u0442\u0435\u043a\u0441\u0442\u0430:",instructions_audio:"\u0412\u044a\u0432\u0435\u0434\u0435\u0442\u0435 \u0447\u0443\u0442\u043e\u0442\u043e:",help_btn:"\u041f\u043e\u043c\u043e\u0449",play_again:"\u041f\u043e\u0432\u0442\u043e\u0440\u043d\u043e \u043f\u0443\u0441\u043a\u0430\u043d\u0435 \u043d\u0430 \u0437\u0432\u0443\u043a\u0430",cant_hear_this:"\u0418\u0437\u0442\u0435\u0433\u043b\u044f\u043d\u0435 \u043d\u0430 \u0437\u0432\u0443\u043a\u0430 \u0432\u044a\u0432 \u0444\u043e\u0440\u043c\u0430\u0442 MP3",
-incorrect_try_again:"\u041d\u0435\u043f\u0440\u0430\u0432\u0438\u043b\u043d\u043e. \u041e\u043f\u0438\u0442\u0430\u0439\u0442\u0435 \u043e\u0442\u043d\u043e\u0432\u043e.",image_alt_text:"\u0418\u0437\u043e\u0431\u0440\u0430\u0436\u0435\u043d\u0438\u0435 \u043d\u0430 \u043f\u0440\u043e\u0432\u0435\u0440\u043a\u0430\u0442\u0430 \u0441 reCAPTCHA",privacy_and_terms:"\u041f\u043e\u0432\u0435\u0440\u0438\u0442\u0435\u043b\u043d\u043e\u0441\u0442 \u0438 \u041e\u0431\u0449\u0438 \u0443\u0441\u043b\u043e\u0432\u0438\u044f"},
-bn:{visual_challenge:"\u098f\u0995\u099f\u09bf \u09a6\u09c3\u09b6\u09cd\u09af\u09ae\u09be\u09a8 \u09aa\u09cd\u09b0\u09a4\u09bf\u09a6\u09cd\u09ac\u09a8\u09cd\u09a6\u09cd\u09ac\u09bf\u09a4\u09be \u09aa\u09be\u09a8",audio_challenge:"\u098f\u0995\u099f\u09bf \u0985\u09a1\u09bf\u0993 \u09aa\u09cd\u09b0\u09a4\u09bf\u09a6\u09cd\u09ac\u09a8\u09cd\u09a6\u09cd\u09ac\u09bf\u09a4\u09be  \u09aa\u09be\u09a8",refresh_btn:"\u098f\u0995\u099f\u09bf \u09a8\u09a4\u09c1\u09a8 \u09aa\u09cd\u09b0\u09a4\u09bf\u09a6\u09cd\u09ac\u09a8\u09cd\u09a6\u09cd\u09ac\u09bf\u09a4\u09be  \u09aa\u09be\u09a8",
-instructions_visual:"",instructions_audio:"\u0986\u09aa\u09a8\u09bf \u09af\u09be \u09b6\u09c1\u09a8\u099b\u09c7\u09a8 \u09a4\u09be \u09b2\u09bf\u0996\u09c1\u09a8:",help_btn:"\u09b8\u09b9\u09be\u09df\u09a4\u09be",play_again:"\u0986\u09ac\u09be\u09b0 \u09b8\u09be\u0989\u09a8\u09cd\u09a1 \u09aa\u09cd\u09b2\u09c7 \u0995\u09b0\u09c1\u09a8",cant_hear_this:"MP3 \u09b0\u09c2\u09aa\u09c7 \u09b6\u09ac\u09cd\u09a6 \u09a1\u09be\u0989\u09a8\u09b2\u09cb\u09a1 \u0995\u09b0\u09c1\u09a8",incorrect_try_again:"\u09ac\u09c7\u09a0\u09bf\u0995\u09f7 \u0986\u09ac\u09be\u09b0 \u099a\u09c7\u09b7\u09cd\u099f\u09be \u0995\u09b0\u09c1\u09a8\u09f7",
-image_alt_text:"reCAPTCHA \u099a\u09cd\u09af\u09be\u09b2\u09c7\u099e\u09cd\u099c \u099a\u09bf\u09a4\u09cd\u09b0",privacy_and_terms:"\u0997\u09cb\u09aa\u09a8\u09c0\u09af\u09bc\u09a4\u09be \u0993 \u09b6\u09b0\u09cd\u09a4\u09be\u09ac\u09b2\u09c0"},ca:{visual_challenge:"Obt\u00e9n un repte visual",audio_challenge:"Obteniu una pista sonora",refresh_btn:"Obteniu una pista nova",instructions_visual:"Escriviu el text:",instructions_audio:"Escriviu el que escolteu:",help_btn:"Ajuda",play_again:"Torna a reproduir el so",
-cant_hear_this:"Baixa el so com a MP3",incorrect_try_again:"No \u00e9s correcte. Torna-ho a provar.",image_alt_text:"Imatge del repte de reCAPTCHA",privacy_and_terms:"Privadesa i condicions"},cs:{visual_challenge:"Zobrazit vizu\u00e1ln\u00ed podobu v\u00fdrazu",audio_challenge:"P\u0159ehr\u00e1t zvukovou podobu v\u00fdrazu",refresh_btn:"Zobrazit nov\u00fd v\u00fdraz",instructions_visual:"Zadejte text:",instructions_audio:"Napi\u0161te, co jste sly\u0161eli:",help_btn:"N\u00e1pov\u011bda",play_again:"Znovu p\u0159ehr\u00e1t zvuk",
-cant_hear_this:"St\u00e1hnout zvuk ve form\u00e1tu MP3",incorrect_try_again:"\u0160patn\u011b. Zkuste to znovu.",image_alt_text:"Obr\u00e1zek reCAPTCHA",privacy_and_terms:"Ochrana soukrom\u00ed a smluvn\u00ed podm\u00ednky"},da:{visual_challenge:"Hent en visuel udfordring",audio_challenge:"Hent en lydudfordring",refresh_btn:"Hent en ny udfordring",instructions_visual:"Indtast teksten:",instructions_audio:"Indtast det, du h\u00f8rer:",help_btn:"Hj\u00e6lp",play_again:"Afspil lyden igen",cant_hear_this:"Download lyd som MP3",
-incorrect_try_again:"Forkert. Pr\u00f8v igen.",image_alt_text:"reCAPTCHA-udfordringsbillede",privacy_and_terms:"Privatliv og vilk\u00e5r"},de:{visual_challenge:"Captcha abrufen",audio_challenge:"Audio-Captcha abrufen",refresh_btn:"Neues Captcha abrufen",instructions_visual:"Geben Sie den angezeigten Text ein:",instructions_audio:"Geben Sie das Geh\u00f6rte ein:",help_btn:"Hilfe",play_again:"Wort erneut abspielen",cant_hear_this:"Wort als MP3 herunterladen",incorrect_try_again:"Falsch. Bitte versuchen Sie es erneut.",
-image_alt_text:"reCAPTCHA-Bild",privacy_and_terms:"Datenschutzerkl\u00e4rung & Nutzungsbedingungen"},el:{visual_challenge:"\u039f\u03c0\u03c4\u03b9\u03ba\u03ae \u03c0\u03c1\u03cc\u03ba\u03bb\u03b7\u03c3\u03b7",audio_challenge:"\u0397\u03c7\u03b7\u03c4\u03b9\u03ba\u03ae \u03c0\u03c1\u03cc\u03ba\u03bb\u03b7\u03c3\u03b7",refresh_btn:"\u039d\u03ad\u03b1 \u03c0\u03c1\u03cc\u03ba\u03bb\u03b7\u03c3\u03b7",instructions_visual:"\u03a0\u03bb\u03b7\u03ba\u03c4\u03c1\u03bf\u03bb\u03bf\u03b3\u03ae\u03c3\u03c4\u03b5 \u03c4\u03bf \u03ba\u03b5\u03af\u03bc\u03b5\u03bd\u03bf:",
-instructions_audio:"\u03a0\u03bb\u03b7\u03ba\u03c4\u03c1\u03bf\u03bb\u03bf\u03b3\u03ae\u03c3\u03c4\u03b5 \u03cc\u03c4\u03b9 \u03b1\u03ba\u03bf\u03cd\u03c4\u03b5:",help_btn:"\u0392\u03bf\u03ae\u03b8\u03b5\u03b9\u03b1",play_again:"\u0391\u03bd\u03b1\u03c0\u03b1\u03c1\u03b1\u03b3\u03c9\u03b3\u03ae \u03ae\u03c7\u03bf\u03c5 \u03be\u03b1\u03bd\u03ac",cant_hear_this:"\u039b\u03ae\u03c8\u03b7 \u03ae\u03c7\u03bf\u03c5 \u03c9\u03c2 \u039c\u03a13",incorrect_try_again:"\u039b\u03ac\u03b8\u03bf\u03c2. \u0394\u03bf\u03ba\u03b9\u03bc\u03ac\u03c3\u03c4\u03b5 \u03be\u03b1\u03bd\u03ac.",
-image_alt_text:"\u0395\u03b9\u03ba\u03cc\u03bd\u03b1 \u03c0\u03c1\u03cc\u03ba\u03bb\u03b7\u03c3\u03b7\u03c2 reCAPTCHA",privacy_and_terms:"\u0391\u03c0\u03cc\u03c1\u03c1\u03b7\u03c4\u03bf \u03ba\u03b1\u03b9 \u03cc\u03c1\u03bf\u03b9"},"en-GB":w,"en-US":w,es:ka,"es-419":{visual_challenge:"Enfrentar un desaf\u00edo visual",audio_challenge:"Enfrentar un desaf\u00edo de audio",refresh_btn:"Enfrentar un nuevo desaf\u00edo",instructions_visual:"Escriba el texto:",instructions_audio:"Escribe lo que escuchas:",
-help_btn:"Ayuda",play_again:"Reproducir sonido de nuevo",cant_hear_this:"Descargar sonido en formato MP3",incorrect_try_again:"Incorrecto. Vuelve a intentarlo.",image_alt_text:"Imagen del desaf\u00edo de la reCAPTCHA",privacy_and_terms:"Privacidad y condiciones"},"es-ES":ka,et:{visual_challenge:"Kuva kuvap\u00f5hine robotil\u00f5ks",audio_challenge:"Kuva helip\u00f5hine robotil\u00f5ks",refresh_btn:"Kuva uus robotil\u00f5ks",instructions_visual:"Tippige tekst:",instructions_audio:"Tippige, mida kuulete.",
-help_btn:"Abi",play_again:"Esita heli uuesti",cant_hear_this:"Laadi heli alla MP3-vormingus",incorrect_try_again:"Vale. Proovige uuesti.",image_alt_text:"reCAPTCHA robotil\u00f5ksu kujutis",privacy_and_terms:"Privaatsus ja tingimused"},eu:{visual_challenge:"Eskuratu ikusizko erronka",audio_challenge:"Eskuratu audio-erronka",refresh_btn:"Eskuratu erronka berria",instructions_visual:"",instructions_audio:"Idatzi entzuten duzuna:",help_btn:"Laguntza",play_again:"Erreproduzitu soinua berriro",cant_hear_this:"Deskargatu soinua MP3 gisa",
-incorrect_try_again:"Ez da zuzena. Saiatu berriro.",image_alt_text:"reCAPTCHA erronkaren irudia",privacy_and_terms:"Pribatutasuna eta baldintzak"},fa:{visual_challenge:"\u062f\u0631\u06cc\u0627\u0641\u062a \u06cc\u06a9 \u0645\u0639\u0645\u0627\u06cc \u062f\u06cc\u062f\u0627\u0631\u06cc",audio_challenge:"\u062f\u0631\u06cc\u0627\u0641\u062a \u06cc\u06a9 \u0645\u0639\u0645\u0627\u06cc \u0635\u0648\u062a\u06cc",refresh_btn:"\u062f\u0631\u06cc\u0627\u0641\u062a \u06cc\u06a9 \u0645\u0639\u0645\u0627\u06cc \u062c\u062f\u06cc\u062f",
-instructions_visual:"",instructions_audio:"\u0622\u0646\u0686\u0647 \u0631\u0627 \u06a9\u0647 \u0645\u06cc\u200c\u0634\u0646\u0648\u06cc\u062f \u062a\u0627\u06cc\u067e \u06a9\u0646\u06cc\u062f:",help_btn:"\u0631\u0627\u0647\u0646\u0645\u0627\u06cc\u06cc",play_again:"\u067e\u062e\u0634 \u0645\u062c\u062f\u062f \u0635\u062f\u0627",cant_hear_this:"\u062f\u0627\u0646\u0644\u0648\u062f \u0635\u062f\u0627 \u0628\u0647 \u0635\u0648\u0631\u062a MP3",incorrect_try_again:"\u0646\u0627\u062f\u0631\u0633\u062a. \u062f\u0648\u0628\u0627\u0631\u0647 \u0627\u0645\u062a\u062d\u0627\u0646 \u06a9\u0646\u06cc\u062f.",
-image_alt_text:"\u062a\u0635\u0648\u06cc\u0631 \u0686\u0627\u0644\u0634\u06cc reCAPTCHA",privacy_and_terms:"\u062d\u0631\u06cc\u0645 \u062e\u0635\u0648\u0635\u06cc \u0648 \u0634\u0631\u0627\u06cc\u0637"},fi:{visual_challenge:"Kuvavahvistus",audio_challenge:"\u00c4\u00e4nivahvistus",refresh_btn:"Uusi kuva",instructions_visual:"Kirjoita teksti:",instructions_audio:"Kirjoita kuulemasi:",help_btn:"Ohje",play_again:"Toista \u00e4\u00e4ni uudelleen",cant_hear_this:"Lataa \u00e4\u00e4ni MP3-tiedostona",
-incorrect_try_again:"V\u00e4\u00e4rin. Yrit\u00e4 uudelleen.",image_alt_text:"reCAPTCHA-kuva",privacy_and_terms:"Tietosuoja ja k\u00e4ytt\u00f6ehdot"},fil:la,fr:ma,"fr-CA":{visual_challenge:"Obtenir un test visuel",audio_challenge:"Obtenir un test audio",refresh_btn:"Obtenir un nouveau test",instructions_visual:"Saisissez le texte\u00a0:",instructions_audio:"Tapez ce que vous entendez\u00a0:",help_btn:"Aide",play_again:"Jouer le son de nouveau",cant_hear_this:"T\u00e9l\u00e9charger le son en format MP3",
-incorrect_try_again:"Erreur, essayez \u00e0 nouveau",image_alt_text:"Image reCAPTCHA",privacy_and_terms:"Confidentialit\u00e9 et conditions d'utilisation"},"fr-FR":ma,gl:{visual_challenge:"Obter unha proba visual",audio_challenge:"Obter unha proba de audio",refresh_btn:"Obter unha proba nova",instructions_visual:"",instructions_audio:"Escribe o que escoitas:",help_btn:"Axuda",play_again:"Reproducir o son de novo",cant_hear_this:"Descargar son como MP3",incorrect_try_again:"Incorrecto. T\u00e9ntao de novo.",
-image_alt_text:"Imaxe de proba de reCAPTCHA",privacy_and_terms:"Privacidade e condici\u00f3ns"},gu:{visual_challenge:"\u0a8f\u0a95 \u0aa6\u0ac3\u0ab6\u0acd\u0aaf\u0abe\u0aa4\u0acd\u0aae\u0a95 \u0aaa\u0aa1\u0a95\u0abe\u0ab0 \u0aae\u0ac7\u0ab3\u0ab5\u0acb",audio_challenge:"\u0a8f\u0a95 \u0a91\u0aa1\u0abf\u0a93 \u0aaa\u0aa1\u0a95\u0abe\u0ab0 \u0aae\u0ac7\u0ab3\u0ab5\u0acb",refresh_btn:"\u0a8f\u0a95 \u0aa8\u0ab5\u0acb \u0aaa\u0aa1\u0a95\u0abe\u0ab0 \u0aae\u0ac7\u0ab3\u0ab5\u0acb",instructions_visual:"",
-instructions_audio:"\u0aa4\u0aae\u0ac7 \u0a9c\u0ac7 \u0ab8\u0abe\u0a82\u0aad\u0ab3\u0acb \u0a9b\u0acb \u0aa4\u0ac7 \u0ab2\u0a96\u0acb:",help_btn:"\u0ab8\u0ab9\u0abe\u0aaf",play_again:"\u0aa7\u0acd\u0ab5\u0aa8\u0abf \u0aab\u0ab0\u0ac0\u0aa5\u0ac0 \u0a9a\u0ab2\u0abe\u0ab5\u0acb",cant_hear_this:"MP3 \u0aa4\u0ab0\u0ac0\u0a95\u0ac7 \u0aa7\u0acd\u0ab5\u0aa8\u0abf\u0aa8\u0ac7 \u0aa1\u0abe\u0a89\u0aa8\u0ab2\u0acb\u0aa1 \u0a95\u0ab0\u0acb",incorrect_try_again:"\u0a96\u0acb\u0a9f\u0ac1\u0a82. \u0aab\u0ab0\u0ac0 \u0aaa\u0acd\u0ab0\u0aaf\u0abe\u0ab8 \u0a95\u0ab0\u0acb.",
-image_alt_text:"reCAPTCHA \u0aaa\u0aa1\u0a95\u0abe\u0ab0 \u0a9b\u0aac\u0ac0",privacy_and_terms:"\u0a97\u0acb\u0aaa\u0aa8\u0ac0\u0aaf\u0aa4\u0abe \u0a85\u0aa8\u0ac7 \u0ab6\u0ab0\u0aa4\u0acb"},hi:{visual_challenge:"\u0915\u094b\u0908 \u0935\u093f\u091c\u0941\u0905\u0932 \u091a\u0941\u0928\u094c\u0924\u0940 \u0932\u0947\u0902",audio_challenge:"\u0915\u094b\u0908 \u0911\u0921\u093f\u092f\u094b \u091a\u0941\u0928\u094c\u0924\u0940 \u0932\u0947\u0902",refresh_btn:"\u0915\u094b\u0908 \u0928\u0908 \u091a\u0941\u0928\u094c\u0924\u0940 \u0932\u0947\u0902",
-instructions_visual:"\u091f\u0947\u0915\u094d\u0938\u094d\u091f \u091f\u093e\u0907\u092a \u0915\u0930\u0947\u0902:",instructions_audio:"\u091c\u094b \u0906\u092a \u0938\u0941\u0928 \u0930\u0939\u0947 \u0939\u0948\u0902 \u0909\u0938\u0947 \u0932\u093f\u0916\u0947\u0902:",help_btn:"\u0938\u0939\u093e\u092f\u0924\u093e",play_again:"\u0927\u094d\u200d\u0935\u0928\u093f \u092a\u0941\u0928: \u091a\u0932\u093e\u090f\u0902",cant_hear_this:"\u0927\u094d\u200d\u0935\u0928\u093f \u0915\u094b MP3 \u0915\u0947 \u0930\u0942\u092a \u092e\u0947\u0902 \u0921\u093e\u0909\u0928\u0932\u094b\u0921 \u0915\u0930\u0947\u0902",
-incorrect_try_again:"\u0917\u0932\u0924. \u092a\u0941\u0928: \u092a\u094d\u0930\u092f\u093e\u0938 \u0915\u0930\u0947\u0902.",image_alt_text:"reCAPTCHA \u091a\u0941\u0928\u094c\u0924\u0940 \u091a\u093f\u0924\u094d\u0930",privacy_and_terms:"\u0917\u094b\u092a\u0928\u0940\u092f\u0924\u093e \u0914\u0930 \u0936\u0930\u094d\u0924\u0947\u0902"},hr:{visual_challenge:"Dohvati vizualni upit",audio_challenge:"Dohvati zvu\u010dni upit",refresh_btn:"Dohvati novi upit",instructions_visual:"Unesite tekst:",instructions_audio:"Upi\u0161ite \u0161to \u010dujete:",
-help_btn:"Pomo\u0107",play_again:"Ponovi zvuk",cant_hear_this:"Preuzmi zvuk u MP3 formatu",incorrect_try_again:"Nije to\u010dno. Poku\u0161ajte ponovno.",image_alt_text:"Slikovni izazov reCAPTCHA",privacy_and_terms:"Privatnost i odredbe"},hu:{visual_challenge:"Vizu\u00e1lis kih\u00edv\u00e1s k\u00e9r\u00e9se",audio_challenge:"Hangkih\u00edv\u00e1s k\u00e9r\u00e9se",refresh_btn:"\u00daj kih\u00edv\u00e1s k\u00e9r\u00e9se",instructions_visual:"\u00cdrja be a sz\u00f6veget:",instructions_audio:"\u00cdrja le, amit hall:",
-help_btn:"S\u00fag\u00f3",play_again:"Hang ism\u00e9telt lej\u00e1tsz\u00e1sa",cant_hear_this:"Hang let\u00f6lt\u00e9se MP3 form\u00e1tumban",incorrect_try_again:"Hib\u00e1s. Pr\u00f3b\u00e1lkozzon \u00fajra.",image_alt_text:"reCAPTCHA ellen\u0151rz\u0151 k\u00e9p",privacy_and_terms:"Adatv\u00e9delem \u00e9s Szerz\u0151d\u00e9si Felt\u00e9telek"},hy:{visual_challenge:"\u054d\u057f\u0561\u0576\u0561\u056c \u057f\u0565\u057d\u0578\u0572\u0561\u056f\u0561\u0576 \u056d\u0576\u0564\u056b\u0580",audio_challenge:"\u054d\u057f\u0561\u0576\u0561\u056c \u0571\u0561\u0575\u0576\u0561\u0575\u056b\u0576 \u056d\u0576\u0564\u056b\u0580",
-refresh_btn:"\u054d\u057f\u0561\u0576\u0561\u056c \u0576\u0578\u0580 \u056d\u0576\u0564\u056b\u0580",instructions_visual:"\u0544\u0578\u0582\u057f\u0584\u0561\u0563\u0580\u0565\u0584 \u057f\u0565\u0584\u057d\u057f\u0568\u055d",instructions_audio:"\u0544\u0578\u0582\u057f\u0584\u0561\u0563\u0580\u0565\u0584 \u0561\u0575\u0576, \u056b\u0576\u0579 \u056c\u057d\u0578\u0582\u0574 \u0565\u0584\u055d",help_btn:"\u0555\u0563\u0576\u0578\u0582\u0569\u0575\u0578\u0582\u0576",play_again:"\u0546\u057e\u0561\u0563\u0561\u0580\u056f\u0565\u056c \u0571\u0561\u0575\u0576\u0568 \u056f\u0580\u056f\u056b\u0576",
-cant_hear_this:"\u0532\u0565\u057c\u0576\u0565\u056c \u0571\u0561\u0575\u0576\u0568 \u0578\u0580\u057a\u0565\u057d MP3",incorrect_try_again:"\u054d\u056d\u0561\u056c \u0567: \u0553\u0578\u0580\u0571\u0565\u0584 \u056f\u0580\u056f\u056b\u0576:",image_alt_text:"reCAPTCHA \u057a\u0561\u057f\u056f\u0565\u0580\u0578\u057e \u056d\u0576\u0564\u056b\u0580",privacy_and_terms:"\u0533\u0561\u0572\u057f\u0576\u056b\u0578\u0582\u0569\u0575\u0561\u0576 & \u057a\u0561\u0575\u0574\u0561\u0576\u0576\u0565\u0580"},
-id:na,is:{visual_challenge:"F\u00e1 a\u00f0gangspr\u00f3f sem mynd",audio_challenge:"F\u00e1 a\u00f0gangspr\u00f3f sem hlj\u00f3\u00f0skr\u00e1",refresh_btn:"F\u00e1 n\u00fdtt a\u00f0gangspr\u00f3f",instructions_visual:"",instructions_audio:"Sl\u00e1\u00f0u inn \u00fea\u00f0 sem \u00fe\u00fa heyrir:",help_btn:"Hj\u00e1lp",play_again:"Spila hlj\u00f3\u00f0 aftur",cant_hear_this:"S\u00e6kja hlj\u00f3\u00f0 sem MP3",incorrect_try_again:"Rangt. Reyndu aftur.",image_alt_text:"mynd reCAPTCHA a\u00f0gangspr\u00f3fs",
-privacy_and_terms:"Pers\u00f3nuvernd og skilm\u00e1lar"},it:{visual_challenge:"Verifica visiva",audio_challenge:"Verifica audio",refresh_btn:"Nuova verifica",instructions_visual:"Digita il testo:",instructions_audio:"Digita ci\u00f2 che senti:",help_btn:"Guida",play_again:"Riproduci di nuovo audio",cant_hear_this:"Scarica audio in MP3",incorrect_try_again:"Sbagliato. Riprova.",image_alt_text:"Immagine di verifica reCAPTCHA",privacy_and_terms:"Privacy e Termini"},iw:oa,ja:{visual_challenge:"\u753b\u50cf\u3067\u78ba\u8a8d\u3057\u307e\u3059",
-audio_challenge:"\u97f3\u58f0\u3067\u78ba\u8a8d\u3057\u307e\u3059",refresh_btn:"\u5225\u306e\u5358\u8a9e\u3067\u3084\u308a\u76f4\u3057\u307e\u3059",instructions_visual:"\u30c6\u30ad\u30b9\u30c8\u3092\u5165\u529b:",instructions_audio:"\u805e\u3053\u3048\u305f\u5358\u8a9e\u3092\u5165\u529b\u3057\u307e\u3059:",help_btn:"\u30d8\u30eb\u30d7",play_again:"\u3082\u3046\u4e00\u5ea6\u805e\u304f",cant_hear_this:"MP3 \u3067\u97f3\u58f0\u3092\u30c0\u30a6\u30f3\u30ed\u30fc\u30c9",incorrect_try_again:"\u6b63\u3057\u304f\u3042\u308a\u307e\u305b\u3093\u3002\u3082\u3046\u4e00\u5ea6\u3084\u308a\u76f4\u3057\u3066\u304f\u3060\u3055\u3044\u3002",
-image_alt_text:"reCAPTCHA \u78ba\u8a8d\u7528\u753b\u50cf",privacy_and_terms:"\u30d7\u30e9\u30a4\u30d0\u30b7\u30fc\u3068\u5229\u7528\u898f\u7d04"},kn:{visual_challenge:"\u0ca6\u0cc3\u0cb6\u0ccd\u0caf \u0cb8\u0cb5\u0cbe\u0cb2\u0cca\u0c82\u0ca6\u0ca8\u0ccd\u0ca8\u0cc1 \u0cb8\u0ccd\u0cb5\u0cc0\u0c95\u0cb0\u0cbf\u0cb8\u0cbf",audio_challenge:"\u0c86\u0ca1\u0cbf\u0caf\u0ccb \u0cb8\u0cb5\u0cbe\u0cb2\u0cca\u0c82\u0ca6\u0ca8\u0ccd\u0ca8\u0cc1 \u0cb8\u0ccd\u0cb5\u0cc0\u0c95\u0cb0\u0cbf\u0cb8\u0cbf",refresh_btn:"\u0cb9\u0cca\u0cb8 \u0cb8\u0cb5\u0cbe\u0cb2\u0cca\u0c82\u0ca6\u0ca8\u0ccd\u0ca8\u0cc1 \u0caa\u0ca1\u0cc6\u0caf\u0cbf\u0cb0\u0cbf",
-instructions_visual:"",instructions_audio:"\u0ca8\u0cbf\u0cae\u0c97\u0cc6 \u0c95\u0cc7\u0cb3\u0cbf\u0cb8\u0cc1\u0cb5\u0cc1\u0ca6\u0ca8\u0ccd\u0ca8\u0cc1 \u0c9f\u0cc8\u0caa\u0ccd\u200c \u0cae\u0cbe\u0ca1\u0cbf:",help_btn:"\u0cb8\u0cb9\u0cbe\u0caf",play_again:"\u0ca7\u0ccd\u0cb5\u0ca8\u0cbf\u0caf\u0ca8\u0ccd\u0ca8\u0cc1 \u0cae\u0ca4\u0ccd\u0ca4\u0cc6 \u0caa\u0ccd\u0cb2\u0cc7 \u0cae\u0cbe\u0ca1\u0cbf",cant_hear_this:"\u0ca7\u0ccd\u0cb5\u0ca8\u0cbf\u0caf\u0ca8\u0ccd\u0ca8\u0cc1 MP3 \u0cb0\u0cc2\u0caa\u0ca6\u0cb2\u0ccd\u0cb2\u0cbf \u0ca1\u0ccc\u0ca8\u0ccd\u200c\u0cb2\u0ccb\u0ca1\u0ccd \u0cae\u0cbe\u0ca1\u0cbf",
-incorrect_try_again:"\u0ca4\u0caa\u0ccd\u0caa\u0cbe\u0c97\u0cbf\u0ca6\u0cc6. \u0cae\u0ca4\u0ccd\u0ca4\u0cca\u0cae\u0ccd\u0cae\u0cc6 \u0caa\u0ccd\u0cb0\u0caf\u0ca4\u0ccd\u0ca8\u0cbf\u0cb8\u0cbf.",image_alt_text:"reCAPTCHA \u0cb8\u0cb5\u0cbe\u0cb2\u0cc1 \u0c9a\u0cbf\u0ca4\u0ccd\u0cb0",privacy_and_terms:"\u0c97\u0ccc\u0caa\u0ccd\u0caf\u0ca4\u0cc6 \u0cae\u0ca4\u0ccd\u0ca4\u0cc1 \u0ca8\u0cbf\u0caf\u0cae\u0c97\u0cb3\u0cc1"},ko:{visual_challenge:"\uadf8\ub9bc\uc73c\ub85c \ubcf4\uc548\ubb38\uc790 \ubc1b\uae30",
-audio_challenge:"\uc74c\uc131\uc73c\ub85c \ubcf4\uc548\ubb38\uc790 \ubc1b\uae30",refresh_btn:"\ubcf4\uc548\ubb38\uc790 \uc0c8\ub85c \ubc1b\uae30",instructions_visual:"\ud14d\uc2a4\ud2b8 \uc785\ub825:",instructions_audio:"\uc74c\uc131 \ubcf4\uc548\ubb38\uc790 \uc785\ub825:",help_btn:"\ub3c4\uc6c0\ub9d0",play_again:"\uc74c\uc131 \ub2e4\uc2dc \ub4e3\uae30",cant_hear_this:"\uc74c\uc131\uc744 MP3\ub85c \ub2e4\uc6b4\ub85c\ub4dc",incorrect_try_again:"\ud2c0\ub838\uc2b5\ub2c8\ub2e4. \ub2e4\uc2dc \uc2dc\ub3c4\ud574 \uc8fc\uc138\uc694.",
-image_alt_text:"reCAPTCHA \ubcf4\uc548\ubb38\uc790 \uc774\ubbf8\uc9c0",privacy_and_terms:"\uac1c\uc778\uc815\ubcf4 \ubcf4\ud638 \ubc0f \uc57d\uad00"},ln:ma,lt:{visual_challenge:"Gauti vaizdin\u012f atpa\u017einimo test\u0105",audio_challenge:"Gauti garso atpa\u017einimo test\u0105",refresh_btn:"Gauti nauj\u0105 atpa\u017einimo test\u0105",instructions_visual:"\u012eveskite tekst\u0105:",instructions_audio:"\u012eveskite tai, k\u0105 girdite:",help_btn:"Pagalba",play_again:"Dar kart\u0105 paleisti gars\u0105",
-cant_hear_this:"Atsisi\u0173sti gars\u0105 kaip MP3",incorrect_try_again:"Neteisingai. Bandykite dar kart\u0105.",image_alt_text:"Testo \u201ereCAPTCHA\u201c vaizdas",privacy_and_terms:"Privatumas ir s\u0105lygos"},lv:{visual_challenge:"Sa\u0146emt vizu\u0101lu izaicin\u0101jumu",audio_challenge:"Sa\u0146emt audio izaicin\u0101jumu",refresh_btn:"Sa\u0146emt jaunu izaicin\u0101jumu",instructions_visual:"Ievadiet tekstu:",instructions_audio:"Ierakstiet dzirdamo:",help_btn:"Pal\u012bdz\u012bba",play_again:"V\u0113lreiz atska\u0146ot ska\u0146u",
-cant_hear_this:"Lejupiel\u0101d\u0113t ska\u0146u MP3\u00a0form\u0101t\u0101",incorrect_try_again:"Nepareizi. M\u0113\u0123iniet v\u0113lreiz.",image_alt_text:"reCAPTCHA izaicin\u0101juma att\u0113ls",privacy_and_terms:"Konfidencialit\u0101te un noteikumi"},ml:{visual_challenge:"\u0d12\u0d30\u0d41 \u0d26\u0d43\u0d36\u0d4d\u0d2f \u0d1a\u0d32\u0d1e\u0d4d\u0d1a\u0d4d \u0d28\u0d47\u0d1f\u0d41\u0d15",audio_challenge:"\u0d12\u0d30\u0d41 \u0d13\u0d21\u0d3f\u0d2f\u0d4b \u0d1a\u0d32\u0d1e\u0d4d\u0d1a\u0d4d \u0d28\u0d47\u0d1f\u0d41\u0d15",
-refresh_btn:"\u0d12\u0d30\u0d41 \u0d2a\u0d41\u0d24\u0d3f\u0d2f \u0d1a\u0d32\u0d1e\u0d4d\u0d1a\u0d4d \u0d28\u0d47\u0d1f\u0d41\u0d15",instructions_visual:"",instructions_audio:"\u0d15\u0d47\u0d7e\u0d15\u0d4d\u0d15\u0d41\u0d28\u0d4d\u0d28\u0d24\u0d4d \u0d1f\u0d48\u0d2a\u0d4d\u0d2a\u0d4d \u0d1a\u0d46\u0d2f\u0d4d\u0d2f\u0d42:",help_btn:"\u0d38\u0d39\u0d3e\u0d2f\u0d02",play_again:"\u0d36\u0d2c\u0d4d\u200c\u0d26\u0d02 \u0d35\u0d40\u0d23\u0d4d\u0d1f\u0d41\u0d02 \u0d2a\u0d4d\u0d32\u0d47 \u0d1a\u0d46\u0d2f\u0d4d\u0d2f\u0d41\u0d15",
-cant_hear_this:"\u0d36\u0d2c\u0d4d\u200c\u0d26\u0d02 MP3 \u0d06\u0d2f\u0d3f \u0d21\u0d57\u0d7a\u0d32\u0d4b\u0d21\u0d4d \u0d1a\u0d46\u0d2f\u0d4d\u0d2f\u0d41\u0d15",incorrect_try_again:"\u0d24\u0d46\u0d31\u0d4d\u0d31\u0d3e\u0d23\u0d4d. \u0d35\u0d40\u0d23\u0d4d\u0d1f\u0d41\u0d02 \u0d36\u0d4d\u0d30\u0d2e\u0d3f\u0d15\u0d4d\u0d15\u0d41\u0d15.",image_alt_text:"reCAPTCHA \u0d1a\u0d32\u0d1e\u0d4d\u0d1a\u0d4d \u0d07\u0d2e\u0d47\u0d1c\u0d4d",privacy_and_terms:"\u0d38\u0d4d\u0d35\u0d15\u0d3e\u0d30\u0d4d\u0d2f\u0d24\u0d2f\u0d41\u0d02 \u0d28\u0d3f\u0d2c\u0d28\u0d4d\u0d27\u0d28\u0d15\u0d33\u0d41\u0d02"},
-mr:{visual_challenge:"\u0926\u0943\u0936\u094d\u200d\u092f\u092e\u093e\u0928 \u0906\u0935\u094d\u0939\u093e\u0928 \u092a\u094d\u0930\u093e\u092a\u094d\u0924 \u0915\u0930\u093e",audio_challenge:"\u0911\u0921\u0940\u0913 \u0906\u0935\u094d\u0939\u093e\u0928 \u092a\u094d\u0930\u093e\u092a\u094d\u0924 \u0915\u0930\u093e",refresh_btn:"\u090f\u0915 \u0928\u0935\u0940\u0928 \u0906\u0935\u094d\u0939\u093e\u0928 \u092a\u094d\u0930\u093e\u092a\u094d\u0924 \u0915\u0930\u093e",instructions_visual:"",instructions_audio:"\u0906\u092a\u0932\u094d\u092f\u093e\u0932\u093e \u091c\u0947 \u0910\u0915\u0942 \u092f\u0947\u0908\u0932 \u0924\u0947 \u091f\u093e\u0907\u092a \u0915\u0930\u093e:",
-help_btn:"\u092e\u0926\u0924",play_again:"\u0927\u094d\u200d\u0935\u0928\u0940 \u092a\u0941\u0928\u094d\u0939\u093e \u092a\u094d\u200d\u0932\u0947 \u0915\u0930\u093e",cant_hear_this:"MP3 \u0930\u0941\u092a\u093e\u0924 \u0927\u094d\u200d\u0935\u0928\u0940 \u0921\u093e\u0909\u0928\u0932\u094b\u0921 \u0915\u0930\u093e",incorrect_try_again:"\u0905\u092f\u094b\u0917\u094d\u200d\u092f. \u092a\u0941\u0928\u094d\u200d\u0939\u093e \u092a\u094d\u0930\u092f\u0924\u094d\u200d\u0928 \u0915\u0930\u093e.",image_alt_text:"reCAPTCHA \u0906\u0935\u094d\u200d\u0939\u093e\u0928 \u092a\u094d\u0930\u0924\u093f\u092e\u093e",
-privacy_and_terms:"\u0917\u094b\u092a\u0928\u0940\u092f\u0924\u093e \u0906\u0923\u093f \u0905\u091f\u0940"},ms:{visual_challenge:"Dapatkan cabaran visual",audio_challenge:"Dapatkan cabaran audio",refresh_btn:"Dapatkan cabaran baru",instructions_visual:"Taipkan teksnya:",instructions_audio:"Taip apa yang didengari:",help_btn:"Bantuan",play_again:"Mainkan bunyi sekali lagi",cant_hear_this:"Muat turun bunyi sebagai MP3",incorrect_try_again:"Tidak betul. Cuba lagi.",image_alt_text:"Imej cabaran reCAPTCHA",
-privacy_and_terms:"Privasi & Syarat"},nl:{visual_challenge:"Een visuele uitdaging proberen",audio_challenge:"Een audio-uitdaging proberen",refresh_btn:"Een nieuwe uitdaging proberen",instructions_visual:"Typ de tekst:",instructions_audio:"Typ wat u hoort:",help_btn:"Help",play_again:"Geluid opnieuw afspelen",cant_hear_this:"Geluid downloaden als MP3",incorrect_try_again:"Onjuist. Probeer het opnieuw.",image_alt_text:"reCAPTCHA-uitdagingsafbeelding",privacy_and_terms:"Privacy en voorwaarden"},no:{visual_challenge:"F\u00e5 en bildeutfordring",
-audio_challenge:"F\u00e5 en lydutfordring",refresh_btn:"F\u00e5 en ny utfordring",instructions_visual:"Skriv inn teksten:",instructions_audio:"Skriv inn det du h\u00f8rer:",help_btn:"Hjelp",play_again:"Spill av lyd p\u00e5 nytt",cant_hear_this:"Last ned lyd som MP3",incorrect_try_again:"Feil. Pr\u00f8v p\u00e5 nytt.",image_alt_text:"reCAPTCHA-utfordringsbilde",privacy_and_terms:"Personvern og vilk\u00e5r"},pl:{visual_challenge:"Poka\u017c podpowied\u017a wizualn\u0105",audio_challenge:"Odtw\u00f3rz podpowied\u017a d\u017awi\u0119kow\u0105",
-refresh_btn:"Nowa podpowied\u017a",instructions_visual:"Przepisz tekst:",instructions_audio:"Wpisz us\u0142yszane s\u0142owa:",help_btn:"Pomoc",play_again:"Odtw\u00f3rz d\u017awi\u0119k ponownie",cant_hear_this:"Pobierz d\u017awi\u0119k jako plik MP3",incorrect_try_again:"Nieprawid\u0142owo. Spr\u00f3buj ponownie.",image_alt_text:"Zadanie obrazkowe reCAPTCHA",privacy_and_terms:"Prywatno\u015b\u0107 i warunki"},pt:pa,"pt-BR":pa,"pt-PT":{visual_challenge:"Obter um desafio visual",audio_challenge:"Obter um desafio de \u00e1udio",
-refresh_btn:"Obter um novo desafio",instructions_visual:"Introduza o texto:",instructions_audio:"Escreva o que ouvir:",help_btn:"Ajuda",play_again:"Reproduzir som novamente",cant_hear_this:"Transferir som como MP3",incorrect_try_again:"Incorreto. Tente novamente.",image_alt_text:"Imagem de teste reCAPTCHA",privacy_and_terms:"Privacidade e Termos de Utiliza\u00e7\u00e3o"},ro:qa,ru:{visual_challenge:"\u0412\u0438\u0437\u0443\u0430\u043b\u044c\u043d\u0430\u044f \u043f\u0440\u043e\u0432\u0435\u0440\u043a\u0430",
-audio_challenge:"\u0417\u0432\u0443\u043a\u043e\u0432\u0430\u044f \u043f\u0440\u043e\u0432\u0435\u0440\u043a\u0430",refresh_btn:"\u041e\u0431\u043d\u043e\u0432\u0438\u0442\u044c",instructions_visual:"\u0412\u0432\u0435\u0434\u0438\u0442\u0435 \u0442\u0435\u043a\u0441\u0442:",instructions_audio:"\u0412\u0432\u0435\u0434\u0438\u0442\u0435 \u0442\u043e, \u0447\u0442\u043e \u0441\u043b\u044b\u0448\u0438\u0442\u0435:",help_btn:"\u0421\u043f\u0440\u0430\u0432\u043a\u0430",play_again:"\u041f\u0440\u043e\u0441\u043b\u0443\u0448\u0430\u0442\u044c \u0435\u0449\u0435 \u0440\u0430\u0437",
-cant_hear_this:"\u0417\u0430\u0433\u0440\u0443\u0437\u0438\u0442\u044c MP3-\u0444\u0430\u0439\u043b",incorrect_try_again:"\u041d\u0435\u043f\u0440\u0430\u0432\u0438\u043b\u044c\u043d\u043e. \u041f\u043e\u0432\u0442\u043e\u0440\u0438\u0442\u0435 \u043f\u043e\u043f\u044b\u0442\u043a\u0443.",image_alt_text:"\u041f\u0440\u043e\u0432\u0435\u0440\u043a\u0430 \u043f\u043e \u0441\u043b\u043e\u0432\u0443 reCAPTCHA",privacy_and_terms:"\u041f\u0440\u0430\u0432\u0438\u043b\u0430 \u0438 \u043f\u0440\u0438\u043d\u0446\u0438\u043f\u044b"},
-sk:{visual_challenge:"Zobrazi\u0165 vizu\u00e1lnu podobu",audio_challenge:"Prehra\u0165 zvukov\u00fa podobu",refresh_btn:"Zobrazi\u0165 nov\u00fd v\u00fdraz",instructions_visual:"Zadajte text:",instructions_audio:"Zadajte, \u010do po\u010dujete:",help_btn:"Pomocn\u00edk",play_again:"Znova prehra\u0165 zvuk",cant_hear_this:"Prevzia\u0165 zvuk v podobe s\u00faboru MP3",incorrect_try_again:"Nespr\u00e1vne. Sk\u00faste to znova.",image_alt_text:"Obr\u00e1zok zadania reCAPTCHA",privacy_and_terms:"Ochrana osobn\u00fdch \u00fadajov a Zmluvn\u00e9 podmienky"},
-sl:{visual_challenge:"Vizualni preskus",audio_challenge:"Zvo\u010dni preskus",refresh_btn:"Nov preskus",instructions_visual:"Vnesite besedilo:",instructions_audio:"Natipkajte, kaj sli\u0161ite:",help_btn:"Pomo\u010d",play_again:"Znova predvajaj zvok",cant_hear_this:"Prenesi zvok kot MP3",incorrect_try_again:"Napa\u010dno. Poskusite znova.",image_alt_text:"Slika izziva reCAPTCHA",privacy_and_terms:"Zasebnost in pogoji"},sr:{visual_challenge:"\u041f\u0440\u0438\u043c\u0438\u0442\u0435 \u0432\u0438\u0437\u0443\u0435\u043b\u043d\u0438 \u0443\u043f\u0438\u0442",
-audio_challenge:"\u041f\u0440\u0438\u043c\u0438\u0442\u0435 \u0430\u0443\u0434\u0438\u043e \u0443\u043f\u0438\u0442",refresh_btn:"\u041f\u0440\u0438\u043c\u0438\u0442\u0435 \u043d\u043e\u0432\u0438 \u0443\u043f\u0438\u0442",instructions_visual:"\u0423\u043d\u0435\u0441\u0438\u0442\u0435 \u0442\u0435\u043a\u0441\u0442:",instructions_audio:"\u041e\u0442\u043a\u0443\u0446\u0430\u0458\u0442\u0435 \u043e\u043d\u043e \u0448\u0442\u043e \u0447\u0443\u0458\u0435\u0442\u0435:",help_btn:"\u041f\u043e\u043c\u043e\u045b",
-play_again:"\u041f\u043e\u043d\u043e\u0432\u043e \u043f\u0443\u0441\u0442\u0438 \u0437\u0432\u0443\u043a",cant_hear_this:"\u041f\u0440\u0435\u0443\u0437\u043c\u0438 \u0437\u0432\u0443\u043a \u043a\u0430\u043e MP3 \u0441\u043d\u0438\u043c\u0430\u043a",incorrect_try_again:"\u041d\u0435\u0442\u0430\u0447\u043d\u043e. \u041f\u043e\u043a\u0443\u0448\u0430\u0458\u0442\u0435 \u043f\u043e\u043d\u043e\u0432\u043e.",image_alt_text:"\u0421\u043b\u0438\u043a\u0430 reCAPTCHA \u043f\u0440\u043e\u0432\u0435\u0440\u0435",
-privacy_and_terms:"\u041f\u0440\u0438\u0432\u0430\u0442\u043d\u043e\u0441\u0442 \u0438 \u0443\u0441\u043b\u043e\u0432\u0438"},sv:{visual_challenge:"H\u00e4mta captcha i bildformat",audio_challenge:"H\u00e4mta captcha i ljudformat",refresh_btn:"H\u00e4mta ny captcha",instructions_visual:"Skriv texten:",instructions_audio:"Skriv det du h\u00f6r:",help_btn:"Hj\u00e4lp",play_again:"Spela upp ljudet igen",cant_hear_this:"H\u00e4mta ljud som MP3",incorrect_try_again:"Fel. F\u00f6rs\u00f6k igen.",image_alt_text:"reCAPTCHA-bild",
-privacy_and_terms:"Sekretess och villkor"},sw:{visual_challenge:"Pata herufi za kusoma",audio_challenge:"Pata herufi za kusikiliza",refresh_btn:"Pata herufi mpya",instructions_visual:"",instructions_audio:"Charaza unachosikia:",help_btn:"Usaidizi",play_again:"Cheza sauti tena",cant_hear_this:"Pakua sauti kama MP3",incorrect_try_again:"Sio sahihi. Jaribu tena.",image_alt_text:"picha ya changamoto ya reCAPTCHA",privacy_and_terms:"Faragha & Masharti"},ta:{visual_challenge:"\u0baa\u0bbe\u0bb0\u0bcd\u0bb5\u0bc8 \u0b9a\u0bc7\u0bb2\u0b9e\u0bcd\u0b9a\u0bc8\u0baa\u0bcd \u0baa\u0bc6\u0bb1\u0bc1\u0b95",
-audio_challenge:"\u0b86\u0b9f\u0bbf\u0baf\u0bcb \u0b9a\u0bc7\u0bb2\u0b9e\u0bcd\u0b9a\u0bc8\u0baa\u0bcd \u0baa\u0bc6\u0bb1\u0bc1\u0b95",refresh_btn:"\u0baa\u0bc1\u0ba4\u0bbf\u0baf \u0b9a\u0bc7\u0bb2\u0b9e\u0bcd\u0b9a\u0bc8\u0baa\u0bcd \u0baa\u0bc6\u0bb1\u0bc1\u0b95",instructions_visual:"",instructions_audio:"\u0b95\u0bc7\u0b9f\u0bcd\u0baa\u0ba4\u0bc8 \u0b9f\u0bc8\u0baa\u0bcd \u0b9a\u0bc6\u0baf\u0bcd\u0b95:",help_btn:"\u0b89\u0ba4\u0bb5\u0bbf",play_again:"\u0b92\u0bb2\u0bbf\u0baf\u0bc8 \u0bae\u0bc0\u0ba3\u0bcd\u0b9f\u0bc1\u0bae\u0bcd \u0b87\u0baf\u0b95\u0bcd\u0b95\u0bc1",
-cant_hear_this:"\u0b92\u0bb2\u0bbf\u0baf\u0bc8 MP3 \u0b86\u0b95 \u0baa\u0ba4\u0bbf\u0bb5\u0bbf\u0bb1\u0b95\u0bcd\u0b95\u0bc1\u0b95",incorrect_try_again:"\u0ba4\u0bb5\u0bb1\u0bbe\u0ba9\u0ba4\u0bc1. \u0bae\u0bc0\u0ba3\u0bcd\u0b9f\u0bc1\u0bae\u0bcd \u0bae\u0bc1\u0baf\u0bb2\u0bb5\u0bc1\u0bae\u0bcd.",image_alt_text:"reCAPTCHA \u0b9a\u0bc7\u0bb2\u0b9e\u0bcd\u0b9a\u0bcd \u0baa\u0b9f\u0bae\u0bcd",privacy_and_terms:"\u0ba4\u0ba9\u0bbf\u0baf\u0bc1\u0bb0\u0bbf\u0bae\u0bc8 & \u0bb5\u0bbf\u0ba4\u0bbf\u0bae\u0bc1\u0bb1\u0bc8\u0b95\u0bb3\u0bcd"},
-te:{visual_challenge:"\u0c12\u0c15 \u0c26\u0c43\u0c36\u0c4d\u0c2f\u0c2e\u0c3e\u0c28 \u0c38\u0c35\u0c3e\u0c32\u0c41\u0c28\u0c41 \u0c38\u0c4d\u0c35\u0c40\u0c15\u0c30\u0c3f\u0c02\u0c1a\u0c02\u0c21\u0c3f",audio_challenge:"\u0c12\u0c15 \u0c06\u0c21\u0c3f\u0c2f\u0c4b \u0c38\u0c35\u0c3e\u0c32\u0c41\u0c28\u0c41 \u0c38\u0c4d\u0c35\u0c40\u0c15\u0c30\u0c3f\u0c02\u0c1a\u0c02\u0c21\u0c3f",refresh_btn:"\u0c15\u0c4d\u0c30\u0c4a\u0c24\u0c4d\u0c24 \u0c38\u0c35\u0c3e\u0c32\u0c41\u0c28\u0c41 \u0c38\u0c4d\u0c35\u0c40\u0c15\u0c30\u0c3f\u0c02\u0c1a\u0c02\u0c21\u0c3f",
-instructions_visual:"",instructions_audio:"\u0c2e\u0c40\u0c30\u0c41 \u0c35\u0c3f\u0c28\u0c4d\u0c28\u0c26\u0c3f \u0c1f\u0c48\u0c2a\u0c4d \u0c1a\u0c47\u0c2f\u0c02\u0c21\u0c3f:",help_btn:"\u0c38\u0c39\u0c3e\u0c2f\u0c02",play_again:"\u0c27\u0c4d\u0c35\u0c28\u0c3f\u0c28\u0c3f \u0c2e\u0c33\u0c4d\u0c32\u0c40 \u0c2a\u0c4d\u0c32\u0c47 \u0c1a\u0c47\u0c2f\u0c3f",cant_hear_this:"\u0c27\u0c4d\u0c35\u0c28\u0c3f\u0c28\u0c3f MP3 \u0c35\u0c32\u0c46 \u0c21\u0c4c\u0c28\u0c4d\u200c\u0c32\u0c4b\u0c21\u0c4d \u0c1a\u0c47\u0c2f\u0c3f",
-incorrect_try_again:"\u0c24\u0c2a\u0c4d\u0c2a\u0c41. \u0c2e\u0c33\u0c4d\u0c32\u0c40 \u0c2a\u0c4d\u0c30\u0c2f\u0c24\u0c4d\u0c28\u0c3f\u0c02\u0c1a\u0c02\u0c21\u0c3f.",image_alt_text:"reCAPTCHA \u0c38\u0c35\u0c3e\u0c32\u0c41 \u0c1a\u0c3f\u0c24\u0c4d\u0c30\u0c02",privacy_and_terms:"\u0c17\u0c4b\u0c2a\u0c4d\u0c2f\u0c24 & \u0c28\u0c3f\u0c2c\u0c02\u0c27\u0c28\u0c32\u0c41"},th:{visual_challenge:"\u0e23\u0e31\u0e1a\u0e04\u0e27\u0e32\u0e21\u0e17\u0e49\u0e32\u0e17\u0e32\u0e22\u0e14\u0e49\u0e32\u0e19\u0e20\u0e32\u0e1e",
-audio_challenge:"\u0e23\u0e31\u0e1a\u0e04\u0e27\u0e32\u0e21\u0e17\u0e49\u0e32\u0e17\u0e32\u0e22\u0e14\u0e49\u0e32\u0e19\u0e40\u0e2a\u0e35\u0e22\u0e07",refresh_btn:"\u0e23\u0e31\u0e1a\u0e04\u0e27\u0e32\u0e21\u0e17\u0e49\u0e32\u0e17\u0e32\u0e22\u0e43\u0e2b\u0e21\u0e48",instructions_visual:"\u0e1e\u0e34\u0e21\u0e1e\u0e4c\u0e02\u0e49\u0e2d\u0e04\u0e27\u0e32\u0e21\u0e19\u0e35\u0e49:",instructions_audio:"\u0e1e\u0e34\u0e21\u0e1e\u0e4c\u0e2a\u0e34\u0e48\u0e07\u0e17\u0e35\u0e48\u0e04\u0e38\u0e13\u0e44\u0e14\u0e49\u0e22\u0e34\u0e19:",
-help_btn:"\u0e04\u0e27\u0e32\u0e21\u0e0a\u0e48\u0e27\u0e22\u0e40\u0e2b\u0e25\u0e37\u0e2d",play_again:"\u0e40\u0e25\u0e48\u0e19\u0e40\u0e2a\u0e35\u0e22\u0e07\u0e2d\u0e35\u0e01\u0e04\u0e23\u0e31\u0e49\u0e07",cant_hear_this:"\u0e14\u0e32\u0e27\u0e42\u0e2b\u0e25\u0e14\u0e40\u0e2a\u0e35\u0e22\u0e07\u0e40\u0e1b\u0e47\u0e19 MP3",incorrect_try_again:"\u0e44\u0e21\u0e48\u0e16\u0e39\u0e01\u0e15\u0e49\u0e2d\u0e07 \u0e25\u0e2d\u0e07\u0e2d\u0e35\u0e01\u0e04\u0e23\u0e31\u0e49\u0e07",image_alt_text:"\u0e23\u0e2b\u0e31\u0e2a\u0e20\u0e32\u0e1e reCAPTCHA",
-privacy_and_terms:"\u0e19\u0e42\u0e22\u0e1a\u0e32\u0e22\u0e2a\u0e48\u0e27\u0e19\u0e1a\u0e38\u0e04\u0e04\u0e25\u0e41\u0e25\u0e30\u0e02\u0e49\u0e2d\u0e01\u0e33\u0e2b\u0e19\u0e14"},tr:{visual_challenge:"G\u00f6rsel sorgu al",audio_challenge:"Sesli sorgu al",refresh_btn:"Yeniden y\u00fckle",instructions_visual:"Metni yaz\u0131n:",instructions_audio:"Duydu\u011funuzu yaz\u0131n:",help_btn:"Yard\u0131m",play_again:"Sesi tekrar \u00e7al",cant_hear_this:"Sesi MP3 olarak indir",incorrect_try_again:"Yanl\u0131\u015f. Tekrar deneyin.",
-image_alt_text:"reCAPTCHA sorusu resmi",privacy_and_terms:"Gizlilik ve \u015eartlar"},uk:{visual_challenge:"\u041e\u0442\u0440\u0438\u043c\u0430\u0442\u0438 \u0432\u0456\u0437\u0443\u0430\u043b\u044c\u043d\u0438\u0439 \u0442\u0435\u043a\u0441\u0442",audio_challenge:"\u041e\u0442\u0440\u0438\u043c\u0430\u0442\u0438 \u0430\u0443\u0434\u0456\u043e\u0437\u0430\u043f\u0438\u0441",refresh_btn:"\u041e\u043d\u043e\u0432\u0438\u0442\u0438 \u0442\u0435\u043a\u0441\u0442",instructions_visual:"\u0412\u0432\u0435\u0434\u0456\u0442\u044c \u0442\u0435\u043a\u0441\u0442:",
-instructions_audio:"\u0412\u0432\u0435\u0434\u0456\u0442\u044c \u043f\u043e\u0447\u0443\u0442\u0435:",help_btn:"\u0414\u043e\u0432\u0456\u0434\u043a\u0430",play_again:"\u0412\u0456\u0434\u0442\u0432\u043e\u0440\u0438\u0442\u0438 \u0437\u0430\u043f\u0438\u0441 \u0449\u0435 \u0440\u0430\u0437",cant_hear_this:"\u0417\u0430\u0432\u0430\u043d\u0442\u0430\u0436\u0438\u0442\u0438 \u0437\u0430\u043f\u0438\u0441 \u044f\u043a MP3",incorrect_try_again:"\u041d\u0435\u043f\u0440\u0430\u0432\u0438\u043b\u044c\u043d\u043e. \u0421\u043f\u0440\u043e\u0431\u0443\u0439\u0442\u0435 \u0449\u0435 \u0440\u0430\u0437.",
-image_alt_text:"\u0417\u043e\u0431\u0440\u0430\u0436\u0435\u043d\u043d\u044f \u0437\u0430\u0432\u0434\u0430\u043d\u043d\u044f reCAPTCHA",privacy_and_terms:"\u041a\u043e\u043d\u0444\u0456\u0434\u0435\u043d\u0446\u0456\u0439\u043d\u0456\u0441\u0442\u044c \u0456 \u0443\u043c\u043e\u0432\u0438"},ur:{visual_challenge:"\u0627\u06cc\u06a9 \u0645\u0631\u0626\u06cc \u0686\u06cc\u0644\u0646\u062c \u062d\u0627\u0635\u0644 \u06a9\u0631\u06cc\u06ba",audio_challenge:"\u0627\u06cc\u06a9 \u0622\u0688\u06cc\u0648 \u0686\u06cc\u0644\u0646\u062c \u062d\u0627\u0635\u0644 \u06a9\u0631\u06cc\u06ba",
-refresh_btn:"\u0627\u06cc\u06a9 \u0646\u06cc\u0627 \u0686\u06cc\u0644\u0646\u062c \u062d\u0627\u0635\u0644 \u06a9\u0631\u06cc\u06ba",instructions_visual:"",instructions_audio:"\u062c\u0648 \u0633\u0646\u0627\u0626\u06cc \u062f\u06cc\u062a\u0627 \u06c1\u06d2 \u0648\u06c1 \u0679\u0627\u0626\u067e \u06a9\u0631\u06cc\u06ba:",help_btn:"\u0645\u062f\u062f",play_again:"\u0622\u0648\u0627\u0632 \u062f\u0648\u0628\u0627\u0631\u06c1 \u0686\u0644\u0627\u0626\u06cc\u06ba",cant_hear_this:"\u0622\u0648\u0627\u0632 \u06a9\u0648 MP3 \u06a9\u06d2 \u0628\u0637\u0648\u0631 \u0688\u0627\u0624\u0646 \u0644\u0648\u0688 \u06a9\u0631\u06cc\u06ba",
-incorrect_try_again:"\u063a\u0644\u0637\u06d4 \u062f\u0648\u0628\u0627\u0631\u06c1 \u06a9\u0648\u0634\u0634 \u06a9\u0631\u06cc\u06ba\u06d4",image_alt_text:"reCAPTCHA \u0686\u06cc\u0644\u0646\u062c \u0648\u0627\u0644\u06cc \u0634\u0628\u06cc\u06c1",privacy_and_terms:"\u0631\u0627\u0632\u062f\u0627\u0631\u06cc \u0648 \u0634\u0631\u0627\u0626\u0637"},vi:{visual_challenge:"Nh\u1eadn th\u1eed th\u00e1ch h\u00ecnh \u1ea3nh",audio_challenge:"Nh\u1eadn th\u1eed th\u00e1ch \u00e2m thanh",refresh_btn:"Nh\u1eadn th\u1eed th\u00e1ch m\u1edbi",
-instructions_visual:"Nh\u1eadp v\u0103n b\u1ea3n:",instructions_audio:"Nh\u1eadp n\u1ed9i dung b\u1ea1n nghe th\u1ea5y:",help_btn:"Tr\u1ee3 gi\u00fap",play_again:"Ph\u00e1t l\u1ea1i \u00e2m thanh",cant_hear_this:"T\u1ea3i \u00e2m thanh xu\u1ed1ng d\u01b0\u1edbi d\u1ea1ng MP3",incorrect_try_again:"Kh\u00f4ng ch\u00ednh x\u00e1c. H\u00e3y th\u1eed l\u1ea1i.",image_alt_text:"H\u00ecnh x\u00e1c th\u1ef1c reCAPTCHA",privacy_and_terms:"B\u1ea3o m\u1eadt v\u00e0 \u0111i\u1ec1u kho\u1ea3n"},"zh-CN":ra,"zh-HK":{visual_challenge:"\u56de\u7b54\u5716\u50cf\u9a57\u8b49\u554f\u984c",
-audio_challenge:"\u53d6\u5f97\u8a9e\u97f3\u9a57\u8b49\u554f\u984c",refresh_btn:"\u63db\u4e00\u500b\u9a57\u8b49\u554f\u984c",instructions_visual:"\u8f38\u5165\u6587\u5b57\uff1a",instructions_audio:"\u9375\u5165\u60a8\u6240\u807d\u5230\u7684\uff1a",help_btn:"\u8aaa\u660e",play_again:"\u518d\u6b21\u64ad\u653e\u8072\u97f3",cant_hear_this:"\u5c07\u8072\u97f3\u4e0b\u8f09\u70ba MP3",incorrect_try_again:"\u4e0d\u6b63\u78ba\uff0c\u518d\u8a66\u4e00\u6b21\u3002",image_alt_text:"reCAPTCHA \u9a57\u8b49\u6587\u5b57\u5716\u7247",
-privacy_and_terms:"\u79c1\u96b1\u6b0a\u8207\u689d\u6b3e"},"zh-TW":{visual_challenge:"\u53d6\u5f97\u5716\u7247\u9a57\u8b49\u554f\u984c",audio_challenge:"\u53d6\u5f97\u8a9e\u97f3\u9a57\u8b49\u554f\u984c",refresh_btn:"\u53d6\u5f97\u65b0\u7684\u9a57\u8b49\u554f\u984c",instructions_visual:"\u8acb\u8f38\u5165\u5716\u7247\u4e2d\u7684\u6587\u5b57\uff1a",instructions_audio:"\u8acb\u8f38\u5165\u8a9e\u97f3\u5167\u5bb9\uff1a",help_btn:"\u8aaa\u660e",play_again:"\u518d\u6b21\u64ad\u653e",cant_hear_this:"\u4ee5 MP3 \u683c\u5f0f\u4e0b\u8f09\u8072\u97f3",
-incorrect_try_again:"\u9a57\u8b49\u78bc\u6709\u8aa4\uff0c\u8acb\u518d\u8a66\u4e00\u6b21\u3002",image_alt_text:"reCAPTCHA \u9a57\u8b49\u6587\u5b57\u5716\u7247",privacy_and_terms:"\u96b1\u79c1\u6b0a\u8207\u689d\u6b3e"},zu:{visual_challenge:"Thola inselelo ebonakalayo",audio_challenge:"Thola inselelo yokulalelwayo",refresh_btn:"Thola inselelo entsha",instructions_visual:"",instructions_audio:"Bhala okuzwayo:",help_btn:"Usizo",play_again:"Phinda udlale okulalelwayo futhi",cant_hear_this:"Layisha umsindo njenge-MP3",
-incorrect_try_again:"Akulungile. Zama futhi.",image_alt_text:"umfanekiso oyinselelo we-reCAPTCHA",privacy_and_terms:"Okwangasese kanye nemigomo"},tl:la,he:oa,"in":na,mo:qa,zh:ra};var x=function(a){if(Error.captureStackTrace)Error.captureStackTrace(this,x);else{var b=Error().stack;b&&(this.stack=b)}a&&(this.message=String(a))};u(x,Error);x.prototype.name="CustomError";var ta;var ua=function(a,b){for(var c=a.split("%s"),d="",e=Array.prototype.slice.call(arguments,1);e.length&&1<c.length;)d+=c.shift()+e.shift();return d+c.join("%s")},va=String.prototype.trim?function(a){return a.trim()}:function(a){return a.replace(/^[\s\xa0]+|[\s\xa0]+$/g,"")},Da=function(a){if(!wa.test(a))return a;-1!=a.indexOf("&")&&(a=a.replace(xa,"&amp;"));-1!=a.indexOf("<")&&(a=a.replace(ya,"&lt;"));-1!=a.indexOf(">")&&(a=a.replace(za,"&gt;"));-1!=a.indexOf('"')&&(a=a.replace(Aa,"&quot;"));-1!=a.indexOf("'")&&
-(a=a.replace(Ba,"&#39;"));-1!=a.indexOf("\x00")&&(a=a.replace(Ca,"&#0;"));return a},xa=/&/g,ya=/</g,za=/>/g,Aa=/"/g,Ba=/'/g,Ca=/\x00/g,wa=/[\x00&<>"']/,Ea=function(a,b){return a<b?-1:a>b?1:0},Fa=function(a){return String(a).replace(/\-([a-z])/g,function(a,c){return c.toUpperCase()})},Ga=function(a){var b=q(void 0)?"undefined".replace(/([-()\[\]{}+?*.$\^|,:#<!\\])/g,"\\$1").replace(/\x08/g,"\\x08"):"\\s";return a.replace(new RegExp("(^"+(b?"|["+b+"]+":"")+")([a-z])","g"),function(a,b,e){return b+e.toUpperCase()})};var Ha=function(a,b){b.unshift(a);x.call(this,ua.apply(null,b));b.shift()};u(Ha,x);Ha.prototype.name="AssertionError";
-var Ia=function(a,b,c,d){var e="Assertion failed";if(c)var e=e+(": "+c),g=d;else a&&(e+=": "+a,g=b);throw new Ha(""+e,g||[]);},y=function(a,b,c){a||Ia("",null,b,Array.prototype.slice.call(arguments,2))},Ja=function(a,b){throw new Ha("Failure"+(a?": "+a:""),Array.prototype.slice.call(arguments,1));},Ka=function(a,b,c){q(a)||Ia("Expected string but got %s: %s.",[n(a),a],b,Array.prototype.slice.call(arguments,2));return a},La=function(a,b,c){r(a)||Ia("Expected function but got %s: %s.",[n(a),a],b,Array.prototype.slice.call(arguments,
-2))};var z=Array.prototype,Ma=z.indexOf?function(a,b,c){y(null!=a.length);return z.indexOf.call(a,b,c)}:function(a,b,c){c=null==c?0:0>c?Math.max(0,a.length+c):c;if(q(a))return q(b)&&1==b.length?a.indexOf(b,c):-1;for(;c<a.length;c++)if(c in a&&a[c]===b)return c;return-1},Na=z.forEach?function(a,b,c){y(null!=a.length);z.forEach.call(a,b,c)}:function(a,b,c){for(var d=a.length,e=q(a)?a.split(""):a,g=0;g<d;g++)g in e&&b.call(c,e[g],g,a)},Oa=z.map?function(a,b,c){y(null!=a.length);return z.map.call(a,b,c)}:
-function(a,b,c){for(var d=a.length,e=Array(d),g=q(a)?a.split(""):a,f=0;f<d;f++)f in g&&(e[f]=b.call(c,g[f],f,a));return e},Pa=z.some?function(a,b,c){y(null!=a.length);return z.some.call(a,b,c)}:function(a,b,c){for(var d=a.length,e=q(a)?a.split(""):a,g=0;g<d;g++)if(g in e&&b.call(c,e[g],g,a))return!0;return!1},Qa=function(a,b){var c=Ma(a,b),d;if(d=0<=c)y(null!=a.length),z.splice.call(a,c,1);return d},Ra=function(a){var b=a.length;if(0<b){for(var c=Array(b),d=0;d<b;d++)c[d]=a[d];return c}return[]},
-Sa=function(a,b,c){y(null!=a.length);return 2>=arguments.length?z.slice.call(a,b):z.slice.call(a,b,c)};var Ta=function(a,b){for(var c in a)b.call(void 0,a[c],c,a)},Ua=function(a){var b=[],c=0,d;for(d in a)b[c++]=d;return b},Va=function(a){for(var b in a)return!1;return!0},Xa=function(){var a=Wa()?k.google_ad:null,b={},c;for(c in a)b[c]=a[c];return b},Ya="constructor hasOwnProperty isPrototypeOf propertyIsEnumerable toLocaleString toString valueOf".split(" "),Za=function(a,b){for(var c,d,e=1;e<arguments.length;e++){d=arguments[e];for(c in d)a[c]=d[c];for(var g=0;g<Ya.length;g++)c=Ya[g],Object.prototype.hasOwnProperty.call(d,
-c)&&(a[c]=d[c])}},$a=function(a){var b=arguments.length;if(1==b&&p(arguments[0]))return $a.apply(null,arguments[0]);for(var c={},d=0;d<b;d++)c[arguments[d]]=!0;return c};var A;t:{var ab=k.navigator;if(ab){var bb=ab.userAgent;if(bb){A=bb;break t}}A=""}var B=function(a){return-1!=A.indexOf(a)};var cb=B("Opera")||B("OPR"),C=B("Trident")||B("MSIE"),D=B("Gecko")&&-1==A.toLowerCase().indexOf("webkit")&&!(B("Trident")||B("MSIE")),E=-1!=A.toLowerCase().indexOf("webkit"),db=function(){var a=k.document;return a?a.documentMode:void 0},eb=function(){var a="",b;if(cb&&k.opera)return a=k.opera.version,r(a)?a():a;D?b=/rv\:([^\);]+)(\)|;)/:C?b=/\b(?:MSIE|rv)[: ]([^\);]+)(\)|;)/:E&&(b=/WebKit\/(\S+)/);b&&(a=(a=b.exec(A))?a[1]:"");return C&&(b=db(),b>parseFloat(a))?String(b):a}(),fb={},F=function(a){var b;
-if(!(b=fb[a])){b=0;for(var c=va(String(eb)).split("."),d=va(String(a)).split("."),e=Math.max(c.length,d.length),g=0;0==b&&g<e;g++){var f=c[g]||"",m=d[g]||"",$=RegExp("(\\d*)(\\D*)","g"),K=RegExp("(\\d*)(\\D*)","g");do{var G=$.exec(f)||["","",""],aa=K.exec(m)||["","",""];if(0==G[0].length&&0==aa[0].length)break;b=Ea(0==G[1].length?0:parseInt(G[1],10),0==aa[1].length?0:parseInt(aa[1],10))||Ea(0==G[2].length,0==aa[2].length)||Ea(G[2],aa[2])}while(0==b)}b=fb[a]=0<=b}return b},gb=k.document,hb=gb&&C?db()||
-("CSS1Compat"==gb.compatMode?parseInt(eb,10):5):void 0;var ib=function(a){if(8192>a.length)return String.fromCharCode.apply(null,a);for(var b="",c=0;c<a.length;c+=8192)var d=Sa(a,c,c+8192),b=b+String.fromCharCode.apply(null,d);return b},jb=function(a){return Oa(a,function(a){a=a.toString(16);return 1<a.length?a:"0"+a}).join("")};var kb=null,lb=null,mb=function(a){if(!kb){kb={};lb={};for(var b=0;65>b;b++)kb[b]="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=".charAt(b),lb[kb[b]]=b,62<=b&&(lb["ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_.".charAt(b)]=b)}for(var b=lb,c=[],d=0;d<a.length;){var e=b[a.charAt(d++)],g=d<a.length?b[a.charAt(d)]:0;++d;var f=d<a.length?b[a.charAt(d)]:64;++d;var m=d<a.length?b[a.charAt(d)]:64;++d;if(null==e||null==g||null==f||null==m)throw Error();c.push(e<<2|g>>
-4);64!=f&&(c.push(g<<4&240|f>>2),64!=m&&c.push(f<<6&192|m))}return c};var H=function(){this.disposed_=this.disposed_;this.onDisposeCallbacks_=this.onDisposeCallbacks_};H.prototype.disposed_=!1;H.prototype.dispose=function(){this.disposed_||(this.disposed_=!0,this.disposeInternal())};var nb=function(a,b){a.onDisposeCallbacks_||(a.onDisposeCallbacks_=[]);a.onDisposeCallbacks_.push(l(void 0)?s(b,void 0):b)};H.prototype.disposeInternal=function(){if(this.onDisposeCallbacks_)for(;this.onDisposeCallbacks_.length;)this.onDisposeCallbacks_.shift()()};
-var ob=function(a){a&&"function"==typeof a.dispose&&a.dispose()};var pb=!C||C&&9<=hb;!D&&!C||C&&C&&9<=hb||D&&F("1.9.1");C&&F("9");var sb=function(a){return a?new qb(rb(a)):ta||(ta=new qb)},tb=function(a,b){return q(b)?a.getElementById(b):b},vb=function(a,b){Ta(b,function(b,d){"style"==d?a.style.cssText=b:"class"==d?a.className=b:"for"==d?a.htmlFor=b:d in ub?a.setAttribute(ub[d],b):0==d.lastIndexOf("aria-",0)||0==d.lastIndexOf("data-",0)?a.setAttribute(d,b):a[d]=b})},ub={cellpadding:"cellPadding",cellspacing:"cellSpacing",colspan:"colSpan",frameborder:"frameBorder",height:"height",maxlength:"maxLength",role:"role",rowspan:"rowSpan",
-type:"type",usemap:"useMap",valign:"vAlign",width:"width"},xb=function(a,b,c){return wb(document,arguments)},wb=function(a,b){var c=b[0],d=b[1];if(!pb&&d&&(d.name||d.type)){c=["<",c];d.name&&c.push(' name="',Da(d.name),'"');if(d.type){c.push(' type="',Da(d.type),'"');var e={};Za(e,d);delete e.type;d=e}c.push(">");c=c.join("")}c=a.createElement(c);d&&(q(d)?c.className=d:p(d)?c.className=d.join(" "):vb(c,d));2<b.length&&yb(a,c,b);return c},yb=function(a,b,c){function d(c){c&&b.appendChild(q(c)?a.createTextNode(c):
-c)}for(var e=2;e<c.length;e++){var g=c[e];!ca(g)||da(g)&&0<g.nodeType?d(g):Na(zb(g)?Ra(g):g,d)}},Ab=function(a){for(var b;b=a.firstChild;)a.removeChild(b)},Bb=function(a){a&&a.parentNode&&a.parentNode.removeChild(a)},rb=function(a){y(a,"Node cannot be null or undefined.");return 9==a.nodeType?a:a.ownerDocument||a.document},zb=function(a){if(a&&"number"==typeof a.length){if(da(a))return"function"==typeof a.item||"string"==typeof a.item;if(r(a))return"function"==typeof a.item}return!1},qb=function(a){this.document_=
-a||k.document||document};h=qb.prototype;h.getDomHelper=sb;h.getElement=function(a){return tb(this.document_,a)};h.$=qb.prototype.getElement;h.createDom=function(a,b,c){return wb(this.document_,arguments)};h.createElement=function(a){return this.document_.createElement(a)};h.createTextNode=function(a){return this.document_.createTextNode(String(a))};h.appendChild=function(a,b){a.appendChild(b)};var Cb=function(a){k.setTimeout(function(){throw a;},0)},Db,Eb=function(){var a=k.MessageChannel;"undefined"===typeof a&&"undefined"!==typeof window&&window.postMessage&&window.addEventListener&&(a=function(){var a=document.createElement("iframe");a.style.display="none";a.src="";document.documentElement.appendChild(a);var b=a.contentWindow,a=b.document;a.open();a.write("");a.close();var c="callImmediate"+Math.random(),d="file:"==b.location.protocol?"*":b.location.protocol+"//"+b.location.host,a=s(function(a){if(("*"==
-d||a.origin==d)&&a.data==c)this.port1.onmessage()},this);b.addEventListener("message",a,!1);this.port1={};this.port2={postMessage:function(){b.postMessage(c,d)}}});if("undefined"!==typeof a&&!B("Trident")&&!B("MSIE")){var b=new a,c={},d=c;b.port1.onmessage=function(){if(l(c.next)){c=c.next;var a=c.cb;c.cb=null;a()}};return function(a){d.next={cb:a};d=d.next;b.port2.postMessage(0)}}return"undefined"!==typeof document&&"onreadystatechange"in document.createElement("script")?function(a){var b=document.createElement("script");
-b.onreadystatechange=function(){b.onreadystatechange=null;b.parentNode.removeChild(b);b=null;a();a=null};document.documentElement.appendChild(b)}:function(a){k.setTimeout(a,0)}};var Kb=function(a,b){Fb||Gb();Hb||(Fb(),Hb=!0);Ib.push(new Jb(a,b))},Fb,Gb=function(){if(k.Promise&&k.Promise.resolve){var a=k.Promise.resolve();Fb=function(){a.then(Lb)}}else Fb=function(){var a=Lb;!r(k.setImmediate)||k.Window&&k.Window.prototype.setImmediate==k.setImmediate?(Db||(Db=Eb()),Db(a)):k.setImmediate(a)}},Hb=!1,Ib=[],Lb=function(){for(;Ib.length;){var a=Ib;Ib=[];for(var b=0;b<a.length;b++){var c=a[b];try{c.fn.call(c.scope)}catch(d){Cb(d)}}}Hb=!1},Jb=function(a,b){this.fn=a;this.scope=
-b};var Mb=function(a){a.prototype.then=a.prototype.then;a.prototype.$goog_Thenable=!0},Nb=function(a){if(!a)return!1;try{return!!a.$goog_Thenable}catch(b){return!1}};var L=function(a,b){this.state_=0;this.result_=void 0;this.callbackEntries_=this.parent_=null;this.hadUnhandledRejection_=this.executing_=!1;try{var c=this;a.call(b,function(a){I(c,2,a)},function(a){if(!(a instanceof J))try{if(a instanceof Error)throw a;throw Error("Promise rejected.");}catch(b){}I(c,3,a)})}catch(d){I(this,3,d)}};
-L.prototype.then=function(a,b,c){null!=a&&La(a,"opt_onFulfilled should be a function.");null!=b&&La(b,"opt_onRejected should be a function. Did you pass opt_context as the second argument instead of the third?");return Ob(this,r(a)?a:null,r(b)?b:null,c)};Mb(L);L.prototype.cancel=function(a){0==this.state_&&Kb(function(){var b=new J(a);Pb(this,b)},this)};
-var Pb=function(a,b){if(0==a.state_)if(a.parent_){var c=a.parent_;if(c.callbackEntries_){for(var d=0,e=-1,g=0,f;f=c.callbackEntries_[g];g++)if(f=f.child)if(d++,f==a&&(e=g),0<=e&&1<d)break;0<=e&&(0==c.state_&&1==d?Pb(c,b):(d=c.callbackEntries_.splice(e,1)[0],d.child&&Qb(c),d.onRejected(b)))}}else I(a,3,b)},Sb=function(a,b){a.callbackEntries_&&a.callbackEntries_.length||2!=a.state_&&3!=a.state_||Rb(a);a.callbackEntries_||(a.callbackEntries_=[]);a.callbackEntries_.push(b)},Ob=function(a,b,c,d){var e=
-{child:null,onFulfilled:null,onRejected:null};e.child=new L(function(a,f){e.onFulfilled=b?function(c){try{var e=b.call(d,c);a(e)}catch(K){f(K)}}:a;e.onRejected=c?function(b){try{var e=c.call(d,b);!l(e)&&b instanceof J?f(b):a(e)}catch(K){f(K)}}:f});e.child.parent_=a;Sb(a,e);return e.child};L.prototype.unblockAndFulfill_=function(a){y(1==this.state_);this.state_=0;I(this,2,a)};L.prototype.unblockAndReject_=function(a){y(1==this.state_);this.state_=0;I(this,3,a)};
-var I=function(a,b,c){if(0==a.state_){if(a==c)b=3,c=new TypeError("Promise cannot resolve to itself");else{if(Nb(c)){a.state_=1;c.then(a.unblockAndFulfill_,a.unblockAndReject_,a);return}if(da(c))try{var d=c.then;if(r(d)){Tb(a,c,d);return}}catch(e){b=3,c=e}}a.result_=c;a.state_=b;Rb(a);3!=b||c instanceof J||Ub(a,c)}},Tb=function(a,b,c){a.state_=1;var d=!1,e=function(b){d||(d=!0,a.unblockAndFulfill_(b))},g=function(b){d||(d=!0,a.unblockAndReject_(b))};try{c.call(b,e,g)}catch(f){g(f)}},Rb=function(a){a.executing_||
-(a.executing_=!0,Kb(a.executeCallbacks_,a))};L.prototype.executeCallbacks_=function(){for(;this.callbackEntries_&&this.callbackEntries_.length;){var a=this.callbackEntries_;this.callbackEntries_=[];for(var b=0;b<a.length;b++){var c=a[b],d=this.result_;if(2==this.state_)c.onFulfilled(d);else c.child&&Qb(this),c.onRejected(d)}}this.executing_=!1};
-var Qb=function(a){for(;a&&a.hadUnhandledRejection_;a=a.parent_)a.hadUnhandledRejection_=!1},Ub=function(a,b){a.hadUnhandledRejection_=!0;Kb(function(){a.hadUnhandledRejection_&&Vb.call(null,b)})},Vb=Cb,J=function(a){x.call(this,a)};u(J,x);J.prototype.name="cancel";/*
- Portions of this code are from MochiKit, received by
- The Closure Authors under the MIT license. All other code is Copyright
- 2005-2009 The Closure Authors. All Rights Reserved.
-*/
-var M=function(a,b){this.sequence_=[];this.onCancelFunction_=a;this.defaultScope_=b||null;this.hadError_=this.fired_=!1;this.result_=void 0;this.silentlyCanceled_=this.blocking_=this.blocked_=!1;this.unhandledErrorId_=0;this.parent_=null;this.branches_=0};
-M.prototype.cancel=function(a){if(this.fired_)this.result_ instanceof M&&this.result_.cancel();else{if(this.parent_){var b=this.parent_;delete this.parent_;a?b.cancel(a):(b.branches_--,0>=b.branches_&&b.cancel())}this.onCancelFunction_?this.onCancelFunction_.call(this.defaultScope_,this):this.silentlyCanceled_=!0;this.fired_||Wb(this,new Xb)}};M.prototype.continue_=function(a,b){this.blocked_=!1;Yb(this,a,b)};
-var Yb=function(a,b,c){a.fired_=!0;a.result_=c;a.hadError_=!b;Zb(a)},ac=function(a){if(a.fired_){if(!a.silentlyCanceled_)throw new $b;a.silentlyCanceled_=!1}};M.prototype.callback=function(a){ac(this);bc(a);Yb(this,!0,a)};var Wb=function(a,b){ac(a);bc(b);Yb(a,!1,b)},bc=function(a){y(!(a instanceof M),"An execution sequence may not be initiated with a blocking Deferred.")},cc=function(a,b,c,d){y(!a.blocking_,"Blocking Deferreds can not be re-used");a.sequence_.push([b,c,d]);a.fired_&&Zb(a)};
-M.prototype.then=function(a,b,c){var d,e,g=new L(function(a,b){d=a;e=b});cc(this,d,function(a){a instanceof Xb?g.cancel():e(a)});return g.then(a,b,c)};Mb(M);
-var dc=function(a){return Pa(a.sequence_,function(a){return r(a[1])})},Zb=function(a){if(a.unhandledErrorId_&&a.fired_&&dc(a)){var b=a.unhandledErrorId_,c=ec[b];c&&(k.clearTimeout(c.id_),delete ec[b]);a.unhandledErrorId_=0}a.parent_&&(a.parent_.branches_--,delete a.parent_);for(var b=a.result_,d=c=!1;a.sequence_.length&&!a.blocked_;){var e=a.sequence_.shift(),g=e[0],f=e[1],e=e[2];if(g=a.hadError_?f:g)try{var m=g.call(e||a.defaultScope_,b);l(m)&&(a.hadError_=a.hadError_&&(m==b||m instanceof Error),
-a.result_=b=m);Nb(b)&&(d=!0,a.blocked_=!0)}catch($){b=$,a.hadError_=!0,dc(a)||(c=!0)}}a.result_=b;d&&(m=s(a.continue_,a,!0),d=s(a.continue_,a,!1),b instanceof M?(cc(b,m,d),b.blocking_=!0):b.then(m,d));c&&(b=new fc(b),ec[b.id_]=b,a.unhandledErrorId_=b.id_)},$b=function(){x.call(this)};u($b,x);$b.prototype.message="Deferred has already fired";$b.prototype.name="AlreadyCalledError";var Xb=function(){x.call(this)};u(Xb,x);Xb.prototype.message="Deferred was canceled";Xb.prototype.name="CanceledError";
-var fc=function(a){this.id_=k.setTimeout(s(this.throwError,this),0);this.error_=a};fc.prototype.throwError=function(){y(ec[this.id_],"Cannot throw an error that is not scheduled.");delete ec[this.id_];throw this.error_;};var ec={};var kc=function(a){var b={},c=b.document||document,d=document.createElement("SCRIPT"),e={script_:d,timeout_:void 0},g=new M(gc,e),f=null,m=null!=b.timeout?b.timeout:5E3;0<m&&(f=window.setTimeout(function(){hc(d,!0);Wb(g,new ic(1,"Timeout reached for loading script "+a))},m),e.timeout_=f);d.onload=d.onreadystatechange=function(){d.readyState&&"loaded"!=d.readyState&&"complete"!=d.readyState||(hc(d,b.cleanupWhenDone||!1,f),g.callback(null))};d.onerror=function(){hc(d,!0,f);Wb(g,new ic(0,"Error while loading script "+
-a))};vb(d,{type:"text/javascript",charset:"UTF-8",src:a});jc(c).appendChild(d);return g},jc=function(a){var b=a.getElementsByTagName("HEAD");return b&&0!=b.length?b[0]:a.documentElement},gc=function(){if(this&&this.script_){var a=this.script_;a&&"SCRIPT"==a.tagName&&hc(a,!0,this.timeout_)}},hc=function(a,b,c){null!=c&&k.clearTimeout(c);a.onload=ba;a.onerror=ba;a.onreadystatechange=ba;b&&window.setTimeout(function(){Bb(a)},0)},ic=function(a,b){var c="Jsloader error (code #"+a+")";b&&(c+=": "+b);x.call(this,
-c);this.code=a};u(ic,x);var lc=function(a){lc[" "](a);return a};lc[" "]=ba;var mc=!C||C&&9<=hb,nc=C&&!F("9");!E||F("528");D&&F("1.9b")||C&&F("8")||cb&&F("9.5")||E&&F("528");D&&!F("8")||C&&F("9");var N=function(a,b){this.type=a;this.currentTarget=this.target=b;this.defaultPrevented=this.propagationStopped_=!1;this.returnValue_=!0};N.prototype.disposeInternal=function(){};N.prototype.dispose=function(){};N.prototype.preventDefault=function(){this.defaultPrevented=!0;this.returnValue_=!1};var O=function(a,b){N.call(this,a?a.type:"");this.relatedTarget=this.currentTarget=this.target=null;this.charCode=this.keyCode=this.button=this.screenY=this.screenX=this.clientY=this.clientX=this.offsetY=this.offsetX=0;this.metaKey=this.shiftKey=this.altKey=this.ctrlKey=!1;this.event_=this.state=null;if(a){var c=this.type=a.type;this.target=a.target||a.srcElement;this.currentTarget=b;var d=a.relatedTarget;if(d){if(D){var e;t:{try{lc(d.nodeName);e=!0;break t}catch(g){}e=!1}e||(d=null)}}else"mouseover"==
-c?d=a.fromElement:"mouseout"==c&&(d=a.toElement);this.relatedTarget=d;this.offsetX=E||void 0!==a.offsetX?a.offsetX:a.layerX;this.offsetY=E||void 0!==a.offsetY?a.offsetY:a.layerY;this.clientX=void 0!==a.clientX?a.clientX:a.pageX;this.clientY=void 0!==a.clientY?a.clientY:a.pageY;this.screenX=a.screenX||0;this.screenY=a.screenY||0;this.button=a.button;this.keyCode=a.keyCode||0;this.charCode=a.charCode||("keypress"==c?a.keyCode:0);this.ctrlKey=a.ctrlKey;this.altKey=a.altKey;this.shiftKey=a.shiftKey;this.metaKey=
-a.metaKey;this.state=a.state;this.event_=a;a.defaultPrevented&&this.preventDefault()}};u(O,N);O.prototype.preventDefault=function(){O.superClass_.preventDefault.call(this);var a=this.event_;if(a.preventDefault)a.preventDefault();else if(a.returnValue=!1,nc)try{if(a.ctrlKey||112<=a.keyCode&&123>=a.keyCode)a.keyCode=-1}catch(b){}};O.prototype.disposeInternal=function(){};var oc="closure_listenable_"+(1E6*Math.random()|0),pc=0;var qc=function(a,b,c,d,e){this.listener=a;this.proxy=null;this.src=b;this.type=c;this.capture=!!d;this.handler=e;this.key=++pc;this.removed=this.callOnce=!1},rc=function(a){a.removed=!0;a.listener=null;a.proxy=null;a.src=null;a.handler=null};var P=function(a){this.src=a;this.listeners={};this.typeCount_=0};P.prototype.add=function(a,b,c,d,e){var g=a.toString();a=this.listeners[g];a||(a=this.listeners[g]=[],this.typeCount_++);var f=sc(a,b,d,e);-1<f?(b=a[f],c||(b.callOnce=!1)):(b=new qc(b,this.src,g,!!d,e),b.callOnce=c,a.push(b));return b};
-P.prototype.remove=function(a,b,c,d){a=a.toString();if(!(a in this.listeners))return!1;var e=this.listeners[a];b=sc(e,b,c,d);return-1<b?(rc(e[b]),y(null!=e.length),z.splice.call(e,b,1),0==e.length&&(delete this.listeners[a],this.typeCount_--),!0):!1};var tc=function(a,b){var c=b.type;if(!(c in a.listeners))return!1;var d=Qa(a.listeners[c],b);d&&(rc(b),0==a.listeners[c].length&&(delete a.listeners[c],a.typeCount_--));return d};
-P.prototype.removeAll=function(a){a=a&&a.toString();var b=0,c;for(c in this.listeners)if(!a||c==a){for(var d=this.listeners[c],e=0;e<d.length;e++)++b,rc(d[e]);delete this.listeners[c];this.typeCount_--}return b};P.prototype.getListener=function(a,b,c,d){a=this.listeners[a.toString()];var e=-1;a&&(e=sc(a,b,c,d));return-1<e?a[e]:null};var sc=function(a,b,c,d){for(var e=0;e<a.length;++e){var g=a[e];if(!g.removed&&g.listener==b&&g.capture==!!c&&g.handler==d)return e}return-1};var uc="closure_lm_"+(1E6*Math.random()|0),vc={},wc=0,xc=function(a,b,c,d,e){if(p(b)){for(var g=0;g<b.length;g++)xc(a,b[g],c,d,e);return null}c=yc(c);if(a&&a[oc])a=a.listen(b,c,d,e);else{if(!b)throw Error("Invalid event type");var g=!!d,f=zc(a);f||(a[uc]=f=new P(a));c=f.add(b,c,!1,d,e);c.proxy||(d=Ac(),c.proxy=d,d.src=a,d.listener=c,a.addEventListener?a.addEventListener(b.toString(),d,g):a.attachEvent(Bc(b.toString()),d),wc++);a=c}return a},Ac=function(){var a=Cc,b=mc?function(c){return a.call(b.src,
-b.listener,c)}:function(c){c=a.call(b.src,b.listener,c);if(!c)return c};return b},Dc=function(a,b,c,d,e){if(p(b))for(var g=0;g<b.length;g++)Dc(a,b[g],c,d,e);else c=yc(c),a&&a[oc]?a.unlisten(b,c,d,e):a&&(a=zc(a))&&(b=a.getListener(b,c,!!d,e))&&Ec(b)},Ec=function(a){if("number"==typeof a||!a||a.removed)return!1;var b=a.src;if(b&&b[oc])return tc(b.eventTargetListeners_,a);var c=a.type,d=a.proxy;b.removeEventListener?b.removeEventListener(c,d,a.capture):b.detachEvent&&b.detachEvent(Bc(c),d);wc--;(c=zc(b))?
-(tc(c,a),0==c.typeCount_&&(c.src=null,b[uc]=null)):rc(a);return!0},Bc=function(a){return a in vc?vc[a]:vc[a]="on"+a},Gc=function(a,b,c,d){var e=1;if(a=zc(a))if(b=a.listeners[b.toString()])for(b=b.concat(),a=0;a<b.length;a++){var g=b[a];g&&g.capture==c&&!g.removed&&(e&=!1!==Fc(g,d))}return Boolean(e)},Fc=function(a,b){var c=a.listener,d=a.handler||a.src;a.callOnce&&Ec(a);return c.call(d,b)},Cc=function(a,b){if(a.removed)return!0;if(!mc){var c;if(!(c=b))t:{c=["window","event"];for(var d=k,e;e=c.shift();)if(null!=
-d[e])d=d[e];else{c=null;break t}c=d}e=c;c=new O(e,this);d=!0;if(!(0>e.keyCode||void 0!=e.returnValue)){t:{var g=!1;if(0==e.keyCode)try{e.keyCode=-1;break t}catch(f){g=!0}if(g||void 0==e.returnValue)e.returnValue=!0}e=[];for(g=c.currentTarget;g;g=g.parentNode)e.push(g);for(var g=a.type,m=e.length-1;!c.propagationStopped_&&0<=m;m--)c.currentTarget=e[m],d&=Gc(e[m],g,!0,c);for(m=0;!c.propagationStopped_&&m<e.length;m++)c.currentTarget=e[m],d&=Gc(e[m],g,!1,c)}return d}return Fc(a,new O(b,this))},zc=function(a){a=
-a[uc];return a instanceof P?a:null},Hc="__closure_events_fn_"+(1E9*Math.random()>>>0),yc=function(a){y(a,"Listener can not be null.");if(r(a))return a;y(a.handleEvent,"An object listener must have handleEvent method.");a[Hc]||(a[Hc]=function(b){return a.handleEvent(b)});return a[Hc]};var Q=function(a){H.call(this);this.handler_=a;this.keys_={}};u(Q,H);var Ic=[];h=Q.prototype;h.listen=function(a,b,c,d){p(b)||(b&&(Ic[0]=b.toString()),b=Ic);for(var e=0;e<b.length;e++){var g=xc(a,b[e],c||this.handleEvent,d||!1,this.handler_||this);if(!g)break;this.keys_[g.key]=g}return this};
-h.unlisten=function(a,b,c,d,e){if(p(b))for(var g=0;g<b.length;g++)this.unlisten(a,b[g],c,d,e);else c=c||this.handleEvent,e=e||this.handler_||this,c=yc(c),d=!!d,b=a&&a[oc]?a.getListener(b,c,d,e):a?(a=zc(a))?a.getListener(b,c,d,e):null:null,b&&(Ec(b),delete this.keys_[b.key]);return this};h.removeAll=function(){Ta(this.keys_,Ec);this.keys_={}};h.disposeInternal=function(){Q.superClass_.disposeInternal.call(this);this.removeAll()};
-h.handleEvent=function(){throw Error("EventHandler.handleEvent not implemented");};var R=function(){H.call(this);this.eventTargetListeners_=new P(this);this.actualEventTarget_=this;this.parentEventTarget_=null};u(R,H);R.prototype[oc]=!0;h=R.prototype;h.setParentEventTarget=function(a){this.parentEventTarget_=a};h.addEventListener=function(a,b,c,d){xc(this,a,b,c,d)};h.removeEventListener=function(a,b,c,d){Dc(this,a,b,c,d)};
-h.dispatchEvent=function(a){Jc(this);var b,c=this.parentEventTarget_;if(c){b=[];for(var d=1;c;c=c.parentEventTarget_)b.push(c),y(1E3>++d,"infinite loop")}c=this.actualEventTarget_;d=a.type||a;if(q(a))a=new N(a,c);else if(a instanceof N)a.target=a.target||c;else{var e=a;a=new N(d,c);Za(a,e)}var e=!0,g;if(b)for(var f=b.length-1;!a.propagationStopped_&&0<=f;f--)g=a.currentTarget=b[f],e=Kc(g,d,!0,a)&&e;a.propagationStopped_||(g=a.currentTarget=c,e=Kc(g,d,!0,a)&&e,a.propagationStopped_||(e=Kc(g,d,!1,a)&&
-e));if(b)for(f=0;!a.propagationStopped_&&f<b.length;f++)g=a.currentTarget=b[f],e=Kc(g,d,!1,a)&&e;return e};h.disposeInternal=function(){R.superClass_.disposeInternal.call(this);this.eventTargetListeners_&&this.eventTargetListeners_.removeAll(void 0);this.parentEventTarget_=null};h.listen=function(a,b,c,d){Jc(this);return this.eventTargetListeners_.add(String(a),b,!1,c,d)};h.unlisten=function(a,b,c,d){return this.eventTargetListeners_.remove(String(a),b,c,d)};
-var Kc=function(a,b,c,d){b=a.eventTargetListeners_.listeners[String(b)];if(!b)return!0;b=b.concat();for(var e=!0,g=0;g<b.length;++g){var f=b[g];if(f&&!f.removed&&f.capture==c){var m=f.listener,$=f.handler||f.src;f.callOnce&&tc(a.eventTargetListeners_,f);e=!1!==m.call($,d)&&e}}return e&&0!=d.returnValue_};R.prototype.getListener=function(a,b,c,d){return this.eventTargetListeners_.getListener(String(a),b,c,d)};var Jc=function(a){y(a.eventTargetListeners_,"Event target is not initialized. Did you call the superclass (goog.events.EventTarget) constructor?")};var S=function(a){R.call(this);this.imageIdToRequestMap_={};this.imageIdToImageMap_={};this.handler_=new Q(this);this.parent_=a};u(S,R);var Lc=[C&&!F("11")?"readystatechange":"load","abort","error"],Mc=function(a,b,c){(c=q(c)?c:c.src)&&(a.imageIdToRequestMap_[b]={src:c,corsRequestType:l(void 0)?void 0:null})};
-S.prototype.start=function(){var a=this.imageIdToRequestMap_;Na(Ua(a),function(b){var c=a[b];if(c&&(delete a[b],!this.disposed_)){var d;d=this.parent_?sb(this.parent_).createDom("img"):new Image;c.corsRequestType&&(d.crossOrigin=c.corsRequestType);this.handler_.listen(d,Lc,this.onNetworkEvent_);this.imageIdToImageMap_[b]=d;d.id=b;d.src=c.src}},this)};
-S.prototype.onNetworkEvent_=function(a){var b=a.currentTarget;if(b){if("readystatechange"==a.type)if("complete"==b.readyState)a.type="load";else return;"undefined"==typeof b.naturalWidth&&("load"==a.type?(b.naturalWidth=b.width,b.naturalHeight=b.height):(b.naturalWidth=0,b.naturalHeight=0));this.dispatchEvent({type:a.type,target:b});!this.disposed_&&(a=b.id,delete this.imageIdToRequestMap_[a],b=this.imageIdToImageMap_[a])&&(delete this.imageIdToImageMap_[a],this.handler_.unlisten(b,Lc,this.onNetworkEvent_),
-Va(this.imageIdToImageMap_)&&Va(this.imageIdToRequestMap_)&&this.dispatchEvent("complete"))}};S.prototype.disposeInternal=function(){delete this.imageIdToRequestMap_;delete this.imageIdToImageMap_;ob(this.handler_);S.superClass_.disposeInternal.call(this)};var T=function(){};T.getInstance=function(){return T.instance_?T.instance_:T.instance_=new T};T.prototype.nextId_=0;var U=function(a){R.call(this);this.dom_=a||sb();this.id_=null;this.inDocument_=!1;this.element_=null;this.googUiComponentHandler_=void 0;this.childIndex_=this.children_=this.parent_=null;this.wasDecorated_=!1};u(U,R);h=U.prototype;h.idGenerator_=T.getInstance();h.getElement=function(){return this.element_};h.setParentEventTarget=function(a){if(this.parent_&&this.parent_!=a)throw Error("Method not supported");U.superClass_.setParentEventTarget.call(this,a)};h.getDomHelper=function(){return this.dom_};
-h.createDom=function(){this.element_=this.dom_.createElement("div")};
-var Oc=function(a,b){if(a.inDocument_)throw Error("Component already rendered");a.element_||a.createDom();b?b.insertBefore(a.element_,null):a.dom_.document_.body.appendChild(a.element_);a.parent_&&!a.parent_.inDocument_||Nc(a)},Nc=function(a){a.inDocument_=!0;Pc(a,function(a){!a.inDocument_&&a.getElement()&&Nc(a)})},Qc=function(a){Pc(a,function(a){a.inDocument_&&Qc(a)});a.googUiComponentHandler_&&a.googUiComponentHandler_.removeAll();a.inDocument_=!1};
-U.prototype.disposeInternal=function(){this.inDocument_&&Qc(this);this.googUiComponentHandler_&&(this.googUiComponentHandler_.dispose(),delete this.googUiComponentHandler_);Pc(this,function(a){a.dispose()});!this.wasDecorated_&&this.element_&&Bb(this.element_);this.parent_=this.element_=this.childIndex_=this.children_=null;U.superClass_.disposeInternal.call(this)};var Pc=function(a,b){a.children_&&Na(a.children_,b,void 0)};
-U.prototype.removeChild=function(a,b){if(a){var c=q(a)?a:a.id_||(a.id_=":"+(a.idGenerator_.nextId_++).toString(36)),d;this.childIndex_&&c?(d=this.childIndex_,d=(c in d?d[c]:void 0)||null):d=null;a=d;if(c&&a){d=this.childIndex_;c in d&&delete d[c];Qa(this.children_,a);b&&(Qc(a),a.element_&&Bb(a.element_));c=a;if(null==c)throw Error("Unable to set parent component");c.parent_=null;U.superClass_.setParentEventTarget.call(c,null)}}if(!a)throw Error("Child is not in parent component");return a};var V=function(a,b,c){U.call(this,c);this.captchaImage_=a;this.adImage_=b&&300==b.naturalWidth&&57==b.naturalHeight?b:null};u(V,U);V.prototype.createDom=function(){V.superClass_.createDom.call(this);var a=this.getElement();this.captchaImage_.alt=W.image_alt_text;this.getDomHelper().appendChild(a,this.captchaImage_);this.adImage_&&(this.adImage_.alt=W.image_alt_text,this.getDomHelper().appendChild(a,this.adImage_),this.adImage_&&Rc(this.adImage_)&&(a.innerHTML+='<div id="recaptcha-ad-choices"><div class="recaptcha-ad-choices-collapsed"><img height="15" width="30" alt="AdChoices" border="0" src="//www.gstatic.com/recaptcha/api/img/adicon.png"/></div><div class="recaptcha-ad-choices-expanded"><a href="https://support.google.com/adsense/troubleshooter/1631343" target="_blank"><img height="15" width="75" alt="AdChoices" border="0" src="//www.gstatic.com/recaptcha/api/img/adchoices.png"/></a></div></div>'))};
-var Rc=function(a){var b=Sc(a,"visibility");a=Sc(a,"display");return"hidden"!=b&&"none"!=a},Sc=function(a,b){var c;t:{c=rb(a);if(c.defaultView&&c.defaultView.getComputedStyle&&(c=c.defaultView.getComputedStyle(a,null))){c=c[b]||c.getPropertyValue(b)||"";break t}c=""}if(!(c=c||(a.currentStyle?a.currentStyle[b]:null))&&(c=a.style[Fa(b)],"undefined"===typeof c)){c=a.style;var d;t:if(d=Fa(b),void 0===a.style[d]){var e=(E?"Webkit":D?"Moz":C?"ms":cb?"O":null)+Ga(d);if(void 0!==a.style[e]){d=e;break t}}c=
-c[d]||""}return c};V.prototype.disposeInternal=function(){delete this.captchaImage_;delete this.adImage_;V.superClass_.disposeInternal.call(this)};var Tc=function(a,b,c){H.call(this);this.listener_=a;this.interval_=b||0;this.handler_=c;this.callback_=s(this.doAction_,this)};u(Tc,H);h=Tc.prototype;h.id_=0;h.disposeInternal=function(){Tc.superClass_.disposeInternal.call(this);this.stop();delete this.listener_;delete this.handler_};
-h.start=function(a){this.stop();var b=this.callback_;a=l(a)?a:this.interval_;if(!r(b))if(b&&"function"==typeof b.handleEvent)b=s(b.handleEvent,b);else throw Error("Invalid listener argument");this.id_=2147483647<a?-1:k.setTimeout(b,a||0)};h.stop=function(){this.isActive()&&k.clearTimeout(this.id_);this.id_=0};h.isActive=function(){return 0!=this.id_};h.doAction_=function(){this.id_=0;this.listener_&&this.listener_.call(this.handler_)};var Uc=function(a,b){H.call(this);this.listener_=a;this.handler_=b;this.delay_=new Tc(s(this.onTick_,this),0,this)};u(Uc,H);h=Uc.prototype;h.interval_=0;h.runUntil_=0;h.disposeInternal=function(){this.delay_.dispose();delete this.listener_;delete this.handler_;Uc.superClass_.disposeInternal.call(this)};h.start=function(a,b){this.stop();var c=b||0;this.interval_=Math.max(a||0,0);this.runUntil_=0>c?-1:ha()+c;this.delay_.start(0>c?this.interval_:Math.min(this.interval_,c))};h.stop=function(){this.delay_.stop()};
-h.isActive=function(){return this.delay_.isActive()};h.onSuccess=function(){};h.onFailure=function(){};h.onTick_=function(){if(this.listener_.call(this.handler_))this.onSuccess();else if(0>this.runUntil_)this.delay_.start(this.interval_);else{var a=this.runUntil_-ha();if(0>=a)this.onFailure();else this.delay_.start(Math.min(this.interval_,a))}};$a("area base br col command embed hr img input keygen link meta param source track wbr".split(" "));$a("action","cite","data","formaction","href","manifest","poster","src");$a("link","script","style");var Vc={sanitizedContentKindHtml:!0},Wc={sanitizedContentKindText:!0},Xc=function(){throw Error("Do not instantiate directly");};Xc.prototype.contentDir=null;Xc.prototype.toString=function(){return this.content};var bd=function(a){var b=Yc;y(b,"Soy template may not be null.");var c=sb().createElement("DIV");a=Zc(b(a||$c,void 0,void 0));b=a.match(ad);y(!b,"This template starts with a %s, which cannot be a child of a <div>, as required by soy internals. Consider using goog.soy.renderElement instead.\nTemplate output: %s",b&&b[0],a);c.innerHTML=a;return 1==c.childNodes.length&&(a=c.firstChild,1==a.nodeType)?a:c},Zc=function(a){if(!da(a))return String(a);if(a instanceof Xc){if(a.contentKind===Vc)return Ka(a.content);
-if(a.contentKind===Wc)return Da(a.content)}Ja("Soy template output is unsafe for use as HTML: "+a);return"zSoyz"},ad=/^<(body|caption|col|colgroup|head|html|tr|td|tbody|thead|tfoot)>/i,$c={};C&&F(8);var cd=function(){Xc.call(this)};u(cd,Xc);cd.prototype.contentKind=Vc;var dd=function(a){function b(a){this.content=a}b.prototype=a.prototype;return function(a,d){var e=new b(String(a));void 0!==d&&(e.contentDir=d);return e}}(cd);(function(a){function b(a){this.content=a}b.prototype=a.prototype;return function(a,d){var e=String(a);if(!e)return"";e=new b(e);void 0!==d&&(e.contentDir=d);return e}})(cd);
-var ed={"\x00":"\\x00","\b":"\\x08","\t":"\\t","\n":"\\n","\x0B":"\\x0b","\f":"\\f","\r":"\\r",'"':"\\x22",$:"\\x24","&":"\\x26","'":"\\x27","(":"\\x28",")":"\\x29","*":"\\x2a","+":"\\x2b",",":"\\x2c","-":"\\x2d",".":"\\x2e","/":"\\/",":":"\\x3a","<":"\\x3c","=":"\\x3d",">":"\\x3e","?":"\\x3f","[":"\\x5b","\\":"\\\\","]":"\\x5d","^":"\\x5e","{":"\\x7b","|":"\\x7c","}":"\\x7d","\u0085":"\\x85","\u2028":"\\u2028","\u2029":"\\u2029"},fd=function(a){return ed[a]},gd=/[\x00\x08-\x0d\x22\x26\x27\/\x3c-\x3e\\\x85\u2028\u2029]/g;var Yc=function(a){return dd('<script type="text/javascript">var challenge = \''+String(a.challenge).replace(gd,fd)+"'; var publisherId = '"+String(a.publisherId).replace(gd,fd)+"';"+("ca-mongoogle"==a.publisherId?'google_page_url = "3pcerttesting.com/dab/recaptcha.html";':"")+"\n    google_ad_client = publisherId;\n    google_ad_type = 'html';\n    google_ad_output = 'js';\n    google_image_size = '300x57';\n    google_captcha_token = challenge;\n    google_ad_request_done = function(ad) {\n      window.parent.recaptcha.ads.adutils.googleAdRequestDone(ad);\n    };\n    \x3c/script><script type=\"text/javascript\" src=\"//pagead2.googlesyndication.com/pagead/show_ads.js\">\x3c/script>")};
-Yc.soyTemplateName="recaptcha.soy.ads.iframeAdsLoader.main";var Wa=function(){var a=k.google_ad;return!!(a&&a.token&&a.imageAdUrl&&a.hashedAnswer&&a.salt&&a.delayedImpressionUrl&&a.engagementUrl)},hd=function(){k.google_ad&&(k.google_ad=null)},id=function(a){a=a||document.body;var b=k.google_ad;b&&b.searchUpliftUrl&&(b=xb("iframe",{src:'data:text/html;charset=utf-8,<body><img src="https://'+b.searchUpliftUrl+'"></img></body>',style:"display:none"}),a.appendChild(b))},jd=0,kd=function(a){var b=new S;Mc(b,"recaptcha-url-"+jd++,a);b.start()},ld=function(a,b){var c=
-RecaptchaState.publisher_id;hd();var d=xb("iframe",{id:"recaptcha-loader-"+jd++,style:"display: none"});document.body.appendChild(d);var e=d.contentWindow?d.contentWindow.document:d.contentDocument;e.open("text/html","replace");e.write(bd({challenge:a,publisherId:c}).innerHTML);e.close();c=new Uc(function(){return!!k.google_ad});c.onSuccess=function(){Bb(d);b()};c.onFailure=function(){Bb(d);b()};c.start(50,2E3)};t("recaptcha.ads.adutils.googleAdRequestDone",function(a){k.google_ad=a});var md=function(){this.blockSize=-1};var nd=function(){this.blockSize=-1;this.blockSize=64;this.chain_=Array(4);this.block_=Array(this.blockSize);this.totalLength_=this.blockLength_=0;this.reset()};u(nd,md);nd.prototype.reset=function(){this.chain_[0]=1732584193;this.chain_[1]=4023233417;this.chain_[2]=2562383102;this.chain_[3]=271733878;this.totalLength_=this.blockLength_=0};
-var od=function(a,b,c){c||(c=0);var d=Array(16);if(q(b))for(var e=0;16>e;++e)d[e]=b.charCodeAt(c++)|b.charCodeAt(c++)<<8|b.charCodeAt(c++)<<16|b.charCodeAt(c++)<<24;else for(e=0;16>e;++e)d[e]=b[c++]|b[c++]<<8|b[c++]<<16|b[c++]<<24;b=a.chain_[0];c=a.chain_[1];var e=a.chain_[2],g=a.chain_[3],f=0,f=b+(g^c&(e^g))+d[0]+3614090360&4294967295;b=c+(f<<7&4294967295|f>>>25);f=g+(e^b&(c^e))+d[1]+3905402710&4294967295;g=b+(f<<12&4294967295|f>>>20);f=e+(c^g&(b^c))+d[2]+606105819&4294967295;e=g+(f<<17&4294967295|
-f>>>15);f=c+(b^e&(g^b))+d[3]+3250441966&4294967295;c=e+(f<<22&4294967295|f>>>10);f=b+(g^c&(e^g))+d[4]+4118548399&4294967295;b=c+(f<<7&4294967295|f>>>25);f=g+(e^b&(c^e))+d[5]+1200080426&4294967295;g=b+(f<<12&4294967295|f>>>20);f=e+(c^g&(b^c))+d[6]+2821735955&4294967295;e=g+(f<<17&4294967295|f>>>15);f=c+(b^e&(g^b))+d[7]+4249261313&4294967295;c=e+(f<<22&4294967295|f>>>10);f=b+(g^c&(e^g))+d[8]+1770035416&4294967295;b=c+(f<<7&4294967295|f>>>25);f=g+(e^b&(c^e))+d[9]+2336552879&4294967295;g=b+(f<<12&4294967295|
-f>>>20);f=e+(c^g&(b^c))+d[10]+4294925233&4294967295;e=g+(f<<17&4294967295|f>>>15);f=c+(b^e&(g^b))+d[11]+2304563134&4294967295;c=e+(f<<22&4294967295|f>>>10);f=b+(g^c&(e^g))+d[12]+1804603682&4294967295;b=c+(f<<7&4294967295|f>>>25);f=g+(e^b&(c^e))+d[13]+4254626195&4294967295;g=b+(f<<12&4294967295|f>>>20);f=e+(c^g&(b^c))+d[14]+2792965006&4294967295;e=g+(f<<17&4294967295|f>>>15);f=c+(b^e&(g^b))+d[15]+1236535329&4294967295;c=e+(f<<22&4294967295|f>>>10);f=b+(e^g&(c^e))+d[1]+4129170786&4294967295;b=c+(f<<
-5&4294967295|f>>>27);f=g+(c^e&(b^c))+d[6]+3225465664&4294967295;g=b+(f<<9&4294967295|f>>>23);f=e+(b^c&(g^b))+d[11]+643717713&4294967295;e=g+(f<<14&4294967295|f>>>18);f=c+(g^b&(e^g))+d[0]+3921069994&4294967295;c=e+(f<<20&4294967295|f>>>12);f=b+(e^g&(c^e))+d[5]+3593408605&4294967295;b=c+(f<<5&4294967295|f>>>27);f=g+(c^e&(b^c))+d[10]+38016083&4294967295;g=b+(f<<9&4294967295|f>>>23);f=e+(b^c&(g^b))+d[15]+3634488961&4294967295;e=g+(f<<14&4294967295|f>>>18);f=c+(g^b&(e^g))+d[4]+3889429448&4294967295;c=
-e+(f<<20&4294967295|f>>>12);f=b+(e^g&(c^e))+d[9]+568446438&4294967295;b=c+(f<<5&4294967295|f>>>27);f=g+(c^e&(b^c))+d[14]+3275163606&4294967295;g=b+(f<<9&4294967295|f>>>23);f=e+(b^c&(g^b))+d[3]+4107603335&4294967295;e=g+(f<<14&4294967295|f>>>18);f=c+(g^b&(e^g))+d[8]+1163531501&4294967295;c=e+(f<<20&4294967295|f>>>12);f=b+(e^g&(c^e))+d[13]+2850285829&4294967295;b=c+(f<<5&4294967295|f>>>27);f=g+(c^e&(b^c))+d[2]+4243563512&4294967295;g=b+(f<<9&4294967295|f>>>23);f=e+(b^c&(g^b))+d[7]+1735328473&4294967295;
-e=g+(f<<14&4294967295|f>>>18);f=c+(g^b&(e^g))+d[12]+2368359562&4294967295;c=e+(f<<20&4294967295|f>>>12);f=b+(c^e^g)+d[5]+4294588738&4294967295;b=c+(f<<4&4294967295|f>>>28);f=g+(b^c^e)+d[8]+2272392833&4294967295;g=b+(f<<11&4294967295|f>>>21);f=e+(g^b^c)+d[11]+1839030562&4294967295;e=g+(f<<16&4294967295|f>>>16);f=c+(e^g^b)+d[14]+4259657740&4294967295;c=e+(f<<23&4294967295|f>>>9);f=b+(c^e^g)+d[1]+2763975236&4294967295;b=c+(f<<4&4294967295|f>>>28);f=g+(b^c^e)+d[4]+1272893353&4294967295;g=b+(f<<11&4294967295|
-f>>>21);f=e+(g^b^c)+d[7]+4139469664&4294967295;e=g+(f<<16&4294967295|f>>>16);f=c+(e^g^b)+d[10]+3200236656&4294967295;c=e+(f<<23&4294967295|f>>>9);f=b+(c^e^g)+d[13]+681279174&4294967295;b=c+(f<<4&4294967295|f>>>28);f=g+(b^c^e)+d[0]+3936430074&4294967295;g=b+(f<<11&4294967295|f>>>21);f=e+(g^b^c)+d[3]+3572445317&4294967295;e=g+(f<<16&4294967295|f>>>16);f=c+(e^g^b)+d[6]+76029189&4294967295;c=e+(f<<23&4294967295|f>>>9);f=b+(c^e^g)+d[9]+3654602809&4294967295;b=c+(f<<4&4294967295|f>>>28);f=g+(b^c^e)+d[12]+
-3873151461&4294967295;g=b+(f<<11&4294967295|f>>>21);f=e+(g^b^c)+d[15]+530742520&4294967295;e=g+(f<<16&4294967295|f>>>16);f=c+(e^g^b)+d[2]+3299628645&4294967295;c=e+(f<<23&4294967295|f>>>9);f=b+(e^(c|~g))+d[0]+4096336452&4294967295;b=c+(f<<6&4294967295|f>>>26);f=g+(c^(b|~e))+d[7]+1126891415&4294967295;g=b+(f<<10&4294967295|f>>>22);f=e+(b^(g|~c))+d[14]+2878612391&4294967295;e=g+(f<<15&4294967295|f>>>17);f=c+(g^(e|~b))+d[5]+4237533241&4294967295;c=e+(f<<21&4294967295|f>>>11);f=b+(e^(c|~g))+d[12]+1700485571&
-4294967295;b=c+(f<<6&4294967295|f>>>26);f=g+(c^(b|~e))+d[3]+2399980690&4294967295;g=b+(f<<10&4294967295|f>>>22);f=e+(b^(g|~c))+d[10]+4293915773&4294967295;e=g+(f<<15&4294967295|f>>>17);f=c+(g^(e|~b))+d[1]+2240044497&4294967295;c=e+(f<<21&4294967295|f>>>11);f=b+(e^(c|~g))+d[8]+1873313359&4294967295;b=c+(f<<6&4294967295|f>>>26);f=g+(c^(b|~e))+d[15]+4264355552&4294967295;g=b+(f<<10&4294967295|f>>>22);f=e+(b^(g|~c))+d[6]+2734768916&4294967295;e=g+(f<<15&4294967295|f>>>17);f=c+(g^(e|~b))+d[13]+1309151649&
-4294967295;c=e+(f<<21&4294967295|f>>>11);f=b+(e^(c|~g))+d[4]+4149444226&4294967295;b=c+(f<<6&4294967295|f>>>26);f=g+(c^(b|~e))+d[11]+3174756917&4294967295;g=b+(f<<10&4294967295|f>>>22);f=e+(b^(g|~c))+d[2]+718787259&4294967295;e=g+(f<<15&4294967295|f>>>17);f=c+(g^(e|~b))+d[9]+3951481745&4294967295;a.chain_[0]=a.chain_[0]+b&4294967295;a.chain_[1]=a.chain_[1]+(e+(f<<21&4294967295|f>>>11))&4294967295;a.chain_[2]=a.chain_[2]+e&4294967295;a.chain_[3]=a.chain_[3]+g&4294967295};
-nd.prototype.update=function(a,b){l(b)||(b=a.length);for(var c=b-this.blockSize,d=this.block_,e=this.blockLength_,g=0;g<b;){if(0==e)for(;g<=c;)od(this,a,g),g+=this.blockSize;if(q(a))for(;g<b;){if(d[e++]=a.charCodeAt(g++),e==this.blockSize){od(this,d);e=0;break}}else for(;g<b;)if(d[e++]=a[g++],e==this.blockSize){od(this,d);e=0;break}}this.blockLength_=e;this.totalLength_+=b};var X=function(){Q.call(this);this.callback_=this.element_=null;this.md5_=new nd};u(X,Q);var pd=function(a,b,c,d,e){a.unwatch();a.element_=b;a.callback_=e;a.listen(b,"keyup",s(a.onChanged_,a,c,d))};X.prototype.unwatch=function(){this.element_&&this.callback_&&(this.removeAll(),this.callback_=this.element_=null)};
-X.prototype.onChanged_=function(a,b){var c;c=(c=this.element_.value)?c.replace(/[\s\xa0]+/g,"").toLowerCase():"";this.md5_.reset();this.md5_.update(c+"."+b);c=this.md5_;var d=Array((56>c.blockLength_?c.blockSize:2*c.blockSize)-c.blockLength_);d[0]=128;for(var e=1;e<d.length-8;++e)d[e]=0;for(var g=8*c.totalLength_,e=d.length-8;e<d.length;++e)d[e]=g&255,g/=256;c.update(d);d=Array(16);for(e=g=0;4>e;++e)for(var f=0;32>f;f+=8)d[g++]=c.chain_[e]>>>f&255;jb(d).toLowerCase()==a.toLowerCase()&&this.callback_()};
-X.prototype.disposeInternal=function(){this.element_=null;X.superClass_.disposeInternal.call(this)};var rd=function(a,b,c){this.adObject_=a;this.captchaImageUrl_=b;this.opt_successCallback_=c||null;qd(this)};u(rd,H);var qd=function(a){var b=new S;nb(a,ga(ob,b));Mc(b,"recaptcha_challenge_image",a.captchaImageUrl_);Mc(b,"recaptcha_ad_image",a.adObject_.imageAdUrl);var c={};xc(b,"load",s(function(a,b){a[b.target.id]=b.target},a,c));xc(b,"complete",s(a.handleImagesLoaded_,a,c));b.start()};
-rd.prototype.handleImagesLoaded_=function(a){a=new V(a.recaptcha_challenge_image,a.recaptcha_ad_image);nb(this,ga(ob,a));var b=tb(document,"recaptcha_image");Ab(b);Oc(a,b);a.adImage_&&Rc(a.adImage_)&&(kd(this.adObject_.delayedImpressionUrl),a=new X,nb(this,ga(ob,a)),pd(a,tb(document,"recaptcha_response_field"),this.adObject_.hashedAnswer,this.adObject_.salt,s(function(a,b){a.unwatch();kd(b)},this,a,this.adObject_.engagementUrl)),this.opt_successCallback_&&this.opt_successCallback_("04"+this.adObject_.token))};var W=w;t("RecaptchaStr",W);var Y=k.RecaptchaOptions;t("RecaptchaOptions",Y);var sd={tabindex:0,theme:"red",callback:null,lang:null,custom_theme_widget:null,custom_translations:null};t("RecaptchaDefaultOptions",sd);
-var Z={widget:null,timer_id:-1,style_set:!1,theme:null,type:"image",ajax_verify_cb:null,th1:null,th2:null,th3:null,element:"",ad_captcha_plugin:null,reload_timeout:-1,force_reload:!1,$:function(a){return"string"==typeof a?document.getElementById(a):a},attachEvent:function(a,b,c){a&&a.addEventListener?a.addEventListener(b,c,!1):a&&a.attachEvent&&a.attachEvent("on"+b,c)},create:function(a,b,c){Z.destroy();b&&(Z.widget=Z.$(b),Z.element=b);Z._init_options(c);Z._call_challenge(a)},destroy:function(){var a=
-Z.$("recaptcha_challenge_field");a&&a.parentNode.removeChild(a);-1!=Z.timer_id&&clearInterval(Z.timer_id);Z.timer_id=-1;if(a=Z.$("recaptcha_image"))a.innerHTML="";Z.update_widget();Z.widget&&("custom"!=Z.theme?Z.widget.innerHTML="":Z.widget.style.display="none",Z.widget=null)},focus_response_field:function(){var a=Z.$("recaptcha_response_field");a&&a.focus()},get_challenge:function(){return"undefined"==typeof RecaptchaState?null:RecaptchaState.challenge},get_response:function(){var a=Z.$("recaptcha_response_field");
-return a?a.value:null},ajax_verify:function(a){Z.ajax_verify_cb=a;a=Z.get_challenge()||"";var b=Z.get_response()||"";a=Z._get_api_server()+"/ajaxverify?c="+encodeURIComponent(a)+"&response="+encodeURIComponent(b);Z._add_script(a)},_ajax_verify_callback:function(a){Z.ajax_verify_cb(a)},_get_overridable_url:function(a){var b=window.location.protocol;if("undefined"!=typeof _RecaptchaOverrideApiServer)a=_RecaptchaOverrideApiServer;else if("undefined"!=typeof RecaptchaState&&"string"==typeof RecaptchaState.server&&
-0<RecaptchaState.server.length)return RecaptchaState.server.replace(/\/+$/,"");return b+"//"+a},_get_api_server:function(){return Z._get_overridable_url("www.google.com/recaptcha/api")},_get_static_url_root:function(){return Z._get_overridable_url("www.gstatic.com/recaptcha/api")},_call_challenge:function(a){a=Z._get_api_server()+"/challenge?k="+a+"&ajax=1&cachestop="+Math.random();Z.getLang_()&&(a+="&lang="+Z.getLang_());"undefined"!=typeof Y.extra_challenge_params&&(a+="&"+Y.extra_challenge_params);
-Z._add_script(a)},_add_script:function(a){var b=document.createElement("script");b.type="text/javascript";b.src=a;Z._get_script_area().appendChild(b)},_get_script_area:function(){var a=document.getElementsByTagName("head");return a=!a||1>a.length?document.body:a[0]},_hash_merge:function(a){for(var b={},c=0;c<a.length;c++)for(var d in a[c])b[d]=a[c][d];return b},_init_options:function(a){Y=Z._hash_merge([sd,a||{}])},challenge_callback_internal:function(){Z.update_widget();Z._reset_timer();W=Z._hash_merge([w,
-sa[Z.getLang_()]||{},Y.custom_translations||{}]);window.addEventListener&&window.addEventListener("unload",function(){Z.destroy()},!1);Z._is_ie()&&window.attachEvent&&window.attachEvent("onbeforeunload",function(){});if(0<navigator.userAgent.indexOf("KHTML")){var a=document.createElement("iframe");a.src="about:blank";a.style.height="0px";a.style.width="0px";a.style.visibility="hidden";a.style.border="none";a.appendChild(document.createTextNode("This frame prevents back/forward cache problems in Safari."));
-document.body.appendChild(a)}Z._finish_widget()},_add_css:function(a){if(-1!=navigator.appVersion.indexOf("MSIE 5"))document.write('<style type="text/css">'+a+"</style>");else{var b=document.createElement("style");b.type="text/css";b.styleSheet?b.styleSheet.cssText=a:b.appendChild(document.createTextNode(a));Z._get_script_area().appendChild(b)}},_set_style:function(a){Z.style_set||(Z.style_set=!0,Z._add_css(a+"\n\n.recaptcha_is_showing_audio .recaptcha_only_if_image,.recaptcha_isnot_showing_audio .recaptcha_only_if_audio,.recaptcha_had_incorrect_sol .recaptcha_only_if_no_incorrect_sol,.recaptcha_nothad_incorrect_sol .recaptcha_only_if_incorrect_sol{display:none !important}"))},
-_init_builtin_theme:function(){var a=Z.$,b=Z._get_static_url_root(),c=v.VertCss,d=v.VertHtml,e=b+"/img/"+Z.theme,g="gif",b=Z.theme;"clean"==b&&(c=v.CleanCss,d=v.CleanHtml,g="png");c=c.replace(/IMGROOT/g,e);Z._set_style(c);Z.update_widget();Z.widget.innerHTML='<div id="recaptcha_area">'+d+"</div>";c=Z.getLang_();a("recaptcha_privacy")&&null!=c&&"en"==c.substring(0,2).toLowerCase()&&null!=W.privacy_and_terms&&0<W.privacy_and_terms.length&&(c=document.createElement("a"),c.href="http://www.google.com/intl/en/policies/",
-c.target="_blank",c.innerHTML=W.privacy_and_terms,a("recaptcha_privacy").appendChild(c));c=function(b,c,d,K){var G=a(b);G.src=e+"/"+c+"."+g;c=W[d];G.alt=c;b=a(b+"_btn");b.title=c;Z.attachEvent(b,"click",K)};c("recaptcha_reload","refresh","refresh_btn",function(){Z.reload_internal("r")});c("recaptcha_switch_audio","audio","audio_challenge",function(){Z.switch_type("audio")});c("recaptcha_switch_img","text","visual_challenge",function(){Z.switch_type("image")});c("recaptcha_whatsthis","help","help_btn",
-Z.showhelp);"clean"==b&&(a("recaptcha_logo").src=e+"/logo."+g);a("recaptcha_table").className="recaptchatable recaptcha_theme_"+Z.theme;b=function(b,c){var d=a(b);d&&(RecaptchaState.rtl&&"span"==d.tagName.toLowerCase()&&(d.dir="rtl"),d.appendChild(document.createTextNode(W[c])))};b("recaptcha_instructions_image","instructions_visual");b("recaptcha_instructions_audio","instructions_audio");b("recaptcha_instructions_error","incorrect_try_again");a("recaptcha_instructions_image")||a("recaptcha_instructions_audio")||
-(b="audio"==Z.type?W.instructions_audio:W.instructions_visual,b=b.replace(/:$/,""),a("recaptcha_response_field").setAttribute("placeholder",b))},_finish_widget:function(){var a=Z.$,b=Y,c=b.theme;c in{blackglass:1,clean:1,custom:1,red:1,white:1}||(c="red");Z.theme||(Z.theme=c);"custom"!=Z.theme?Z._init_builtin_theme():Z._set_style("");c=document.createElement("span");c.id="recaptcha_challenge_field_holder";c.style.display="none";a("recaptcha_response_field").parentNode.insertBefore(c,a("recaptcha_response_field"));
-a("recaptcha_response_field").setAttribute("autocomplete","off");a("recaptcha_image").style.width="300px";a("recaptcha_image").style.height="57px";a("recaptcha_challenge_field_holder").innerHTML='<input type="hidden" name="recaptcha_challenge_field" id="recaptcha_challenge_field" value=""/>';Z.th_init();Z.should_focus=!1;Z.th3||Z.force_reload?(Z._set_challenge(RecaptchaState.challenge,"image",!0),setTimeout(function(){Z.reload_internal("i")},100)):Z._set_challenge(RecaptchaState.challenge,"image",
-!1);Z.updateTabIndexes_();Z.update_widget();Z.widget&&(Z.widget.style.display="");b.callback&&b.callback()},updateTabIndexes_:function(){var a=Z.$,b=Y;b.tabindex&&(b=b.tabindex,a("recaptcha_response_field").tabIndex=b++,"audio"==Z.type&&a("recaptcha_audio_play_again")&&(a("recaptcha_audio_play_again").tabIndex=b++,a("recaptcha_audio_download"),a("recaptcha_audio_download").tabIndex=b++),"custom"!=Z.theme&&(a("recaptcha_reload_btn").tabIndex=b++,a("recaptcha_switch_audio_btn").tabIndex=b++,a("recaptcha_switch_img_btn").tabIndex=
-b++,a("recaptcha_whatsthis_btn").tabIndex=b,a("recaptcha_privacy").tabIndex=b++))},switch_type:function(a){if(!((new Date).getTime()<Z.reload_timeout)&&(Z.type=a,Z.reload_internal("audio"==Z.type?"a":"v"),"custom"!=Z.theme)){a=Z.$;var b="audio"==Z.type?W.instructions_audio:W.instructions_visual,b=b.replace(/:$/,"");a("recaptcha_response_field").setAttribute("placeholder",b)}},reload:function(){Z.reload_internal("r")},reload_internal:function(a){var b=Y,c=RecaptchaState,d=(new Date).getTime();d<Z.reload_timeout||
-(Z.reload_timeout=d+1E3,"undefined"==typeof a&&(a="r"),d=Z._get_api_server()+"/reload?c="+c.challenge+"&k="+c.site+"&reason="+a+"&type="+Z.type,Z.getLang_()&&(d+="&lang="+Z.getLang_()),"undefined"!=typeof b.extra_challenge_params&&(d+="&"+b.extra_challenge_params),Z.th_callback_invoke(),Z.th1&&(d+="&th="+Z.th1,Z.th1=""),"audio"==Z.type&&(d=b.audio_beta_12_08?d+"&audio_beta_12_08=1":d+"&new_audio_default=1"),Z.should_focus="t"!=a&&"i"!=a,Z._add_script(d),ob(Z.ad_captcha_plugin),c.publisher_id=null)},
-th_callback_invoke:function(){if(Z.th3)try{var a=Z.th3.exec();a&&1600>a.length&&(Z.th1=a)}catch(b){Z.th1=""}},finish_reload:function(a,b,c,d){RecaptchaState.payload_url=c;RecaptchaState.is_incorrect=!1;RecaptchaState.publisher_id=d;Z._set_challenge(a,b,!1);Z.updateTabIndexes_()},_set_challenge:function(a,b,c){"image"==b&&RecaptchaState.publisher_id?ld(a,function(){Z._set_challenge_internal(a,b,c)}):Z._set_challenge_internal(a,b,c)},_set_challenge_internal:function(a,b,c){var d=Z.$,e=RecaptchaState;
-e.challenge=a;Z.type=b;d("recaptcha_challenge_field").value=e.challenge;c||("audio"==b?(d("recaptcha_image").innerHTML=Z.getAudioCaptchaHtml(),Z._loop_playback()):"image"==b&&(a=e.payload_url,a||(a=Z._get_api_server()+"/image?c="+e.challenge,Z.th_callback_invoke(),Z.th1&&(a+="&th="+Z.th1,Z.th1="")),id(d("recaptcha_widget_div")),Wa()?Z.ad_captcha_plugin=new rd(Xa(),a,function(a){RecaptchaState.challenge=a;d("recaptcha_challenge_field").value=a}):d("recaptcha_image").innerHTML='<img id="recaptcha_challenge_image" alt="'+
-W.image_alt_text+'" height="57" width="300" src="'+a+'" />',hd()));Z._css_toggle("recaptcha_had_incorrect_sol","recaptcha_nothad_incorrect_sol",e.is_incorrect);Z._css_toggle("recaptcha_is_showing_audio","recaptcha_isnot_showing_audio","audio"==b);Z._clear_input();Z.should_focus&&Z.focus_response_field();Z._reset_timer()},_reset_timer:function(){clearInterval(Z.timer_id);var a=Math.max(1E3*(RecaptchaState.timeout-60),6E4);Z.timer_id=setInterval(function(){Z.reload_internal("t")},a);return a},showhelp:function(){window.open(Z._get_help_link(),
-"recaptcha_popup","width=460,height=580,location=no,menubar=no,status=no,toolbar=no,scrollbars=yes,resizable=yes")},_clear_input:function(){Z.$("recaptcha_response_field").value=""},_displayerror:function(a){var b=Z.$;b("recaptcha_image").innerHTML="";b("recaptcha_image").appendChild(document.createTextNode(a))},reloaderror:function(a){Z._displayerror(a)},_is_ie:function(){return 0<navigator.userAgent.indexOf("MSIE")&&!window.opera},_css_toggle:function(a,b,c){Z.update_widget();var d=Z.widget;d||
-(d=document.body);var e=d.className,e=e.replace(new RegExp("(^|\\s+)"+a+"(\\s+|$)")," "),e=e.replace(new RegExp("(^|\\s+)"+b+"(\\s+|$)")," ");d.className=e+(" "+(c?a:b))},_get_help_link:function(){var a="https://support.google.com/recaptcha/";Z.getLang_()&&(a+="?hl="+Z.getLang_());return a},playAgain:function(){Z.$("recaptcha_image").innerHTML=Z.getAudioCaptchaHtml();Z._loop_playback()},_loop_playback:function(){var a=Z.$("recaptcha_audio_play_again");a&&Z.attachEvent(a,"click",function(){Z.playAgain();
-return!1})},getAudioCaptchaHtml:function(){var a=RecaptchaState.payload_url;a||(a=Z._get_api_server()+"/audio.mp3?c="+RecaptchaState.challenge,Z.th_callback_invoke(),Z.th1&&(a+="&th="+Z.th1,Z.th1=""));var b=Z._get_api_server()+"/swf/audiocaptcha.swf?v2",b=Z._is_ie()?'<object classid="clsid:D27CDB6E-AE6D-11cf-96B8-444553540000" id="audiocaptcha" width="0" height="0" codebase="https://fpdownload.macromedia.com/get/flashplayer/current/swflash.cab"><param name="movie" value="'+b+'" /><param name="quality" value="high" /><param name="bgcolor" value="#869ca7" /><param name="allowScriptAccess" value="always" /></object><br/>':
-'<embed src="'+b+'" quality="high" bgcolor="#869ca7" width="0" height="0" name="audiocaptcha" align="middle" play="true" loop="false" quality="high" allowScriptAccess="always" type="application/x-shockwave-flash" pluginspage="http://www.adobe.com/go/getflashplayer" /></embed>',c="";Z.checkFlashVer()&&(c="<br/>"+Z.getSpan_('<a id="recaptcha_audio_play_again" class="recaptcha_audio_cant_hear_link">'+W.play_again+"</a>"));c+="<br/>"+Z.getSpan_('<a id="recaptcha_audio_download" class="recaptcha_audio_cant_hear_link" target="_blank" href="'+
-a+'">'+W.cant_hear_this+"</a>");return b+c},getSpan_:function(a){return"<span"+(RecaptchaState&&RecaptchaState.rtl?' dir="rtl"':"")+">"+a+"</span>"},gethttpwavurl:function(){if("audio"!=Z.type)return"";var a=RecaptchaState.payload_url;a||(a=Z._get_api_server()+"/image?c="+RecaptchaState.challenge,Z.th_callback_invoke(),Z.th1&&(a+="&th="+Z.th1,Z.th1=""));return a},checkFlashVer:function(){var a=-1!=navigator.appVersion.indexOf("MSIE"),b=-1!=navigator.appVersion.toLowerCase().indexOf("win"),c=-1!=navigator.userAgent.indexOf("Opera"),
-d=-1;if(null!=navigator.plugins&&0<navigator.plugins.length){if(navigator.plugins["Shockwave Flash 2.0"]||navigator.plugins["Shockwave Flash"])d=navigator.plugins["Shockwave Flash"+(navigator.plugins["Shockwave Flash 2.0"]?" 2.0":"")].description.split(" ")[2].split(".")[0]}else if(a&&b&&!c)try{d=(new ActiveXObject("ShockwaveFlash.ShockwaveFlash.7")).GetVariable("$version").split(" ")[1].split(",")[0]}catch(e){}return 9<=d},getLang_:function(){return Y.lang?Y.lang:"undefined"!=typeof RecaptchaState&&
-RecaptchaState.lang?RecaptchaState.lang:null},challenge_callback:function(){Z.force_reload=!!RecaptchaState.force_reload;if(RecaptchaState.t3){var a=RecaptchaState.t1?ib(mb(RecaptchaState.t1)):"",b=RecaptchaState.t2?ib(mb(RecaptchaState.t2)):"",c=RecaptchaState.t3?ib(mb(RecaptchaState.t3)):"";Z.th2=c;if(a)b=kc(a),cc(b,Z.challenge_callback_internal,null,void 0),cc(b,null,Z.challenge_callback_internal,void 0);else{if(k.execScript)k.execScript(b,"JavaScript");else if(k.eval)null==ia&&(k.eval("var _et_ = 1;"),
-"undefined"!=typeof k._et_?(delete k._et_,ia=!0):ia=!1),ia?k.eval(b):(a=k.document,c=a.createElement("script"),c.type="text/javascript",c.defer=!1,c.appendChild(a.createTextNode(b)),a.body.appendChild(c),a.body.removeChild(c));else throw Error("goog.globalEval not available");Z.challenge_callback_internal()}}else Z.challenge_callback_internal()},th_init:function(){try{k.thintinel&&k.thintinel.th&&(Z.th3=new k.thintinel.th(Z.th2),Z.th2="")}catch(a){}},update_widget:function(){Z.element&&(Z.widget=
-Z.$(Z.element))}};t("Recaptcha",Z);})()

From 5c261107c9b5bf82a51c27fd6cb85e2199e539e3 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 9 Aug 2018 15:34:56 +0100
Subject: [PATCH 493/830] Remove unnecessary resolve_events_with_state_map

We only ever used the synchronous resolve_events_with_state_map in one
place, which is trivial to replace with the async version.
---
 synapse/handlers/federation.py |  2 +-
 synapse/state/__init__.py      | 34 ++++++----------------------------
 synapse/state/v1.py            | 28 ----------------------------
 3 files changed, 7 insertions(+), 57 deletions(-)

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 0ebf0fd18885..3fa7a98445ce 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -1831,7 +1831,7 @@ def do_auth(self, origin, event, context, auth_events):
 
                 room_version = yield self.store.get_room_version(event.room_id)
 
-                new_state = self.state_handler.resolve_events(
+                new_state = yield self.state_handler.resolve_events(
                     room_version,
                     [list(local_view.values()), list(remote_view.values())],
                     event
diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py
index b34970e4d1ee..d7ae22a661f6 100644
--- a/synapse/state/__init__.py
+++ b/synapse/state/__init__.py
@@ -385,6 +385,7 @@ def _state_map_factory(self, ev_ids):
             ev_ids, get_prev_content=False, check_redacted=False,
         )
 
+    @defer.inlineCallbacks
     def resolve_events(self, room_version, state_sets, event):
         logger.info(
             "Resolving state for %s with %d groups", event.room_id, len(state_sets)
@@ -401,15 +402,17 @@ def resolve_events(self, room_version, state_sets, event):
         }
 
         with Measure(self.clock, "state._resolve_events"):
-            new_state = resolve_events_with_state_map(
-                room_version, state_set_ids, state_map,
+            new_state = yield resolve_events_with_factory(
+                room_version, state_set_ids,
+                event_map=state_map,
+                state_map_factory=self._state_map_factory
             )
 
         new_state = {
             key: state_map[ev_id] for key, ev_id in iteritems(new_state)
         }
 
-        return new_state
+        defer.returnValue(new_state)
 
 
 class StateResolutionHandler(object):
@@ -589,31 +592,6 @@ def _make_state_cache_entry(
     )
 
 
-def resolve_events_with_state_map(room_version, state_sets, state_map):
-    """
-    Args:
-        room_version(str): Version of the room
-        state_sets(list): List of dicts of (type, state_key) -> event_id,
-            which are the different state groups to resolve.
-        state_map(dict): a dict from event_id to event, for all events in
-            state_sets.
-
-    Returns
-        dict[(str, str), str]:
-            a map from (type, state_key) to event_id.
-    """
-    if room_version in (RoomVersions.V1, RoomVersions.VDH_TEST,):
-        return v1.resolve_events_with_state_map(
-            state_sets, state_map,
-        )
-    else:
-        # This should only happen if we added a version but forgot to add it to
-        # the list above.
-        raise Exception(
-            "No state resolution algorithm defined for version %r" % (room_version,)
-        )
-
-
 def resolve_events_with_factory(room_version, state_sets, event_map, state_map_factory):
     """
     Args:
diff --git a/synapse/state/v1.py b/synapse/state/v1.py
index 3a1f7054a172..c95477d31890 100644
--- a/synapse/state/v1.py
+++ b/synapse/state/v1.py
@@ -30,34 +30,6 @@
 POWER_KEY = (EventTypes.PowerLevels, "")
 
 
-def resolve_events_with_state_map(state_sets, state_map):
-    """
-    Args:
-        state_sets(list): List of dicts of (type, state_key) -> event_id,
-            which are the different state groups to resolve.
-        state_map(dict): a dict from event_id to event, for all events in
-            state_sets.
-
-    Returns
-        dict[(str, str), str]:
-            a map from (type, state_key) to event_id.
-    """
-    if len(state_sets) == 1:
-        return state_sets[0]
-
-    unconflicted_state, conflicted_state = _seperate(
-        state_sets,
-    )
-
-    auth_events = _create_auth_events_from_maps(
-        unconflicted_state, conflicted_state, state_map
-    )
-
-    return _resolve_with_state(
-        unconflicted_state, conflicted_state, auth_events, state_map
-    )
-
-
 @defer.inlineCallbacks
 def resolve_events_with_factory(state_sets, event_map, state_map_factory):
     """

From 9643a6f7f208f52febc4be7edb1e38f4ff077b4d Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 22 Aug 2018 17:00:29 +0100
Subject: [PATCH 494/830] Update notice format

---
 synapse/api/constants.py                           |  6 ++++--
 .../resource_limits_server_notices.py              | 14 +++++++++++---
 .../test_resource_limits_server_notices.py         | 12 +++++++++---
 3 files changed, 24 insertions(+), 8 deletions(-)

diff --git a/synapse/api/constants.py b/synapse/api/constants.py
index 1fb24578e265..a67862f4ed30 100644
--- a/synapse/api/constants.py
+++ b/synapse/api/constants.py
@@ -80,8 +80,6 @@ class EventTypes(object):
     ServerACL = "m.room.server_acl"
     Pinned = "m.room.pinned_events"
 
-    ServerNoticeLimitReached = "m.server_notice.usage_limit_reached"
-
 
 class RejectedReason(object):
     AUTH_ERROR = "auth_error"
@@ -106,3 +104,7 @@ class ThirdPartyEntityKind(object):
 # vdh-test-version is a placeholder to get room versioning support working and tested
 # until we have a working v2.
 KNOWN_ROOM_VERSIONS = {"1", "vdh-test-version"}
+
+
+ServerNoticeMsgType = "m.server_notice"
+ServerNoticeLimitReached = "m.server_notice.usage_limit_reached"
diff --git a/synapse/server_notices/resource_limits_server_notices.py b/synapse/server_notices/resource_limits_server_notices.py
index 84b91aeb5dcb..575697e54b2c 100644
--- a/synapse/server_notices/resource_limits_server_notices.py
+++ b/synapse/server_notices/resource_limits_server_notices.py
@@ -18,7 +18,11 @@
 
 from twisted.internet import defer
 
-from synapse.api.constants import EventTypes
+from synapse.api.constants import (
+    EventTypes,
+    ServerNoticeLimitReached,
+    ServerNoticeMsgType,
+)
 from synapse.api.errors import AuthError, ResourceLimitError, SynapseError
 from synapse.server_notices.server_notices_manager import SERVER_NOTICE_ROOM_TAG
 
@@ -102,11 +106,13 @@ def maybe_send_server_notice_to_user(self, user_id):
                 # Add block notification
                 content = {
                     'body': event_content,
+                    'msgtype': ServerNoticeMsgType,
+                    'server_notice_type': ServerNoticeLimitReached,
                     'admin_uri': self._config.admin_uri,
                     'limit_type': event_limit_type
                 }
                 event = yield self._server_notices_manager.send_notice(
-                    user_id, content, EventTypes.ServerNoticeLimitReached
+                    user_id, content, EventTypes.Message,
                 )
 
                 content = {
@@ -174,7 +180,9 @@ def _is_room_currently_blocked(self, room_id):
 
         events = yield self._store.get_events(referenced_events)
         for event_id, event in iteritems(events):
-            if event.type == EventTypes.ServerNoticeLimitReached:
+            if event.type != EventTypes.Message:
+                continue
+            if event.content.get("msgtype") == ServerNoticeMsgType:
                 currently_blocked = True
                 # remove event in case we need to disable blocking later on.
                 if event_id in referenced_events:
diff --git a/tests/server_notices/test_resource_limits_server_notices.py b/tests/server_notices/test_resource_limits_server_notices.py
index cc8d2f539c6e..ca9b31128a1a 100644
--- a/tests/server_notices/test_resource_limits_server_notices.py
+++ b/tests/server_notices/test_resource_limits_server_notices.py
@@ -2,7 +2,7 @@
 
 from twisted.internet import defer
 
-from synapse.api.constants import EventTypes
+from synapse.api.constants import EventTypes, ServerNoticeMsgType
 from synapse.api.errors import ResourceLimitError
 from synapse.handlers.auth import AuthHandler
 from synapse.server_notices.resource_limits_server_notices import (
@@ -78,7 +78,10 @@ def test_maybe_send_server_notice_to_user_remove_blocked_notice(self):
         """Test when user has blocked notice, but should have it removed"""
 
         self._rlsn._auth.check_auth_blocking = Mock()
-        mock_event = Mock(type=EventTypes.ServerNoticeLimitReached)
+        mock_event = Mock(
+            type=EventTypes.Message,
+            content={"msgtype": ServerNoticeMsgType},
+        )
         self._rlsn._store.get_events = Mock(return_value=defer.succeed(
             {"123": mock_event}
         ))
@@ -94,7 +97,10 @@ def test_maybe_send_server_notice_to_user_remove_blocked_notice_noop(self):
             side_effect=ResourceLimitError(403, 'foo')
         )
 
-        mock_event = Mock(type=EventTypes.ServerNoticeLimitReached)
+        mock_event = Mock(
+            type=EventTypes.Message,
+            content={"msgtype": ServerNoticeMsgType},
+        )
         self._rlsn._store.get_events = Mock(return_value=defer.succeed(
             {"123": mock_event}
         ))

From 7a0da69eee49ce84ddd8a433288d7ce9cb76e14e Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 23 Aug 2018 10:28:10 +0100
Subject: [PATCH 495/830] Add missing yield

---
 synapse/storage/monthly_active_users.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py
index 06f9a75a9779..fd3b630bd298 100644
--- a/synapse/storage/monthly_active_users.py
+++ b/synapse/storage/monthly_active_users.py
@@ -147,6 +147,7 @@ def _count_users(txn):
             return count
         return self.runInteraction("count_users", _count_users)
 
+    @defer.inlineCallbacks
     def upsert_monthly_active_user(self, user_id):
         """
             Updates or inserts monthly active user member
@@ -155,7 +156,7 @@ def upsert_monthly_active_user(self, user_id):
             Deferred[bool]: True if a new entry was created, False if an
                 existing one was updated.
         """
-        is_insert = self._simple_upsert(
+        is_insert = yield self._simple_upsert(
             desc="upsert_monthly_active_user",
             table="monthly_active_users",
             keyvalues={

From c5842dff1a792843365becfa3ca666e42fdce869 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 23 Aug 2018 10:35:54 +0100
Subject: [PATCH 496/830] Actually run the tests

---
 tests/server_notices/__init__.py | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 tests/server_notices/__init__.py

diff --git a/tests/server_notices/__init__.py b/tests/server_notices/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1

From 4eb8408ed2331bf4a180364e966cb9f93b416202 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 23 Aug 2018 10:46:13 +0100
Subject: [PATCH 497/830] Newsfile

---
 changelog.d/3746.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3746.misc

diff --git a/changelog.d/3746.misc b/changelog.d/3746.misc
new file mode 100644
index 000000000000..fc00ee773a82
--- /dev/null
+++ b/changelog.d/3746.misc
@@ -0,0 +1 @@
+Fix MAU cache invalidation due to missing yield

From 7e6e588e60708be3e58f026bd65bd4680f9cd969 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 23 Aug 2018 16:20:51 +0100
Subject: [PATCH 498/830] Fix bug where we resent "limit exceeded" server
 notices

This was due to a bug where we mutated a cached event's contents
---
 .../resource_limits_server_notices.py         |  6 +-
 .../test_resource_limits_server_notices.py    | 66 +++++++++++++++++++
 2 files changed, 71 insertions(+), 1 deletion(-)

diff --git a/synapse/server_notices/resource_limits_server_notices.py b/synapse/server_notices/resource_limits_server_notices.py
index 575697e54b2c..96eb97771fc4 100644
--- a/synapse/server_notices/resource_limits_server_notices.py
+++ b/synapse/server_notices/resource_limits_server_notices.py
@@ -76,6 +76,10 @@ def maybe_send_server_notice_to_user(self, user_id):
 
         room_id = yield self._server_notices_manager.get_notice_room_for_user(user_id)
 
+        if not room_id:
+            logger.warn("Failed to get server notices room")
+            return
+
         yield self._check_and_set_tags(user_id, room_id)
         currently_blocked, ref_events = yield self._is_room_currently_blocked(room_id)
 
@@ -176,7 +180,7 @@ def _is_room_currently_blocked(self, room_id):
 
         referenced_events = []
         if pinned_state_event is not None:
-            referenced_events = pinned_state_event.content.get('pinned')
+            referenced_events = list(pinned_state_event.content.get('pinned', []))
 
         events = yield self._store.get_events(referenced_events)
         for event_id, event in iteritems(events):
diff --git a/tests/server_notices/test_resource_limits_server_notices.py b/tests/server_notices/test_resource_limits_server_notices.py
index ca9b31128a1a..fede3f52a957 100644
--- a/tests/server_notices/test_resource_limits_server_notices.py
+++ b/tests/server_notices/test_resource_limits_server_notices.py
@@ -144,3 +144,69 @@ def test_maybe_send_server_notice_to_user_not_in_mau_cohort(self):
         yield self._rlsn.maybe_send_server_notice_to_user(self.user_id)
 
         self._send_notice.assert_not_called()
+
+
+class TestResourceLimitsServerNoticesWithRealRooms(unittest.TestCase):
+    @defer.inlineCallbacks
+    def setUp(self):
+        self.hs = yield setup_test_homeserver(self.addCleanup)
+        self.store = self.hs.get_datastore()
+        self.server_notices_sender = self.hs.get_server_notices_sender()
+        self.server_notices_manager = self.hs.get_server_notices_manager()
+        self.event_source = self.hs.get_event_sources()
+
+        # relying on [1] is far from ideal, but the only case where
+        # ResourceLimitsServerNotices class needs to be isolated is this test,
+        # general code should never have a reason to do so ...
+        self._rlsn = self.server_notices_sender._server_notices[1]
+        if not isinstance(self._rlsn, ResourceLimitsServerNotices):
+            raise Exception("Failed to find reference to ResourceLimitsServerNotices")
+
+        self.hs.config.limit_usage_by_mau = True
+        self.hs.config.hs_disabled = False
+        self.hs.config.max_mau_value = 5
+        self.hs.config.server_notices_mxid = "@server:test"
+        self.hs.config.server_notices_mxid_display_name = None
+        self.hs.config.server_notices_mxid_avatar_url = None
+        self.hs.config.server_notices_room_name = "Test Server Notice Room"
+
+        self.user_id = "@user_id:test"
+
+        self.hs.config.admin_uri = "mailto:user@test.com"
+
+    @defer.inlineCallbacks
+    def test_server_notice_only_sent_once(self):
+        self.store.get_monthly_active_count = Mock(
+            return_value=1000,
+        )
+
+        self.store.user_last_seen_monthly_active = Mock(
+            return_value=1000,
+        )
+
+        # Call the function multiple times to ensure we only send the notice once
+        yield self._rlsn.maybe_send_server_notice_to_user(self.user_id)
+        yield self._rlsn.maybe_send_server_notice_to_user(self.user_id)
+        yield self._rlsn.maybe_send_server_notice_to_user(self.user_id)
+
+        # Now lets get the last load of messages in the service notice room and
+        # check that there is only one server notice
+        room_id = yield self.server_notices_manager.get_notice_room_for_user(
+            self.user_id,
+        )
+
+        token = yield self.event_source.get_current_token()
+        events, _ = yield self.store.get_recent_events_for_room(
+            room_id, limit=100, end_token=token.room_key,
+        )
+
+        count = 0
+        for event in events:
+            if event.type != EventTypes.Message:
+                continue
+            if event.content.get("msgtype") != ServerNoticeMsgType:
+                continue
+
+            count += 1
+
+        self.assertEqual(count, 1)

From 60cf1c6e164c153cc98903a2c0cf102825416b03 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 23 Aug 2018 16:34:32 +0100
Subject: [PATCH 499/830] Newsfile

---
 changelog.d/3747.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3747.bugfix

diff --git a/changelog.d/3747.bugfix b/changelog.d/3747.bugfix
new file mode 100644
index 000000000000..c41e2a1213af
--- /dev/null
+++ b/changelog.d/3747.bugfix
@@ -0,0 +1 @@
+Fix bug where we resent "limit exceeded" server notices repeatedly

From cd77270a669fa827912efd86a0723d85e9e388ed Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 23 Aug 2018 19:17:08 +0100
Subject: [PATCH 500/830] Implement trail users

---
 synapse/api/auth.py                     |  6 +++++-
 synapse/config/server.py                |  6 ++++++
 synapse/storage/monthly_active_users.py |  5 +++++
 synapse/storage/registration.py         | 28 ++++++++++++++++++++++++-
 tests/storage/test_registration.py      |  1 +
 5 files changed, 44 insertions(+), 2 deletions(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 4ca40a0f711c..8d2aa5870adc 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -797,11 +797,15 @@ def check_auth_blocking(self, user_id=None):
                 limit_type=self.hs.config.hs_disabled_limit_type
             )
         if self.hs.config.limit_usage_by_mau is True:
-            # If the user is already part of the MAU cohort
+            # If the user is already part of the MAU cohort or a trial user
             if user_id:
                 timestamp = yield self.store.user_last_seen_monthly_active(user_id)
                 if timestamp:
                     return
+
+                is_trial = yield self.store.is_trial_user(user_id)
+                if is_trial:
+                    return
             # Else if there is no room in the MAU bucket, bail
             current_mau = yield self.store.get_monthly_active_count()
             if current_mau >= self.hs.config.max_mau_value:
diff --git a/synapse/config/server.py b/synapse/config/server.py
index 68a612e59482..8eecd28e7d36 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -77,10 +77,15 @@ def read_config(self, config):
             self.max_mau_value = config.get(
                 "max_mau_value", 0,
             )
+
         self.mau_limits_reserved_threepids = config.get(
             "mau_limit_reserved_threepids", []
         )
 
+        self.mau_trial_days = config.get(
+            "mau_trial_days", 0,
+        )
+
         # Options to disable HS
         self.hs_disabled = config.get("hs_disabled", False)
         self.hs_disabled_message = config.get("hs_disabled_message", "")
@@ -365,6 +370,7 @@ def default_config(self, server_name, **kwargs):
           # Enables monthly active user checking
           # limit_usage_by_mau: False
           # max_mau_value: 50
+          # mau_trial_days: 2
           #
           # Sometimes the server admin will want to ensure certain accounts are
           # never blocked by mau checking. These accounts are specified here.
diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py
index fd3b630bd298..d178f5c5badd 100644
--- a/synapse/storage/monthly_active_users.py
+++ b/synapse/storage/monthly_active_users.py
@@ -201,6 +201,11 @@ def populate_monthly_active_users(self, user_id):
             user_id(str): the user_id to query
         """
         if self.hs.config.limit_usage_by_mau:
+            is_trial = yield self.is_trial_user(user_id)
+            if is_trial:
+                # we don't track trial users in the MAU table.
+                return
+
             last_seen_timestamp = yield self.user_last_seen_monthly_active(user_id)
             now = self.hs.get_clock().time_msec()
 
diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py
index 07333f777dfe..26b429e3073c 100644
--- a/synapse/storage/registration.py
+++ b/synapse/storage/registration.py
@@ -26,6 +26,11 @@
 
 
 class RegistrationWorkerStore(SQLBaseStore):
+    def __init__(self, db_conn, hs):
+        super(RegistrationWorkerStore, self).__init__(db_conn, hs)
+
+        self.config = hs.config
+
     @cached()
     def get_user_by_id(self, user_id):
         return self._simple_select_one(
@@ -36,12 +41,33 @@ def get_user_by_id(self, user_id):
             retcols=[
                 "name", "password_hash", "is_guest",
                 "consent_version", "consent_server_notice_sent",
-                "appservice_id",
+                "appservice_id", "creation_ts",
             ],
             allow_none=True,
             desc="get_user_by_id",
         )
 
+    @defer.inlineCallbacks
+    def is_trial_user(self, user_id):
+        """Checks if user is in the "trial" period, i.e. within the first
+        N days of registration defined by `mau_trial_days` config
+
+        Args:
+            user_id (str)
+
+        Returns:
+            Deferred[bool]
+        """
+
+        info = yield self.get_user_by_id(user_id)
+        if not info:
+            defer.returnValue(False)
+
+        now = self.clock.time_msec()
+        trial_duration_ms = self.config.mau_trial_days * 24 * 60 * 60 * 1000
+        is_trial = (now - info["creation_ts"] * 1000) < trial_duration_ms
+        defer.returnValue(is_trial)
+
     @cached()
     def get_user_by_access_token(self, token):
         """Get a user from the given access token.
diff --git a/tests/storage/test_registration.py b/tests/storage/test_registration.py
index 4eda122edc39..3dfb7b903a3a 100644
--- a/tests/storage/test_registration.py
+++ b/tests/storage/test_registration.py
@@ -46,6 +46,7 @@ def test_register(self):
                 "consent_version": None,
                 "consent_server_notice_sent": None,
                 "appservice_id": None,
+                "creation_ts": 1000,
             },
             (yield self.store.get_user_by_id(self.user_id)),
         )

From f2cbbda956a5abd3163665ad87d2fc9500a265d2 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 23 Aug 2018 18:33:04 +0100
Subject: [PATCH 501/830] Unit tests

---
 tests/server.py   |  13 ++-
 tests/test_mau.py | 217 ++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 227 insertions(+), 3 deletions(-)
 create mode 100644 tests/test_mau.py

diff --git a/tests/server.py b/tests/server.py
index c63b2c310000..7dbdb7f8eaea 100644
--- a/tests/server.py
+++ b/tests/server.py
@@ -5,7 +5,7 @@
 
 import attr
 
-from twisted.internet import threads
+from twisted.internet import address, threads
 from twisted.internet.defer import Deferred
 from twisted.python.failure import Failure
 from twisted.test.proto_helpers import MemoryReactorClock
@@ -63,7 +63,9 @@ def requestDone(self, _self):
         self.result["done"] = True
 
     def getPeer(self):
-        return None
+        # We give an address so that getClientIP returns a non null entry,
+        # causing us to record the MAU
+        return address.IPv4Address(b"TCP", "127.0.0.1", 3423)
 
     def getHost(self):
         return None
@@ -91,7 +93,7 @@ def info(self, *args, **kwargs):
         return FakeLogger()
 
 
-def make_request(method, path, content=b""):
+def make_request(method, path, content=b"", access_token=None):
     """
     Make a web request using the given method and path, feed it the
     content, and return the Request and the Channel underneath.
@@ -116,6 +118,11 @@ def make_request(method, path, content=b""):
     req = SynapseRequest(site, channel)
     req.process = lambda: b""
     req.content = BytesIO(content)
+
+    if access_token:
+        req.requestHeaders.addRawHeader(b"Authorization", b"Bearer " + access_token)
+
+    req.requestHeaders.addRawHeader(b"X-Forwarded-For", b"127.0.0.1")
     req.requestReceived(method, path, b"1.1")
 
     return req, channel
diff --git a/tests/test_mau.py b/tests/test_mau.py
new file mode 100644
index 000000000000..0732615447ea
--- /dev/null
+++ b/tests/test_mau.py
@@ -0,0 +1,217 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests REST events for /rooms paths."""
+
+import json
+
+from mock import Mock, NonCallableMock
+
+from synapse.api.constants import LoginType
+from synapse.api.errors import Codes, HttpResponseException, SynapseError
+from synapse.http.server import JsonResource
+from synapse.rest.client.v2_alpha import register, sync
+from synapse.util import Clock
+
+from tests import unittest
+from tests.server import (
+    ThreadedMemoryReactorClock,
+    make_request,
+    render,
+    setup_test_homeserver,
+)
+
+
+class TestMauLimit(unittest.TestCase):
+    def setUp(self):
+        self.reactor = ThreadedMemoryReactorClock()
+        self.clock = Clock(self.reactor)
+
+        self.hs = setup_test_homeserver(
+            self.addCleanup,
+            "red",
+            http_client=None,
+            clock=self.clock,
+            reactor=self.reactor,
+            federation_client=Mock(),
+            ratelimiter=NonCallableMock(spec_set=["send_message"]),
+        )
+
+        self.store = self.hs.get_datastore()
+
+        self.hs.config.registrations_require_3pid = []
+        self.hs.config.enable_registration_captcha = False
+        self.hs.config.recaptcha_public_key = []
+
+        self.hs.config.limit_usage_by_mau = True
+        self.hs.config.hs_disabled = False
+        self.hs.config.max_mau_value = 2
+        self.hs.config.mau_trial_days = 0
+        self.hs.config.server_notices_mxid = "@server:red"
+        self.hs.config.server_notices_mxid_display_name = None
+        self.hs.config.server_notices_mxid_avatar_url = None
+        self.hs.config.server_notices_room_name = "Test Server Notice Room"
+
+        self.resource = JsonResource(self.hs)
+        register.register_servlets(self.hs, self.resource)
+        sync.register_servlets(self.hs, self.resource)
+
+    def test_simple_deny_mau(self):
+        # Create and sync so that the MAU counts get updated
+        token1 = self.create_user("kermit1")
+        self.do_sync_for_user(token1)
+        token2 = self.create_user("kermit2")
+        self.do_sync_for_user(token2)
+
+        # We've created and activated two users, we shouldn't be able to
+        # register new users
+        with self.assertRaises(SynapseError) as cm:
+            self.create_user("kermit3")
+
+        e = cm.exception
+        self.assertEqual(e.code, 403)
+        self.assertEqual(e.errcode, Codes.RESOURCE_LIMIT_EXCEEDED)
+
+    def test_allowed_after_a_month_mau(self):
+        # Create and sync so that the MAU counts get updated
+        token1 = self.create_user("kermit1")
+        self.do_sync_for_user(token1)
+        token2 = self.create_user("kermit2")
+        self.do_sync_for_user(token2)
+
+        # Advance time by 31 days
+        self.reactor.advance(31 * 24 * 60 * 60)
+
+        self.store.reap_monthly_active_users()
+
+        self.reactor.advance(0)
+
+        # We should be able to register more users
+        token3 = self.create_user("kermit3")
+        self.do_sync_for_user(token3)
+
+    def test_trial_delay(self):
+        self.hs.config.mau_trial_days = 1
+
+        # We should be able to register more than the limit initially
+        token1 = self.create_user("kermit1")
+        self.do_sync_for_user(token1)
+        token2 = self.create_user("kermit2")
+        self.do_sync_for_user(token2)
+        token3 = self.create_user("kermit3")
+        self.do_sync_for_user(token3)
+
+        # Advance time by 2 days
+        self.reactor.advance(2 * 24 * 60 * 60)
+
+        # Two users should be able to sync
+        self.do_sync_for_user(token1)
+        self.do_sync_for_user(token2)
+
+        # But the third should fail
+        with self.assertRaises(SynapseError) as cm:
+            self.do_sync_for_user(token3)
+
+        e = cm.exception
+        self.assertEqual(e.code, 403)
+        self.assertEqual(e.errcode, Codes.RESOURCE_LIMIT_EXCEEDED)
+
+        # And new registrations are now denied too
+        with self.assertRaises(SynapseError) as cm:
+            self.create_user("kermit4")
+
+        e = cm.exception
+        self.assertEqual(e.code, 403)
+        self.assertEqual(e.errcode, Codes.RESOURCE_LIMIT_EXCEEDED)
+
+    def test_trial_users_cant_come_back(self):
+        self.hs.config.mau_trial_days = 1
+
+        # We should be able to register more than the limit initially
+        token1 = self.create_user("kermit1")
+        self.do_sync_for_user(token1)
+        token2 = self.create_user("kermit2")
+        self.do_sync_for_user(token2)
+        token3 = self.create_user("kermit3")
+        self.do_sync_for_user(token3)
+
+        # Advance time by 2 days
+        self.reactor.advance(2 * 24 * 60 * 60)
+
+        # Two users should be able to sync
+        self.do_sync_for_user(token1)
+        self.do_sync_for_user(token2)
+
+        # Advance by 2 months so everyone falls out of MAU
+        self.reactor.advance(60 * 24 * 60 * 60)
+        self.store.reap_monthly_active_users()
+        self.reactor.advance(0)
+
+        # We can create as many new users as we want
+        token4 = self.create_user("kermit4")
+        self.do_sync_for_user(token4)
+        token5 = self.create_user("kermit5")
+        self.do_sync_for_user(token5)
+        token6 = self.create_user("kermit6")
+        self.do_sync_for_user(token6)
+
+        # users 2 and 3 can come back to bring us back up to MAU limit
+        self.do_sync_for_user(token2)
+        self.do_sync_for_user(token3)
+
+        # New trial users can still sync
+        self.do_sync_for_user(token4)
+        self.do_sync_for_user(token5)
+        self.do_sync_for_user(token6)
+
+        # But old user cant
+        with self.assertRaises(SynapseError) as cm:
+            self.do_sync_for_user(token1)
+
+        e = cm.exception
+        self.assertEqual(e.code, 403)
+        self.assertEqual(e.errcode, Codes.RESOURCE_LIMIT_EXCEEDED)
+
+    def create_user(self, localpart):
+        request_data = json.dumps({
+            "username": localpart,
+            "password": "monkey",
+            "auth": {"type": LoginType.DUMMY},
+        })
+
+        request, channel = make_request(b"POST", b"/register", request_data)
+        render(request, self.resource, self.reactor)
+
+        if channel.result["code"] != b"200":
+            raise HttpResponseException(
+                int(channel.result["code"]),
+                channel.result["reason"],
+                channel.result["body"],
+            ).to_synapse_error()
+
+        access_token = channel.json_body["access_token"]
+
+        return access_token
+
+    def do_sync_for_user(self, token):
+        request, channel = make_request(b"GET", b"/sync", access_token=token)
+        render(request, self.resource, self.reactor)
+
+        if channel.result["code"] != b"200":
+            raise HttpResponseException(
+                int(channel.result["code"]),
+                channel.result["reason"],
+                channel.result["body"],
+            ).to_synapse_error()

From 28d7b546cb7855e70b82dc7a4aab8a401c5909a2 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 23 Aug 2018 19:18:52 +0100
Subject: [PATCH 502/830] Newsfile

---
 changelog.d/3749.feature | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3749.feature

diff --git a/changelog.d/3749.feature b/changelog.d/3749.feature
new file mode 100644
index 000000000000..9f8837b106de
--- /dev/null
+++ b/changelog.d/3749.feature
@@ -0,0 +1 @@
+Add mau_trial_days config param, so that users only get counted as MAU after N days.

From f584d6108fb9f9e2650404ce6b42fad805ddb9ca Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 24 Aug 2018 10:02:31 +0100
Subject: [PATCH 503/830] Pin to twisted 17.1 or later

Fixes https://github.com/matrix-org/synapse/issues/3741.
---
 synapse/python_dependencies.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index 987eec3ef2d9..9c55e79ef5fe 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -39,7 +39,7 @@
     "signedjson>=1.0.0": ["signedjson>=1.0.0"],
     "pynacl>=1.2.1": ["nacl>=1.2.1", "nacl.bindings"],
     "service_identity>=1.0.0": ["service_identity>=1.0.0"],
-    "Twisted>=16.0.0": ["twisted>=16.0.0"],
+    "Twisted>=17.1.0": ["twisted>=17.1.0"],
 
     # We use crypto.get_elliptic_curve which is only supported in >=0.15
     "pyopenssl>=0.15": ["OpenSSL>=0.15"],

From 05fe8a6c1c11f922555dbd893811101e1befa99a Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 24 Aug 2018 10:04:52 +0100
Subject: [PATCH 504/830] changelog

---
 changelog.d/3751.feature | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3751.feature

diff --git a/changelog.d/3751.feature b/changelog.d/3751.feature
new file mode 100644
index 000000000000..dc9742b15bd6
--- /dev/null
+++ b/changelog.d/3751.feature
@@ -0,0 +1 @@
+Require twisted 17.1 or later (fixes [#3741](https://github.com/matrix-org/synapse/issues/3741)).

From 7f147d623b981644d22cd4db7db0e7baf33b0e2e Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 24 Aug 2018 11:31:31 +0100
Subject: [PATCH 505/830] Fix bug where we broke sync when using
 limit_usage_by_mau

We assumed that we always had service notices configured, but that is
not always true
---
 synapse/server_notices/resource_limits_server_notices.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/synapse/server_notices/resource_limits_server_notices.py b/synapse/server_notices/resource_limits_server_notices.py
index 96eb97771fc4..98495ddbf947 100644
--- a/synapse/server_notices/resource_limits_server_notices.py
+++ b/synapse/server_notices/resource_limits_server_notices.py
@@ -66,6 +66,10 @@ def maybe_send_server_notice_to_user(self, user_id):
         if self._config.limit_usage_by_mau is False:
             return
 
+        if not self._server_notices_manager.is_enabled():
+            # Don't try and send server notices unles they've been enabled
+            return
+
         timestamp = yield self._store.user_last_seen_monthly_active(user_id)
         if timestamp is None:
             # This user will be blocked from receiving the notice anyway.

From 9b1bc593c519219b6006ece5f06756ba764a9eeb Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 24 Aug 2018 11:36:39 +0100
Subject: [PATCH 506/830] Newsfile

---
 changelog.d/3753.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3753.bugfix

diff --git a/changelog.d/3753.bugfix b/changelog.d/3753.bugfix
new file mode 100644
index 000000000000..b4301267df36
--- /dev/null
+++ b/changelog.d/3753.bugfix
@@ -0,0 +1 @@
+Fix bug where we broke sync when using limit_usage_by_mau but hadn't configured server notices

From 1d67b13674c681b0e228049e34fa2fbd43703bb9 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 24 Aug 2018 12:10:01 +0100
Subject: [PATCH 507/830] Fix bug when federation_domain_whitelist is an emtpy
 list

Outbound federation were incorrectly allowed when the config option was
set to an empty list
---
 synapse/http/matrixfederationclient.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 44b61e70a49c..b34bb8e31ab4 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -133,7 +133,7 @@ def _request(self, destination, method, path,
                 failures, connection failures, SSL failures.)
         """
         if (
-            self.hs.config.federation_domain_whitelist and
+            self.hs.config.federation_domain_whitelist is not None and
             destination not in self.hs.config.federation_domain_whitelist
         ):
             raise FederationDeniedError(destination)

From c780d84d6691b686147910a9e8645f2bb4db4c15 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 24 Aug 2018 12:13:00 +0100
Subject: [PATCH 508/830] Newsfile

---
 changelog.d/3754.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3754.bugfix

diff --git a/changelog.d/3754.bugfix b/changelog.d/3754.bugfix
new file mode 100644
index 000000000000..6e3ec8019418
--- /dev/null
+++ b/changelog.d/3754.bugfix
@@ -0,0 +1 @@
+Fix 'federation_domain_whitelist' such that an empty list correctly blocks all outbound federation traffic

From 14d49c51dbbf1222477e076eb4d8d79c0181f901 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 24 Aug 2018 14:44:16 +0100
Subject: [PATCH 509/830] Make content of tag an empty object rather than null

---
 synapse/server_notices/resource_limits_server_notices.py | 2 +-
 synapse/server_notices/server_notices_manager.py         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/synapse/server_notices/resource_limits_server_notices.py b/synapse/server_notices/resource_limits_server_notices.py
index 98495ddbf947..7144dfcbc2fb 100644
--- a/synapse/server_notices/resource_limits_server_notices.py
+++ b/synapse/server_notices/resource_limits_server_notices.py
@@ -153,7 +153,7 @@ def _check_and_set_tags(self, user_id, room_id):
                 need_to_set_tag = False
         if need_to_set_tag:
             yield self._store.add_tag_to_room(
-                user_id, room_id, SERVER_NOTICE_ROOM_TAG, None
+                user_id, room_id, SERVER_NOTICE_ROOM_TAG, {}
             )
 
     @defer.inlineCallbacks
diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py
index 5968104a99ac..a5e472885910 100644
--- a/synapse/server_notices/server_notices_manager.py
+++ b/synapse/server_notices/server_notices_manager.py
@@ -154,7 +154,7 @@ def get_notice_room_for_user(self, user_id):
         )
         room_id = info['room_id']
         yield self._store.add_tag_to_room(
-            user_id, room_id, SERVER_NOTICE_ROOM_TAG, None
+            user_id, room_id, SERVER_NOTICE_ROOM_TAG, {},
         )
 
         logger.info("Created server notices room %s for %s", room_id, user_id)

From cdd24449eeec12f869ceca233085f957d8dc7d19 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 24 Aug 2018 14:50:03 +0100
Subject: [PATCH 510/830] Ensure we wake up /sync when we add tag to notice
 room

---
 synapse/server_notices/resource_limits_server_notices.py | 7 ++++++-
 synapse/server_notices/server_notices_manager.py         | 8 +++++++-
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/synapse/server_notices/resource_limits_server_notices.py b/synapse/server_notices/resource_limits_server_notices.py
index 7144dfcbc2fb..893b9001cd56 100644
--- a/synapse/server_notices/resource_limits_server_notices.py
+++ b/synapse/server_notices/resource_limits_server_notices.py
@@ -46,6 +46,8 @@ def __init__(self, hs):
         self._message_handler = hs.get_message_handler()
         self._state = hs.get_state_handler()
 
+        self._notifier = hs.get_notifier()
+
     @defer.inlineCallbacks
     def maybe_send_server_notice_to_user(self, user_id):
         """Check if we need to send a notice to this user, this will be true in
@@ -152,9 +154,12 @@ def _check_and_set_tags(self, user_id, room_id):
                 # tag already present, nothing to do here
                 need_to_set_tag = False
         if need_to_set_tag:
-            yield self._store.add_tag_to_room(
+            max_id = yield self._store.add_tag_to_room(
                 user_id, room_id, SERVER_NOTICE_ROOM_TAG, {}
             )
+            self._notifier.on_new_event(
+                "account_data_key", max_id, users=[user_id]
+            )
 
     @defer.inlineCallbacks
     def _is_room_currently_blocked(self, room_id):
diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py
index a5e472885910..c5cc6d728e76 100644
--- a/synapse/server_notices/server_notices_manager.py
+++ b/synapse/server_notices/server_notices_manager.py
@@ -39,6 +39,8 @@ def __init__(self, hs):
         self._event_creation_handler = hs.get_event_creation_handler()
         self._is_mine_id = hs.is_mine_id
 
+        self._notifier = hs.get_notifier()
+
     def is_enabled(self):
         """Checks if server notices are enabled on this server.
 
@@ -153,9 +155,13 @@ def get_notice_room_for_user(self, user_id):
             creator_join_profile=join_profile,
         )
         room_id = info['room_id']
-        yield self._store.add_tag_to_room(
+
+        max_id = yield self._store.add_tag_to_room(
             user_id, room_id, SERVER_NOTICE_ROOM_TAG, {},
         )
+        self._notifier.on_new_event(
+            "account_data_key", max_id, users=[user_id]
+        )
 
         logger.info("Created server notices room %s for %s", room_id, user_id)
         defer.returnValue(room_id)

From 45fc0ead10d211351201d1ff5d02cc2cfc2d7595 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 24 Aug 2018 15:04:29 +0100
Subject: [PATCH 511/830] Newsfile

---
 changelog.d/3755.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3755.bugfix

diff --git a/changelog.d/3755.bugfix b/changelog.d/3755.bugfix
new file mode 100644
index 000000000000..6a1f83f0cee1
--- /dev/null
+++ b/changelog.d/3755.bugfix
@@ -0,0 +1 @@
+Fix tagging of server notice rooms

From 01a5a8b9e3f4ce0b4e2419df5dc3645de0428629 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 24 Aug 2018 16:22:37 +0100
Subject: [PATCH 512/830] Fix checking if service notice room is already tagged

This manifested in synapse repeatedly setting the tag for the room
---
 synapse/server_notices/resource_limits_server_notices.py   | 7 +++----
 .../server_notices/test_resource_limits_server_notices.py  | 1 +
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/synapse/server_notices/resource_limits_server_notices.py b/synapse/server_notices/resource_limits_server_notices.py
index 893b9001cd56..5d86114e0978 100644
--- a/synapse/server_notices/resource_limits_server_notices.py
+++ b/synapse/server_notices/resource_limits_server_notices.py
@@ -146,11 +146,10 @@ def _check_and_set_tags(self, user_id, room_id):
             user_id(str): the user in question
             room_id(str): the server notices room for that user
         """
-        tags = yield self._store.get_tags_for_user(user_id)
-        server_notices_tags = tags.get(room_id)
+        tags = yield self._store.get_tags_for_room(user_id, room_id)
         need_to_set_tag = True
-        if server_notices_tags:
-            if server_notices_tags.get(SERVER_NOTICE_ROOM_TAG):
+        if tags:
+            if SERVER_NOTICE_ROOM_TAG in tags:
                 # tag already present, nothing to do here
                 need_to_set_tag = False
         if need_to_set_tag:
diff --git a/tests/server_notices/test_resource_limits_server_notices.py b/tests/server_notices/test_resource_limits_server_notices.py
index fede3f52a957..0ddf4fd50dac 100644
--- a/tests/server_notices/test_resource_limits_server_notices.py
+++ b/tests/server_notices/test_resource_limits_server_notices.py
@@ -55,6 +55,7 @@ def setUp(self):
             returnValue=""
         )
         self._rlsn._store.add_tag_to_room = Mock()
+        self._rlsn._store.get_tags_for_room = Mock(return_value={})
         self.hs.config.admin_uri = "mailto:user@test.com"
 
     @defer.inlineCallbacks

From 05077e06fa527a9b504e34bbd14af53ba66d3f62 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 24 Aug 2018 16:51:27 +0100
Subject: [PATCH 513/830] Change admin_uri to admin_contact in config and
 errors

---
 synapse/api/auth.py                                         | 4 ++--
 synapse/api/errors.py                                       | 6 +++---
 synapse/config/server.py                                    | 4 ++--
 synapse/server_notices/resource_limits_server_notices.py    | 2 +-
 tests/api/test_auth.py                                      | 4 ++--
 tests/server_notices/test_resource_limits_server_notices.py | 4 ++--
 tests/utils.py                                              | 2 +-
 7 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 8d2aa5870adc..a7e3f7a7ac87 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -793,7 +793,7 @@ def check_auth_blocking(self, user_id=None):
             raise ResourceLimitError(
                 403, self.hs.config.hs_disabled_message,
                 errcode=Codes.RESOURCE_LIMIT_EXCEEDED,
-                admin_uri=self.hs.config.admin_uri,
+                admin_contact=self.hs.config.admin_contact,
                 limit_type=self.hs.config.hs_disabled_limit_type
             )
         if self.hs.config.limit_usage_by_mau is True:
@@ -812,7 +812,7 @@ def check_auth_blocking(self, user_id=None):
                 raise ResourceLimitError(
                     403, "Monthly Active User Limit Exceeded",
 
-                    admin_uri=self.hs.config.admin_uri,
+                    admin_contact=self.hs.config.admin_contact,
                     errcode=Codes.RESOURCE_LIMIT_EXCEEDED,
                     limit_type="monthly_active_user"
                 )
diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index c4ddba9889dd..2e7f98404d04 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -239,10 +239,10 @@ class ResourceLimitError(SynapseError):
     def __init__(
         self, code, msg,
         errcode=Codes.RESOURCE_LIMIT_EXCEEDED,
-        admin_uri=None,
+        admin_contact=None,
         limit_type=None,
     ):
-        self.admin_uri = admin_uri
+        self.admin_contact = admin_contact
         self.limit_type = limit_type
         super(ResourceLimitError, self).__init__(code, msg, errcode=errcode)
 
@@ -250,7 +250,7 @@ def error_dict(self):
         return cs_error(
             self.msg,
             self.errcode,
-            admin_uri=self.admin_uri,
+            admin_contact=self.admin_contact,
             limit_type=self.limit_type
         )
 
diff --git a/synapse/config/server.py b/synapse/config/server.py
index 8eecd28e7d36..2faf4721897b 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -93,7 +93,7 @@ def read_config(self, config):
 
         # Admin uri to direct users at should their instance become blocked
         # due to resource constraints
-        self.admin_uri = config.get("admin_uri", None)
+        self.admin_contact = config.get("admin_contact", None)
 
         # FIXME: federation_domain_whitelist needs sytests
         self.federation_domain_whitelist = None
@@ -357,7 +357,7 @@ def default_config(self, server_name, **kwargs):
           # Homeserver blocking
           #
           # How to reach the server admin, used in ResourceLimitError
-          # admin_uri: 'mailto:admin@server.com'
+          # admin_contact: 'mailto:admin@server.com'
           #
           # Global block config
           #
diff --git a/synapse/server_notices/resource_limits_server_notices.py b/synapse/server_notices/resource_limits_server_notices.py
index 893b9001cd56..c584c8f90780 100644
--- a/synapse/server_notices/resource_limits_server_notices.py
+++ b/synapse/server_notices/resource_limits_server_notices.py
@@ -118,7 +118,7 @@ def maybe_send_server_notice_to_user(self, user_id):
                     'body': event_content,
                     'msgtype': ServerNoticeMsgType,
                     'server_notice_type': ServerNoticeLimitReached,
-                    'admin_uri': self._config.admin_uri,
+                    'admin_contact': self._config.admin_contact,
                     'limit_type': event_limit_type
                 }
                 event = yield self._server_notices_manager.send_notice(
diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py
index ed960090c426..54e396d19d5b 100644
--- a/tests/api/test_auth.py
+++ b/tests/api/test_auth.py
@@ -457,7 +457,7 @@ def test_blocking_mau(self):
 
         with self.assertRaises(ResourceLimitError) as e:
             yield self.auth.check_auth_blocking()
-        self.assertEquals(e.exception.admin_uri, self.hs.config.admin_uri)
+        self.assertEquals(e.exception.admin_contact, self.hs.config.admin_contact)
         self.assertEquals(e.exception.errcode, Codes.RESOURCE_LIMIT_EXCEEDED)
         self.assertEquals(e.exception.code, 403)
 
@@ -473,7 +473,7 @@ def test_hs_disabled(self):
         self.hs.config.hs_disabled_message = "Reason for being disabled"
         with self.assertRaises(ResourceLimitError) as e:
             yield self.auth.check_auth_blocking()
-        self.assertEquals(e.exception.admin_uri, self.hs.config.admin_uri)
+        self.assertEquals(e.exception.admin_contact, self.hs.config.admin_contact)
         self.assertEquals(e.exception.errcode, Codes.RESOURCE_LIMIT_EXCEEDED)
         self.assertEquals(e.exception.code, 403)
 
diff --git a/tests/server_notices/test_resource_limits_server_notices.py b/tests/server_notices/test_resource_limits_server_notices.py
index fede3f52a957..494a7bc91eb1 100644
--- a/tests/server_notices/test_resource_limits_server_notices.py
+++ b/tests/server_notices/test_resource_limits_server_notices.py
@@ -55,7 +55,7 @@ def setUp(self):
             returnValue=""
         )
         self._rlsn._store.add_tag_to_room = Mock()
-        self.hs.config.admin_uri = "mailto:user@test.com"
+        self.hs.config.admin_contact = "mailto:user@test.com"
 
     @defer.inlineCallbacks
     def test_maybe_send_server_notice_to_user_flag_off(self):
@@ -172,7 +172,7 @@ def setUp(self):
 
         self.user_id = "@user_id:test"
 
-        self.hs.config.admin_uri = "mailto:user@test.com"
+        self.hs.config.admin_contact = "mailto:user@test.com"
 
     @defer.inlineCallbacks
     def test_server_notice_only_sent_once(self):
diff --git a/tests/utils.py b/tests/utils.py
index 9f7ff94575b6..e8ef10445c72 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -142,7 +142,7 @@ def setup_test_homeserver(
         config.hs_disabled_limit_type = ""
         config.max_mau_value = 50
         config.mau_limits_reserved_threepids = []
-        config.admin_uri = None
+        config.admin_contact = None
 
         # we need a sane default_room_version, otherwise attempts to create rooms will
         # fail.

From 08abe8e13cf0f078c003c38cf5791f947afc4ae7 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 24 Aug 2018 16:24:13 +0100
Subject: [PATCH 514/830] Newsfile

---
 changelog.d/3756.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3756.bugfix

diff --git a/changelog.d/3756.bugfix b/changelog.d/3756.bugfix
new file mode 100644
index 000000000000..6a1f83f0cee1
--- /dev/null
+++ b/changelog.d/3756.bugfix
@@ -0,0 +1 @@
+Fix tagging of server notice rooms

From 69f7f1418fb1b6ff4d1641ca196888f6161ce141 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 24 Aug 2018 16:54:06 +0100
Subject: [PATCH 515/830] Newsfile

---
 changelog.d/3758.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3758.bugfix

diff --git a/changelog.d/3758.bugfix b/changelog.d/3758.bugfix
new file mode 100644
index 000000000000..862739bfe8e3
--- /dev/null
+++ b/changelog.d/3758.bugfix
@@ -0,0 +1 @@
+Fix 'admin_uri' config variable and error parameter to be 'admin_contact' to match the spec.

From 16a31c6fcebe31bbdb655ebcb24ccce426b3e994 Mon Sep 17 00:00:00 2001
From: Hubert Chathi <hubert@uhoreg.ca>
Date: Fri, 24 Aug 2018 22:51:25 -0400
Subject: [PATCH 516/830] update to newer Synapse APIs

---
 synapse/handlers/e2e_room_keys.py    | 2 +-
 tests/handlers/test_e2e_room_keys.py | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py
index a43fc7fc7e5b..c09816b37274 100644
--- a/synapse/handlers/e2e_room_keys.py
+++ b/synapse/handlers/e2e_room_keys.py
@@ -18,7 +18,7 @@
 from twisted.internet import defer
 
 from synapse.api.errors import StoreError, SynapseError, RoomKeysVersionError
-from synapse.util.async import Linearizer
+from synapse.util.async_helpers import Linearizer
 
 logger = logging.getLogger(__name__)
 
diff --git a/tests/handlers/test_e2e_room_keys.py b/tests/handlers/test_e2e_room_keys.py
index 8bfffb5c0e51..7fa426444152 100644
--- a/tests/handlers/test_e2e_room_keys.py
+++ b/tests/handlers/test_e2e_room_keys.py
@@ -52,6 +52,7 @@ def __init__(self, *args, **kwargs):
     @defer.inlineCallbacks
     def setUp(self):
         self.hs = yield utils.setup_test_homeserver(
+            self.addCleanup,
             handlers=None,
             replication_layer=mock.Mock(),
         )

From 0b07f02e19e5ed1eba0e74c3863def9f849b5f9a Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 28 Aug 2018 13:39:49 +0100
Subject: [PATCH 517/830] Make sure that we close db connections opened during
 init

We should explicitly close any db connections we open, because failing to do so
can block other transactions as per
https://github.com/matrix-org/synapse/issues/3682.

Let's also try to factor out some of the boilerplate by having server classes
define their datastore class rather than duplicating the whole of `setup`.
---
 synapse/app/appservice.py        |  5 +----
 synapse/app/client_reader.py     |  5 +----
 synapse/app/event_creator.py     |  5 +----
 synapse/app/federation_reader.py |  5 +----
 synapse/app/federation_sender.py |  5 +----
 synapse/app/frontend_proxy.py    |  5 +----
 synapse/app/homeserver.py        | 14 ++++++++------
 synapse/app/media_repository.py  |  5 +----
 synapse/app/pusher.py            |  5 +----
 synapse/app/synchrotron.py       |  5 +----
 synapse/app/user_dir.py          |  5 +----
 synapse/server.py                | 15 +++++++++++++--
 12 files changed, 31 insertions(+), 48 deletions(-)

diff --git a/synapse/app/appservice.py b/synapse/app/appservice.py
index 3348a8ec6dbc..86b5067400e3 100644
--- a/synapse/app/appservice.py
+++ b/synapse/app/appservice.py
@@ -51,10 +51,7 @@ class AppserviceSlaveStore(
 
 
 class AppserviceServer(HomeServer):
-    def setup(self):
-        logger.info("Setting up.")
-        self.datastore = AppserviceSlaveStore(self.get_db_conn(), self)
-        logger.info("Finished setting up.")
+    DATASTORE_CLASS = AppserviceSlaveStore
 
     def _listen_http(self, listener_config):
         port = listener_config["port"]
diff --git a/synapse/app/client_reader.py b/synapse/app/client_reader.py
index ab79a456467c..ce2b113dbb97 100644
--- a/synapse/app/client_reader.py
+++ b/synapse/app/client_reader.py
@@ -74,10 +74,7 @@ class ClientReaderSlavedStore(
 
 
 class ClientReaderServer(HomeServer):
-    def setup(self):
-        logger.info("Setting up.")
-        self.datastore = ClientReaderSlavedStore(self.get_db_conn(), self)
-        logger.info("Finished setting up.")
+    DATASTORE_CLASS = ClientReaderSlavedStore
 
     def _listen_http(self, listener_config):
         port = listener_config["port"]
diff --git a/synapse/app/event_creator.py b/synapse/app/event_creator.py
index a34c89fa9924..f98e456ea0b5 100644
--- a/synapse/app/event_creator.py
+++ b/synapse/app/event_creator.py
@@ -90,10 +90,7 @@ class EventCreatorSlavedStore(
 
 
 class EventCreatorServer(HomeServer):
-    def setup(self):
-        logger.info("Setting up.")
-        self.datastore = EventCreatorSlavedStore(self.get_db_conn(), self)
-        logger.info("Finished setting up.")
+    DATASTORE_CLASS = EventCreatorSlavedStore
 
     def _listen_http(self, listener_config):
         port = listener_config["port"]
diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py
index 7d8105778d55..60f597350501 100644
--- a/synapse/app/federation_reader.py
+++ b/synapse/app/federation_reader.py
@@ -72,10 +72,7 @@ class FederationReaderSlavedStore(
 
 
 class FederationReaderServer(HomeServer):
-    def setup(self):
-        logger.info("Setting up.")
-        self.datastore = FederationReaderSlavedStore(self.get_db_conn(), self)
-        logger.info("Finished setting up.")
+    DATASTORE_CLASS = FederationReaderSlavedStore
 
     def _listen_http(self, listener_config):
         port = listener_config["port"]
diff --git a/synapse/app/federation_sender.py b/synapse/app/federation_sender.py
index d59007099bec..60dd09aac30c 100644
--- a/synapse/app/federation_sender.py
+++ b/synapse/app/federation_sender.py
@@ -78,10 +78,7 @@ def _get_federation_out_pos(self, db_conn):
 
 
 class FederationSenderServer(HomeServer):
-    def setup(self):
-        logger.info("Setting up.")
-        self.datastore = FederationSenderSlaveStore(self.get_db_conn(), self)
-        logger.info("Finished setting up.")
+    DATASTORE_CLASS = FederationSenderSlaveStore
 
     def _listen_http(self, listener_config):
         port = listener_config["port"]
diff --git a/synapse/app/frontend_proxy.py b/synapse/app/frontend_proxy.py
index 8d484c1cd4a0..8c0b9c67b07f 100644
--- a/synapse/app/frontend_proxy.py
+++ b/synapse/app/frontend_proxy.py
@@ -148,10 +148,7 @@ class FrontendProxySlavedStore(
 
 
 class FrontendProxyServer(HomeServer):
-    def setup(self):
-        logger.info("Setting up.")
-        self.datastore = FrontendProxySlavedStore(self.get_db_conn(), self)
-        logger.info("Finished setting up.")
+    DATASTORE_CLASS = FrontendProxySlavedStore
 
     def _listen_http(self, listener_config):
         port = listener_config["port"]
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 005921dcf733..3eb5b663de5b 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -62,7 +62,7 @@
 from synapse.rest.key.v2 import KeyApiV2Resource
 from synapse.rest.media.v0.content_repository import ContentRepoResource
 from synapse.server import HomeServer
-from synapse.storage import are_all_users_on_domain
+from synapse.storage import DataStore, are_all_users_on_domain
 from synapse.storage.engines import IncorrectDatabaseSetup, create_engine
 from synapse.storage.prepare_database import UpgradeDatabaseException, prepare_database
 from synapse.util.caches import CACHE_SIZE_FACTOR
@@ -111,6 +111,8 @@ def build_resource_for_web_client(hs):
 
 
 class SynapseHomeServer(HomeServer):
+    DATASTORE_CLASS = DataStore
+
     def _listener_http(self, config, listener_config):
         port = listener_config["port"]
         bind_addresses = listener_config["bind_addresses"]
@@ -356,13 +358,13 @@ def setup(config_options):
     logger.info("Preparing database: %s...", config.database_config['name'])
 
     try:
-        db_conn = hs.get_db_conn(run_new_connection=False)
-        prepare_database(db_conn, database_engine, config=config)
-        database_engine.on_new_connection(db_conn)
+        with hs.get_db_conn(run_new_connection=False) as db_conn:
+            prepare_database(db_conn, database_engine, config=config)
+            database_engine.on_new_connection(db_conn)
 
-        hs.run_startup_checks(db_conn, database_engine)
+            hs.run_startup_checks(db_conn, database_engine)
 
-        db_conn.commit()
+            db_conn.commit()
     except UpgradeDatabaseException:
         sys.stderr.write(
             "\nFailed to upgrade database.\n"
diff --git a/synapse/app/media_repository.py b/synapse/app/media_repository.py
index fd1f6cbf7ea6..e3dbb3b4e6bb 100644
--- a/synapse/app/media_repository.py
+++ b/synapse/app/media_repository.py
@@ -60,10 +60,7 @@ class MediaRepositorySlavedStore(
 
 
 class MediaRepositoryServer(HomeServer):
-    def setup(self):
-        logger.info("Setting up.")
-        self.datastore = MediaRepositorySlavedStore(self.get_db_conn(), self)
-        logger.info("Finished setting up.")
+    DATASTORE_CLASS = MediaRepositorySlavedStore
 
     def _listen_http(self, listener_config):
         port = listener_config["port"]
diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py
index a4fc7e91fa04..244c604de91c 100644
--- a/synapse/app/pusher.py
+++ b/synapse/app/pusher.py
@@ -78,10 +78,7 @@ class PusherSlaveStore(
 
 
 class PusherServer(HomeServer):
-    def setup(self):
-        logger.info("Setting up.")
-        self.datastore = PusherSlaveStore(self.get_db_conn(), self)
-        logger.info("Finished setting up.")
+    DATASTORE_CLASS = PusherSlaveStore
 
     def remove_pusher(self, app_id, push_key, user_id):
         self.get_tcp_replication().send_remove_pusher(app_id, push_key, user_id)
diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py
index 27e199866075..6662340797dc 100644
--- a/synapse/app/synchrotron.py
+++ b/synapse/app/synchrotron.py
@@ -249,10 +249,7 @@ def notify_interested_services(self, event):
 
 
 class SynchrotronServer(HomeServer):
-    def setup(self):
-        logger.info("Setting up.")
-        self.datastore = SynchrotronSlavedStore(self.get_db_conn(), self)
-        logger.info("Finished setting up.")
+    DATASTORE_CLASS = SynchrotronSlavedStore
 
     def _listen_http(self, listener_config):
         port = listener_config["port"]
diff --git a/synapse/app/user_dir.py b/synapse/app/user_dir.py
index 1388a42b59aa..96ffcaf073d4 100644
--- a/synapse/app/user_dir.py
+++ b/synapse/app/user_dir.py
@@ -94,10 +94,7 @@ def process_replication_rows(self, stream_name, token, rows):
 
 
 class UserDirectoryServer(HomeServer):
-    def setup(self):
-        logger.info("Setting up.")
-        self.datastore = UserDirectorySlaveStore(self.get_db_conn(), self)
-        logger.info("Finished setting up.")
+    DATASTORE_CLASS = UserDirectorySlaveStore
 
     def _listen_http(self, listener_config):
         port = listener_config["port"]
diff --git a/synapse/server.py b/synapse/server.py
index a6fbc6ec0c64..802d679cdef9 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -81,7 +81,6 @@
 from synapse.server_notices.server_notices_sender import ServerNoticesSender
 from synapse.server_notices.worker_server_notices_sender import WorkerServerNoticesSender
 from synapse.state import StateHandler, StateResolutionHandler
-from synapse.storage import DataStore
 from synapse.streams.events import EventSources
 from synapse.util import Clock
 from synapse.util.distributor import Distributor
@@ -172,6 +171,11 @@ def build_DEPENDENCY(self)
         'room_context_handler',
     ]
 
+    # This is overridden in derived application classes
+    # (such as synapse.app.homeserver.SynapseHomeServer) and gives the class to be
+    # instantiated during setup() for future return by get_datastore()
+    DATASTORE_CLASS = None
+
     def __init__(self, hostname, reactor=None, **kwargs):
         """
         Args:
@@ -188,13 +192,20 @@ def __init__(self, hostname, reactor=None, **kwargs):
         self.distributor = Distributor()
         self.ratelimiter = Ratelimiter()
 
+        self.datastore = None
+
         # Other kwargs are explicit dependencies
         for depname in kwargs:
             setattr(self, depname, kwargs[depname])
 
     def setup(self):
         logger.info("Setting up.")
-        self.datastore = DataStore(self.get_db_conn(), self)
+        if self.DATASTORE_CLASS is None:
+            raise RuntimeError("%s does not define a DATASTORE_CLASS" % (
+                self.__class__.__name__,
+            ))
+        with self.get_db_conn() as conn:
+            self.datastore = self.DATASTORE_CLASS(conn, self)
         logger.info("Finished setting up.")
 
     def get_reactor(self):

From 71990b3caeeb6061d4419af516271af7a2b7b429 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 28 Aug 2018 13:42:20 +0100
Subject: [PATCH 518/830] changelog

---
 changelog.d/3764.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3764.misc

diff --git a/changelog.d/3764.misc b/changelog.d/3764.misc
new file mode 100644
index 000000000000..f3614f1982e1
--- /dev/null
+++ b/changelog.d/3764.misc
@@ -0,0 +1 @@
+Make sure that we close db connections opened during init
\ No newline at end of file

From 32eb1dedd294831f2bafc79a24dae91eee319563 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 28 Aug 2018 17:10:43 +0100
Subject: [PATCH 519/830] use abc.abstractproperty

This gives clearer messages when someone gets it wrong
---
 synapse/server.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/synapse/server.py b/synapse/server.py
index 802d679cdef9..938a05f9dc06 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -19,6 +19,7 @@
 # partial one for unit test mocking.
 
 # Imports required for the default HomeServer() implementation
+import abc
 import logging
 
 from twisted.enterprise import adbapi
@@ -110,6 +111,8 @@ def build_DEPENDENCY(self)
         config (synapse.config.homeserver.HomeserverConfig):
     """
 
+    __metaclass__ = abc.ABCMeta
+
     DEPENDENCIES = [
         'http_client',
         'db_pool',
@@ -174,7 +177,7 @@ def build_DEPENDENCY(self)
     # This is overridden in derived application classes
     # (such as synapse.app.homeserver.SynapseHomeServer) and gives the class to be
     # instantiated during setup() for future return by get_datastore()
-    DATASTORE_CLASS = None
+    DATASTORE_CLASS = abc.abstractproperty()
 
     def __init__(self, hostname, reactor=None, **kwargs):
         """
@@ -200,10 +203,6 @@ def __init__(self, hostname, reactor=None, **kwargs):
 
     def setup(self):
         logger.info("Setting up.")
-        if self.DATASTORE_CLASS is None:
-            raise RuntimeError("%s does not define a DATASTORE_CLASS" % (
-                self.__class__.__name__,
-            ))
         with self.get_db_conn() as conn:
             self.datastore = self.DATASTORE_CLASS(conn, self)
         logger.info("Finished setting up.")

From 414fa36f3e22d87ea6ce4bdd6af54bce5a0de67c Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 28 Aug 2018 17:21:05 +0100
Subject: [PATCH 520/830] Fix up tests

---
 tests/storage/test_base.py | 4 ++--
 tests/test_types.py        | 4 ++--
 tests/utils.py             | 8 ++++++--
 3 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py
index 7cb5f0e4cfcb..829f47d2e8a1 100644
--- a/tests/storage/test_base.py
+++ b/tests/storage/test_base.py
@@ -20,11 +20,11 @@
 
 from twisted.internet import defer
 
-from synapse.server import HomeServer
 from synapse.storage._base import SQLBaseStore
 from synapse.storage.engines import create_engine
 
 from tests import unittest
+from tests.utils import TestHomeServer
 
 
 class SQLBaseStoreTestCase(unittest.TestCase):
@@ -51,7 +51,7 @@ def runWithConnection(func, *args, **kwargs):
         config = Mock()
         config.event_cache_size = 1
         config.database_config = {"name": "sqlite3"}
-        hs = HomeServer(
+        hs = TestHomeServer(
             "test",
             db_pool=self.db_pool,
             config=config,
diff --git a/tests/test_types.py b/tests/test_types.py
index be072d402b85..0f5c8bfaf91b 100644
--- a/tests/test_types.py
+++ b/tests/test_types.py
@@ -14,12 +14,12 @@
 # limitations under the License.
 
 from synapse.api.errors import SynapseError
-from synapse.server import HomeServer
 from synapse.types import GroupID, RoomAlias, UserID
 
 from tests import unittest
+from tests.utils import TestHomeServer
 
-mock_homeserver = HomeServer(hostname="my.domain")
+mock_homeserver = TestHomeServer(hostname="my.domain")
 
 
 class UserIDTestCase(unittest.TestCase):
diff --git a/tests/utils.py b/tests/utils.py
index e8ef10445c72..657c258a6eb3 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -29,7 +29,7 @@
 from synapse.federation.transport import server
 from synapse.http.server import HttpServer
 from synapse.server import HomeServer
-from synapse.storage import PostgresEngine
+from synapse.storage import DataStore, PostgresEngine
 from synapse.storage.engines import create_engine
 from synapse.storage.prepare_database import (
     _get_or_create_schema_state,
@@ -92,10 +92,14 @@ def _cleanup():
         atexit.register(_cleanup)
 
 
+class TestHomeServer(HomeServer):
+    DATASTORE_CLASS = DataStore
+
+
 @defer.inlineCallbacks
 def setup_test_homeserver(
     cleanup_func, name="test", datastore=None, config=None, reactor=None,
-    homeserverToUse=HomeServer, **kargs
+    homeserverToUse=TestHomeServer, **kargs
 ):
     """
     Setup a homeserver suitable for running tests against.  Keyword arguments

From b1580f50fea05ce02dc4663adb23704b33e30bf3 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@arasphere.net>
Date: Tue, 28 Aug 2018 23:25:58 +0100
Subject: [PATCH 521/830] don't return non-LL-member state in incremental sync
 state blocks (#3760)

don't return non-LL-member state in incremental sync state blocks
---
 changelog.d/3760.bugfix  | 1 +
 synapse/handlers/sync.py | 9 ++++++++-
 2 files changed, 9 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/3760.bugfix

diff --git a/changelog.d/3760.bugfix b/changelog.d/3760.bugfix
new file mode 100644
index 000000000000..ce61fb8a2bc8
--- /dev/null
+++ b/changelog.d/3760.bugfix
@@ -0,0 +1 @@
+Don't return non-LL-member state in incremental sync state blocks
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 648debc8aa96..ef20c2296c18 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -745,9 +745,16 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                 state_ids = {}
                 if lazy_load_members:
                     if types:
+                        # We're returning an incremental sync, with no "gap" since
+                        # the previous sync, so normally there would be no state to return
+                        # But we're lazy-loading, so the client might need some more
+                        # member events to understand the events in this timeline.
+                        # So we fish out all the member events corresponding to the
+                        # timeline here, and then dedupe any redundant ones below.
+
                         state_ids = yield self.store.get_state_ids_for_event(
                             batch.events[0].event_id, types=types,
-                            filtered_types=filtered_types,
+                            filtered_types=None,  # we only want members!
                         )
 
             if lazy_load_members and not include_redundant_members:

From 82276a18d10e8b66a509344a94cc328584cee46f Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Wed, 29 Aug 2018 19:06:59 +1000
Subject: [PATCH 522/830] Update CONTRIBUTING to clarify miscs & Markdown
 (#3730)

---
 CONTRIBUTING.rst      | 7 ++++---
 changelog.d/3730.misc | 1 +
 2 files changed, 5 insertions(+), 3 deletions(-)
 create mode 100644 changelog.d/3730.misc

diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
index 3d75853aa7e2..f9de78a4602f 100644
--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -59,9 +59,10 @@ To create a changelog entry, make a new file in the ``changelog.d``
 file named in the format of ``PRnumber.type``. The type can be
 one of ``feature``, ``bugfix``, ``removal`` (also used for
 deprecations), or ``misc`` (for internal-only changes). The content of
-the file is your changelog entry, which can contain RestructuredText
-formatting. A note of contributors is welcomed in changelogs for
-non-misc changes (the content of misc changes is not displayed).
+the file is your changelog entry, which can contain Markdown
+formatting. Adding credits to the changelog is encouraged, we value
+your contributions and would like to have you shouted out in the
+release notes!
 
 For example, a fix in PR #1234 would have its changelog entry in
 ``changelog.d/1234.bugfix``, and contain content like "The security levels of
diff --git a/changelog.d/3730.misc b/changelog.d/3730.misc
new file mode 100644
index 000000000000..b1ea84f732a2
--- /dev/null
+++ b/changelog.d/3730.misc
@@ -0,0 +1 @@
+The CONTRIBUTING guidelines have been updated to mention our use of Markdown and that .misc files have content.

From 79a8a347a6c55a3ec6255227930dd9aad4b0b78f Mon Sep 17 00:00:00 2001
From: Krombel <krombel@krombel.de>
Date: Wed, 29 Aug 2018 16:26:21 +0200
Subject: [PATCH 523/830] fix #3445

itervalues(d) calls d.itervalues() [PY2] and d.values() [PY3]
but SortedDict only implements d.values()
---
 changelog.d/3445.bugfix          | 1 +
 synapse/federation/send_queue.py | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)
 create mode 100644 changelog.d/3445.bugfix

diff --git a/changelog.d/3445.bugfix b/changelog.d/3445.bugfix
new file mode 100644
index 000000000000..f7f1c4778d34
--- /dev/null
+++ b/changelog.d/3445.bugfix
@@ -0,0 +1 @@
+do not use six.itervalues() on SortedDict()
diff --git a/synapse/federation/send_queue.py b/synapse/federation/send_queue.py
index 0bb468385d0c..6f5995735aa2 100644
--- a/synapse/federation/send_queue.py
+++ b/synapse/federation/send_queue.py
@@ -32,7 +32,7 @@
 import logging
 from collections import namedtuple
 
-from six import iteritems, itervalues
+from six import iteritems
 
 from sortedcontainers import SortedDict
 
@@ -117,7 +117,7 @@ def _clear_queue_before_pos(self, position_to_delete):
 
             user_ids = set(
                 user_id
-                for uids in itervalues(self.presence_changed)
+                for uids in self.presence_changed.values()
                 for user_id in uids
             )
 

From 8c0c51ecb337c4dbbec371375e9c376fb2f666f7 Mon Sep 17 00:00:00 2001
From: Krombel <krombel@krombel.de>
Date: Wed, 29 Aug 2018 16:49:26 +0200
Subject: [PATCH 524/830] changelog

---
 changelog.d/3445.bugfix | 1 -
 changelog.d/3768.bugfix | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)
 delete mode 100644 changelog.d/3445.bugfix
 create mode 100644 changelog.d/3768.bugfix

diff --git a/changelog.d/3445.bugfix b/changelog.d/3445.bugfix
deleted file mode 100644
index f7f1c4778d34..000000000000
--- a/changelog.d/3445.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-do not use six.itervalues() on SortedDict()
diff --git a/changelog.d/3768.bugfix b/changelog.d/3768.bugfix
new file mode 100644
index 000000000000..a039a7fa685b
--- /dev/null
+++ b/changelog.d/3768.bugfix
@@ -0,0 +1 @@
+Fix bug in sending presence over federation

From 14e4d4f4bf894f4e70118e03f4f4575e1eb6dab6 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Fri, 31 Aug 2018 00:19:58 +1000
Subject: [PATCH 525/830] Port storage/ to Python 3 (#3725)

---
 changelog.d/3725.misc               |   1 +
 jenkins/prepare_synapse.sh          |   2 +-
 synapse/python_dependencies.py      |   3 +
 synapse/storage/_base.py            |  32 ++++++++-
 synapse/storage/deviceinbox.py      |   2 +-
 synapse/storage/devices.py          |  10 +--
 synapse/storage/end_to_end_keys.py  |   6 +-
 synapse/storage/engines/postgres.py |   9 ++-
 synapse/storage/events.py           |  10 +--
 synapse/storage/events_worker.py    |   9 +--
 synapse/storage/filtering.py        |   6 +-
 synapse/storage/pusher.py           |  14 ++--
 synapse/storage/transactions.py     |   7 +-
 tests/rest/client/v1/utils.py       |  14 ++--
 tests/storage/test_purge.py         | 106 ++++++++++++++++++++++++++++
 tests/unittest.py                   |  11 +++
 tests/utils.py                      |   2 +
 17 files changed, 208 insertions(+), 36 deletions(-)
 create mode 100644 changelog.d/3725.misc
 create mode 100644 tests/storage/test_purge.py

diff --git a/changelog.d/3725.misc b/changelog.d/3725.misc
new file mode 100644
index 000000000000..91ab9d7137b3
--- /dev/null
+++ b/changelog.d/3725.misc
@@ -0,0 +1 @@
+The synapse.storage module has been ported to Python 3.
diff --git a/jenkins/prepare_synapse.sh b/jenkins/prepare_synapse.sh
index a30179f2aad7..d95ca846c414 100755
--- a/jenkins/prepare_synapse.sh
+++ b/jenkins/prepare_synapse.sh
@@ -31,5 +31,5 @@ $TOX_BIN/pip install 'setuptools>=18.5'
 $TOX_BIN/pip install 'pip>=10'
 
 { python synapse/python_dependencies.py
-  echo lxml psycopg2
+  echo lxml
 } | xargs $TOX_BIN/pip install
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index 9c55e79ef5fe..942d7c721f0e 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -78,6 +78,9 @@
     "affinity": {
         "affinity": ["affinity"],
     },
+    "postgres": {
+        "psycopg2>=2.6": ["psycopg2"]
+    }
 }
 
 
diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 08dffd774f81..be61147b9b8d 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -17,9 +17,10 @@
 import threading
 import time
 
-from six import iteritems, iterkeys, itervalues
+from six import PY2, iteritems, iterkeys, itervalues
 from six.moves import intern, range
 
+from canonicaljson import json
 from prometheus_client import Histogram
 
 from twisted.internet import defer
@@ -1216,3 +1217,32 @@ class _RollbackButIsFineException(Exception):
     something went wrong.
     """
     pass
+
+
+def db_to_json(db_content):
+    """
+    Take some data from a database row and return a JSON-decoded object.
+
+    Args:
+        db_content (memoryview|buffer|bytes|bytearray|unicode)
+    """
+    # psycopg2 on Python 3 returns memoryview objects, which we need to
+    # cast to bytes to decode
+    if isinstance(db_content, memoryview):
+        db_content = db_content.tobytes()
+
+    # psycopg2 on Python 2 returns buffer objects, which we need to cast to
+    # bytes to decode
+    if PY2 and isinstance(db_content, buffer):
+        db_content = bytes(db_content)
+
+    # Decode it to a Unicode string before feeding it to json.loads, so we
+    # consistenty get a Unicode-containing object out.
+    if isinstance(db_content, (bytes, bytearray)):
+        db_content = db_content.decode('utf8')
+
+    try:
+        return json.loads(db_content)
+    except Exception:
+        logging.warning("Tried to decode '%r' as JSON and failed", db_content)
+        raise
diff --git a/synapse/storage/deviceinbox.py b/synapse/storage/deviceinbox.py
index 73646da025c4..e06b0bc56dee 100644
--- a/synapse/storage/deviceinbox.py
+++ b/synapse/storage/deviceinbox.py
@@ -169,7 +169,7 @@ def _add_messages_to_local_device_inbox_txn(self, txn, stream_id,
         local_by_user_then_device = {}
         for user_id, messages_by_device in messages_by_user_then_device.items():
             messages_json_for_user = {}
-            devices = messages_by_device.keys()
+            devices = list(messages_by_device.keys())
             if len(devices) == 1 and devices[0] == "*":
                 # Handle wildcard device_ids.
                 sql = (
diff --git a/synapse/storage/devices.py b/synapse/storage/devices.py
index c0943ecf912a..d10ff9e4b982 100644
--- a/synapse/storage/devices.py
+++ b/synapse/storage/devices.py
@@ -24,7 +24,7 @@
 from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.util.caches.descriptors import cached, cachedInlineCallbacks, cachedList
 
-from ._base import Cache, SQLBaseStore
+from ._base import Cache, SQLBaseStore, db_to_json
 
 logger = logging.getLogger(__name__)
 
@@ -411,7 +411,7 @@ def _get_devices_by_remote_txn(self, txn, destination, from_stream_id,
                 if device is not None:
                     key_json = device.get("key_json", None)
                     if key_json:
-                        result["keys"] = json.loads(key_json)
+                        result["keys"] = db_to_json(key_json)
                     device_display_name = device.get("device_display_name", None)
                     if device_display_name:
                         result["device_display_name"] = device_display_name
@@ -466,7 +466,7 @@ def _get_cached_user_device(self, user_id, device_id):
             retcol="content",
             desc="_get_cached_user_device",
         )
-        defer.returnValue(json.loads(content))
+        defer.returnValue(db_to_json(content))
 
     @cachedInlineCallbacks()
     def _get_cached_devices_for_user(self, user_id):
@@ -479,7 +479,7 @@ def _get_cached_devices_for_user(self, user_id):
             desc="_get_cached_devices_for_user",
         )
         defer.returnValue({
-            device["device_id"]: json.loads(device["content"])
+            device["device_id"]: db_to_json(device["content"])
             for device in devices
         })
 
@@ -511,7 +511,7 @@ def _get_devices_with_keys_by_user_txn(self, txn, user_id):
 
                 key_json = device.get("key_json", None)
                 if key_json:
-                    result["keys"] = json.loads(key_json)
+                    result["keys"] = db_to_json(key_json)
                 device_display_name = device.get("device_display_name", None)
                 if device_display_name:
                     result["device_display_name"] = device_display_name
diff --git a/synapse/storage/end_to_end_keys.py b/synapse/storage/end_to_end_keys.py
index 523b4360c3cb..1f1721e82083 100644
--- a/synapse/storage/end_to_end_keys.py
+++ b/synapse/storage/end_to_end_keys.py
@@ -14,13 +14,13 @@
 # limitations under the License.
 from six import iteritems
 
-from canonicaljson import encode_canonical_json, json
+from canonicaljson import encode_canonical_json
 
 from twisted.internet import defer
 
 from synapse.util.caches.descriptors import cached
 
-from ._base import SQLBaseStore
+from ._base import SQLBaseStore, db_to_json
 
 
 class EndToEndKeyStore(SQLBaseStore):
@@ -90,7 +90,7 @@ def get_e2e_device_keys(
 
         for user_id, device_keys in iteritems(results):
             for device_id, device_info in iteritems(device_keys):
-                device_info["keys"] = json.loads(device_info.pop("key_json"))
+                device_info["keys"] = db_to_json(device_info.pop("key_json"))
 
         defer.returnValue(results)
 
diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py
index 8a0386c1a41e..42225f8a2a7e 100644
--- a/synapse/storage/engines/postgres.py
+++ b/synapse/storage/engines/postgres.py
@@ -41,13 +41,18 @@ def on_new_connection(self, db_conn):
         db_conn.set_isolation_level(
             self.module.extensions.ISOLATION_LEVEL_REPEATABLE_READ
         )
+
+        # Set the bytea output to escape, vs the default of hex
+        cursor = db_conn.cursor()
+        cursor.execute("SET bytea_output TO escape")
+
         # Asynchronous commit, don't wait for the server to call fsync before
         # ending the transaction.
         # https://www.postgresql.org/docs/current/static/wal-async-commit.html
         if not self.synchronous_commit:
-            cursor = db_conn.cursor()
             cursor.execute("SET synchronous_commit TO OFF")
-            cursor.close()
+
+        cursor.close()
 
     def is_deadlock(self, error):
         if isinstance(error, self.module.DatabaseError):
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index f39c8c846179..8bf87f38f7d6 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -19,7 +19,7 @@
 from collections import OrderedDict, deque, namedtuple
 from functools import wraps
 
-from six import iteritems
+from six import iteritems, text_type
 from six.moves import range
 
 from canonicaljson import json
@@ -1220,7 +1220,7 @@ def event_dict(event):
                     "sender": event.sender,
                     "contains_url": (
                         "url" in event.content
-                        and isinstance(event.content["url"], basestring)
+                        and isinstance(event.content["url"], text_type)
                     ),
                 }
                 for event, _ in events_and_contexts
@@ -1529,7 +1529,7 @@ def reindex_txn(txn):
 
                     contains_url = "url" in content
                     if contains_url:
-                        contains_url &= isinstance(content["url"], basestring)
+                        contains_url &= isinstance(content["url"], text_type)
                 except (KeyError, AttributeError):
                     # If the event is missing a necessary field then
                     # skip over it.
@@ -1910,9 +1910,9 @@ def _purge_history_txn(
             (room_id,)
         )
         rows = txn.fetchall()
-        max_depth = max(row[0] for row in rows)
+        max_depth = max(row[1] for row in rows)
 
-        if max_depth <= token.topological:
+        if max_depth < token.topological:
             # We need to ensure we don't delete all the events from the database
             # otherwise we wouldn't be able to send any events (due to not
             # having any backwards extremeties)
diff --git a/synapse/storage/events_worker.py b/synapse/storage/events_worker.py
index 59822178ff02..a8326f5296a6 100644
--- a/synapse/storage/events_worker.py
+++ b/synapse/storage/events_worker.py
@@ -12,6 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 import itertools
 import logging
 from collections import namedtuple
@@ -265,7 +266,7 @@ def _fetch_event_list(self, conn, event_list):
         """
         with Measure(self._clock, "_fetch_event_list"):
             try:
-                event_id_lists = zip(*event_list)[0]
+                event_id_lists = list(zip(*event_list))[0]
                 event_ids = [
                     item for sublist in event_id_lists for item in sublist
                 ]
@@ -299,14 +300,14 @@ def fire(lst, res):
                 logger.exception("do_fetch")
 
                 # We only want to resolve deferreds from the main thread
-                def fire(evs):
+                def fire(evs, exc):
                     for _, d in evs:
                         if not d.called:
                             with PreserveLoggingContext():
-                                d.errback(e)
+                                d.errback(exc)
 
                 with PreserveLoggingContext():
-                    self.hs.get_reactor().callFromThread(fire, event_list)
+                    self.hs.get_reactor().callFromThread(fire, event_list, e)
 
     @defer.inlineCallbacks
     def _enqueue_events(self, events, check_redacted=True, allow_rejected=False):
diff --git a/synapse/storage/filtering.py b/synapse/storage/filtering.py
index 2d5896c5b4e1..6ddcc909bfd0 100644
--- a/synapse/storage/filtering.py
+++ b/synapse/storage/filtering.py
@@ -13,14 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from canonicaljson import encode_canonical_json, json
+from canonicaljson import encode_canonical_json
 
 from twisted.internet import defer
 
 from synapse.api.errors import Codes, SynapseError
 from synapse.util.caches.descriptors import cachedInlineCallbacks
 
-from ._base import SQLBaseStore
+from ._base import SQLBaseStore, db_to_json
 
 
 class FilteringStore(SQLBaseStore):
@@ -44,7 +44,7 @@ def get_user_filter(self, user_localpart, filter_id):
             desc="get_user_filter",
         )
 
-        defer.returnValue(json.loads(bytes(def_json).decode("utf-8")))
+        defer.returnValue(db_to_json(def_json))
 
     def add_user_filter(self, user_localpart, user_filter):
         def_json = encode_canonical_json(user_filter)
diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py
index 8443bd4c1b70..c7987bfcdd6e 100644
--- a/synapse/storage/pusher.py
+++ b/synapse/storage/pusher.py
@@ -15,7 +15,8 @@
 # limitations under the License.
 
 import logging
-import types
+
+import six
 
 from canonicaljson import encode_canonical_json, json
 
@@ -27,6 +28,11 @@
 
 logger = logging.getLogger(__name__)
 
+if six.PY2:
+    db_binary_type = buffer
+else:
+    db_binary_type = memoryview
+
 
 class PusherWorkerStore(SQLBaseStore):
     def _decode_pushers_rows(self, rows):
@@ -34,18 +40,18 @@ def _decode_pushers_rows(self, rows):
             dataJson = r['data']
             r['data'] = None
             try:
-                if isinstance(dataJson, types.BufferType):
+                if isinstance(dataJson, db_binary_type):
                     dataJson = str(dataJson).decode("UTF8")
 
                 r['data'] = json.loads(dataJson)
             except Exception as e:
                 logger.warn(
                     "Invalid JSON in data for pusher %d: %s, %s",
-                    r['id'], dataJson, e.message,
+                    r['id'], dataJson, e.args[0],
                 )
                 pass
 
-            if isinstance(r['pushkey'], types.BufferType):
+            if isinstance(r['pushkey'], db_binary_type):
                 r['pushkey'] = str(r['pushkey']).decode("UTF8")
 
         return rows
diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py
index 428e7fa36e37..0c42bd332227 100644
--- a/synapse/storage/transactions.py
+++ b/synapse/storage/transactions.py
@@ -18,14 +18,14 @@
 
 import six
 
-from canonicaljson import encode_canonical_json, json
+from canonicaljson import encode_canonical_json
 
 from twisted.internet import defer
 
 from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.util.caches.descriptors import cached
 
-from ._base import SQLBaseStore
+from ._base import SQLBaseStore, db_to_json
 
 # py2 sqlite has buffer hardcoded as only binary type, so we must use it,
 # despite being deprecated and removed in favor of memoryview
@@ -95,7 +95,8 @@ def _get_received_txn_response(self, txn, transaction_id, origin):
         )
 
         if result and result["response_code"]:
-            return result["response_code"], json.loads(str(result["response_json"]))
+            return result["response_code"], db_to_json(result["response_json"])
+
         else:
             return None
 
diff --git a/tests/rest/client/v1/utils.py b/tests/rest/client/v1/utils.py
index 40dc4ea256da..530dc8ba6d92 100644
--- a/tests/rest/client/v1/utils.py
+++ b/tests/rest/client/v1/utils.py
@@ -240,7 +240,6 @@ def register(self, user_id):
         self.assertEquals(200, code)
         defer.returnValue(response)
 
-    @defer.inlineCallbacks
     def send(self, room_id, body=None, txn_id=None, tok=None, expect_code=200):
         if txn_id is None:
             txn_id = "m%s" % (str(time.time()))
@@ -248,9 +247,16 @@ def send(self, room_id, body=None, txn_id=None, tok=None, expect_code=200):
             body = "body_text_here"
 
         path = "/_matrix/client/r0/rooms/%s/send/m.room.message/%s" % (room_id, txn_id)
-        content = '{"msgtype":"m.text","body":"%s"}' % body
+        content = {"msgtype": "m.text", "body": body}
         if tok:
             path = path + "?access_token=%s" % tok
 
-        (code, response) = yield self.mock_resource.trigger("PUT", path, content)
-        self.assertEquals(expect_code, code, msg=str(response))
+        request, channel = make_request("PUT", path, json.dumps(content).encode('utf8'))
+        render(request, self.resource, self.hs.get_reactor())
+
+        assert int(channel.result["code"]) == expect_code, (
+            "Expected: %d, got: %d, resp: %r"
+            % (expect_code, int(channel.result["code"]), channel.result["body"])
+        )
+
+        return channel.json_body
diff --git a/tests/storage/test_purge.py b/tests/storage/test_purge.py
new file mode 100644
index 000000000000..f671599cb89a
--- /dev/null
+++ b/tests/storage/test_purge.py
@@ -0,0 +1,106 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from synapse.rest.client.v1 import room
+
+from tests.unittest import HomeserverTestCase
+
+
+class PurgeTests(HomeserverTestCase):
+
+    user_id = "@red:server"
+    servlets = [room.register_servlets]
+
+    def make_homeserver(self, reactor, clock):
+        hs = self.setup_test_homeserver("server", http_client=None)
+        return hs
+
+    def prepare(self, reactor, clock, hs):
+        self.room_id = self.helper.create_room_as(self.user_id)
+
+    def test_purge(self):
+        """
+        Purging a room will delete everything before the topological point.
+        """
+        # Send four messages to the room
+        first = self.helper.send(self.room_id, body="test1")
+        second = self.helper.send(self.room_id, body="test2")
+        third = self.helper.send(self.room_id, body="test3")
+        last = self.helper.send(self.room_id, body="test4")
+
+        storage = self.hs.get_datastore()
+
+        # Get the topological token
+        event = storage.get_topological_token_for_event(last["event_id"])
+        self.pump()
+        event = self.successResultOf(event)
+
+        # Purge everything before this topological token
+        purge = storage.purge_history(self.room_id, event, True)
+        self.pump()
+        self.assertEqual(self.successResultOf(purge), None)
+
+        # Try and get the events
+        get_first = storage.get_event(first["event_id"])
+        get_second = storage.get_event(second["event_id"])
+        get_third = storage.get_event(third["event_id"])
+        get_last = storage.get_event(last["event_id"])
+        self.pump()
+
+        # 1-3 should fail and last will succeed, meaning that 1-3 are deleted
+        # and last is not.
+        self.failureResultOf(get_first)
+        self.failureResultOf(get_second)
+        self.failureResultOf(get_third)
+        self.successResultOf(get_last)
+
+    def test_purge_wont_delete_extrems(self):
+        """
+        Purging a room will delete everything before the topological point.
+        """
+        # Send four messages to the room
+        first = self.helper.send(self.room_id, body="test1")
+        second = self.helper.send(self.room_id, body="test2")
+        third = self.helper.send(self.room_id, body="test3")
+        last = self.helper.send(self.room_id, body="test4")
+
+        storage = self.hs.get_datastore()
+
+        # Set the topological token higher than it should be
+        event = storage.get_topological_token_for_event(last["event_id"])
+        self.pump()
+        event = self.successResultOf(event)
+        event = "t{}-{}".format(
+            *list(map(lambda x: x + 1, map(int, event[1:].split("-"))))
+        )
+
+        # Purge everything before this topological token
+        purge = storage.purge_history(self.room_id, event, True)
+        self.pump()
+        f = self.failureResultOf(purge)
+        self.assertIn("greater than forward", f.value.args[0])
+
+        # Try and get the events
+        get_first = storage.get_event(first["event_id"])
+        get_second = storage.get_event(second["event_id"])
+        get_third = storage.get_event(third["event_id"])
+        get_last = storage.get_event(last["event_id"])
+        self.pump()
+
+        # Nothing is deleted.
+        self.successResultOf(get_first)
+        self.successResultOf(get_second)
+        self.successResultOf(get_third)
+        self.successResultOf(get_last)
diff --git a/tests/unittest.py b/tests/unittest.py
index d852e2465a06..8b513bb32b28 100644
--- a/tests/unittest.py
+++ b/tests/unittest.py
@@ -151,6 +151,7 @@ class HomeserverTestCase(TestCase):
         hijack_auth (bool): Whether to hijack auth to return the user specified
         in user_id.
     """
+
     servlets = []
     hijack_auth = True
 
@@ -279,3 +280,13 @@ def setup_test_homeserver(self, *args, **kwargs):
         kwargs = dict(kwargs)
         kwargs.update(self._hs_args)
         return setup_test_homeserver(self.addCleanup, *args, **kwargs)
+
+    def pump(self):
+        """
+        Pump the reactor enough that Deferreds will fire.
+        """
+        self.reactor.pump([0.0] * 100)
+
+    def get_success(self, d):
+        self.pump()
+        return self.successResultOf(d)
diff --git a/tests/utils.py b/tests/utils.py
index 657c258a6eb3..179b592501ce 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -147,6 +147,8 @@ def setup_test_homeserver(
         config.max_mau_value = 50
         config.mau_limits_reserved_threepids = []
         config.admin_contact = None
+        config.rc_messages_per_second = 10000
+        config.rc_message_burst_count = 10000
 
         # we need a sane default_room_version, otherwise attempts to create rooms will
         # fail.

From ea068d6f3cd5ed1bc9a39b2fd43e19d6d40f18da Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Fri, 31 Aug 2018 10:49:14 +0100
Subject: [PATCH 526/830] fix bug where preserved threepid user comes to sign
 up and server is mau blocked

---
 synapse/api/auth.py                      | 10 +++++++++-
 synapse/handlers/register.py             |  3 ++-
 synapse/rest/client/v1_only/register.py  |  6 +++++-
 synapse/rest/client/v2_alpha/register.py |  5 +++++
 tests/api/test_auth.py                   | 17 +++++++++++++++++
 5 files changed, 38 insertions(+), 3 deletions(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index a7e3f7a7ac87..9c207b9537e9 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -775,7 +775,7 @@ def check_in_room_or_world_readable(self, room_id, user_id):
             )
 
     @defer.inlineCallbacks
-    def check_auth_blocking(self, user_id=None):
+    def check_auth_blocking(self, user_id=None, threepid=None):
         """Checks if the user should be rejected for some external reason,
         such as monthly active user limiting or global disable flag
 
@@ -806,6 +806,14 @@ def check_auth_blocking(self, user_id=None):
                 is_trial = yield self.store.is_trial_user(user_id)
                 if is_trial:
                     return
+            elif threepid:
+                # If the user does not exist yet, but is signing up with a
+                # reserved threepid then pass auth check
+                for tp in self.hs.config.mau_limits_reserved_threepids:
+                    if (threepid['medium'] == tp['medium']
+                            and threepid['address'] == tp['address']):
+                        return
+
             # Else if there is no room in the MAU bucket, bail
             current_mau = yield self.store.get_monthly_active_count()
             if current_mau >= self.hs.config.max_mau_value:
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index f03ee1476bac..1e53f2c635c3 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -125,6 +125,7 @@ def register(
         guest_access_token=None,
         make_guest=False,
         admin=False,
+        threepid=None,
     ):
         """Registers a new client on the server.
 
@@ -145,7 +146,7 @@ def register(
             RegistrationError if there was a problem registering.
         """
 
-        yield self.auth.check_auth_blocking()
+        yield self.auth.check_auth_blocking(threepid=threepid)
         password_hash = None
         if password:
             password_hash = yield self.auth_handler().hash(password)
diff --git a/synapse/rest/client/v1_only/register.py b/synapse/rest/client/v1_only/register.py
index 5e99cffbcbd1..2c7bbcb1715b 100644
--- a/synapse/rest/client/v1_only/register.py
+++ b/synapse/rest/client/v1_only/register.py
@@ -281,11 +281,15 @@ def _do_password(self, request, register_json, session):
             register_json["user"].encode("utf-8")
             if "user" in register_json else None
         )
+        threepid = None
+        if session[LoginType.EMAIL_IDENTITY]:
+            threepid = session["threepidCreds"]
 
         handler = self.handlers.registration_handler
         (user_id, token) = yield handler.register(
             localpart=desired_user_id,
-            password=password
+            password=password,
+            threepid=threepid,
         )
 
         if session[LoginType.EMAIL_IDENTITY]:
diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index 2f64155d1316..45113e53867c 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -395,11 +395,16 @@ def on_POST(self, request):
             if desired_username is not None:
                 desired_username = desired_username.lower()
 
+            threepid = None
+            if auth_result:
+                threepid = auth_result.get(LoginType.EMAIL_IDENTITY)
+
             (registered_user_id, _) = yield self.registration_handler.register(
                 localpart=desired_username,
                 password=new_password,
                 guest_access_token=guest_access_token,
                 generate_token=False,
+                threepid=threepid,
             )
 
             # remember that we've now registered that user account, and with
diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py
index 54e396d19d5b..f65a27e5f1ff 100644
--- a/tests/api/test_auth.py
+++ b/tests/api/test_auth.py
@@ -467,6 +467,23 @@ def test_blocking_mau(self):
         )
         yield self.auth.check_auth_blocking()
 
+    @defer.inlineCallbacks
+    def test_reserved_threepid(self):
+        self.hs.config.limit_usage_by_mau = True
+        self.hs.config.max_mau_value = 1
+        threepid = {'medium': 'email', 'address': 'reserved@server.com'}
+        unknown_threepid = {'medium': 'email', 'address': 'unreserved@server.com'}
+        self.hs.config.mau_limits_reserved_threepids = [threepid]
+
+        yield self.store.register(user_id='user1', token="123", password_hash=None)
+        with self.assertRaises(ResourceLimitError):
+            yield self.auth.check_auth_blocking()
+
+        with self.assertRaises(ResourceLimitError):
+            yield self.auth.check_auth_blocking(threepid=unknown_threepid)
+
+        yield self.auth.check_auth_blocking(threepid=threepid)
+
     @defer.inlineCallbacks
     def test_hs_disabled(self):
         self.hs.config.hs_disabled = True

From 3d6aa06577e170a8af5f0e19a3102d4210730c8f Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Fri, 31 Aug 2018 10:56:37 +0100
Subject: [PATCH 527/830] news fragemnt

---
 changelog.d/3777.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3777.bugfix

diff --git a/changelog.d/3777.bugfix b/changelog.d/3777.bugfix
new file mode 100644
index 000000000000..93b5b8fa57cb
--- /dev/null
+++ b/changelog.d/3777.bugfix
@@ -0,0 +1 @@
+Fix fix bug where preserved threepid user comes to sign up and server is mau blocked

From 09f3cf1a7ef0c533d052a5c87257503b710093c6 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Fri, 31 Aug 2018 15:42:51 +0100
Subject: [PATCH 528/830] ensure post registration auth checks do not fail
 erroneously

---
 synapse/api/auth.py                      |  7 ++-----
 synapse/rest/client/v1_only/register.py  |  4 ++++
 synapse/rest/client/v2_alpha/register.py |  4 ++++
 synapse/storage/monthly_active_users.py  | 15 ++++++++++++++-
 4 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 9c207b9537e9..6a97c0611095 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -809,11 +809,8 @@ def check_auth_blocking(self, user_id=None, threepid=None):
             elif threepid:
                 # If the user does not exist yet, but is signing up with a
                 # reserved threepid then pass auth check
-                for tp in self.hs.config.mau_limits_reserved_threepids:
-                    if (threepid['medium'] == tp['medium']
-                            and threepid['address'] == tp['address']):
-                        return
-
+                if is_threepid_reserved(threepid):
+                    return
             # Else if there is no room in the MAU bucket, bail
             current_mau = yield self.store.get_monthly_active_count()
             if current_mau >= self.hs.config.max_mau_value:
diff --git a/synapse/rest/client/v1_only/register.py b/synapse/rest/client/v1_only/register.py
index 2c7bbcb1715b..95873e03d541 100644
--- a/synapse/rest/client/v1_only/register.py
+++ b/synapse/rest/client/v1_only/register.py
@@ -291,6 +291,10 @@ def _do_password(self, request, register_json, session):
             password=password,
             threepid=threepid,
         )
+        # Necessary due to auth checks prior to the threepid being
+        # written to the db
+        if self.store.is_threepid_reserved(threepid):
+            self.store.upsert_monthly_active_user(registered_user_id)
 
         if session[LoginType.EMAIL_IDENTITY]:
             logger.debug("Binding emails %s to %s" % (
diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index 45113e53867c..f22b7577ea06 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -406,6 +406,10 @@ def on_POST(self, request):
                 generate_token=False,
                 threepid=threepid,
             )
+            # Necessary due to auth checks prior to the threepid being
+            # written to the db
+            if self.store.is_threepid_reserved(threepid):
+                self.store.upsert_monthly_active_user(registered_user_id)
 
             # remember that we've now registered that user account, and with
             #  what user ID (since the user may not have specified)
diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py
index d178f5c5badd..173867c4b173 100644
--- a/synapse/storage/monthly_active_users.py
+++ b/synapse/storage/monthly_active_users.py
@@ -36,7 +36,6 @@ def __init__(self, dbconn, hs):
 
     @defer.inlineCallbacks
     def initialise_reserved_users(self, threepids):
-        # TODO Why can't I do this in init?
         store = self.hs.get_datastore()
         reserved_user_list = []
 
@@ -220,3 +219,17 @@ def populate_monthly_active_users(self, user_id):
                     yield self.upsert_monthly_active_user(user_id)
             elif now - last_seen_timestamp > LAST_SEEN_GRANULARITY:
                 yield self.upsert_monthly_active_user(user_id)
+
+    def is_threepid_reserved(self, threepid):
+        """Check the threepid against the reserved threepid config
+        Args:
+            threepid(dict) - The threepid to test for
+        Returns:
+            boolean Is the threepid undertest reserved_user
+        """
+        for tp in self.hs.config.mau_limits_reserved_threepids:
+            if (threepid['medium'] == tp['medium']
+                    and threepid['address'] == tp['address']):
+                return True
+            else:
+                return False

From a796bdd35efbd17c4a1fa48e1245f6d57403b232 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Fri, 31 Aug 2018 15:57:33 +0100
Subject: [PATCH 529/830] typo

---
 changelog.d/3777.bugfix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/changelog.d/3777.bugfix b/changelog.d/3777.bugfix
index 93b5b8fa57cb..46efc543a7b4 100644
--- a/changelog.d/3777.bugfix
+++ b/changelog.d/3777.bugfix
@@ -1 +1 @@
-Fix fix bug where preserved threepid user comes to sign up and server is mau blocked
+Fix bug where preserved threepid user comes to sign up and server is mau blocked

From e8e540630ec96d6ed856fb143dbb62b91e15084d Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Fri, 31 Aug 2018 16:09:15 +0100
Subject: [PATCH 530/830] fix reference to is_threepid_reserved

---
 synapse/api/auth.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 6a97c0611095..a36fb6b3bdf9 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -809,7 +809,7 @@ def check_auth_blocking(self, user_id=None, threepid=None):
             elif threepid:
                 # If the user does not exist yet, but is signing up with a
                 # reserved threepid then pass auth check
-                if is_threepid_reserved(threepid):
+                if self.store.is_threepid_reserved(threepid):
                     return
             # Else if there is no room in the MAU bucket, bail
             current_mau = yield self.store.get_monthly_active_count()

From 0b01281e77aee7e69925f36dbb6a798772a98a45 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Fri, 31 Aug 2018 17:11:11 +0100
Subject: [PATCH 531/830] move threepid checker to config, add missing yields

---
 synapse/api/auth.py                      | 13 +++++++++++--
 synapse/config/server.py                 | 17 +++++++++++++++++
 synapse/rest/client/v1_only/register.py  |  7 ++++---
 synapse/rest/client/v2_alpha/register.py |  5 +++--
 synapse/storage/monthly_active_users.py  | 14 --------------
 tests/utils.py                           |  6 ++++++
 6 files changed, 41 insertions(+), 21 deletions(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index a36fb6b3bdf9..a89687f42017 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -26,6 +26,7 @@
 from synapse import event_auth
 from synapse.api.constants import EventTypes, JoinRules, Membership
 from synapse.api.errors import AuthError, Codes, ResourceLimitError
+from synapse.config.server import is_threepid_reserved
 from synapse.types import UserID
 from synapse.util.caches import CACHE_SIZE_FACTOR, register_cache
 from synapse.util.caches.lrucache import LruCache
@@ -782,6 +783,11 @@ def check_auth_blocking(self, user_id=None, threepid=None):
         Args:
             user_id(str|None): If present, checks for presence against existing
             MAU cohort
+            threepid(dict|None): If present, checks for presence against configured
+            reserved threepid. Used in cases where the user is trying register
+            with a MAU blocked server, normally they would be rejected but their
+            threepid is on the reserved list. user_id and
+            threepid should never be set at the same time.
         """
 
         # Never fail an auth check for the server notices users
@@ -797,6 +803,10 @@ def check_auth_blocking(self, user_id=None, threepid=None):
                 limit_type=self.hs.config.hs_disabled_limit_type
             )
         if self.hs.config.limit_usage_by_mau is True:
+
+            if user_id and threepid:
+                logger.warn("Called with both user_id and threepid, this shoudn't happen")
+
             # If the user is already part of the MAU cohort or a trial user
             if user_id:
                 timestamp = yield self.store.user_last_seen_monthly_active(user_id)
@@ -809,14 +819,13 @@ def check_auth_blocking(self, user_id=None, threepid=None):
             elif threepid:
                 # If the user does not exist yet, but is signing up with a
                 # reserved threepid then pass auth check
-                if self.store.is_threepid_reserved(threepid):
+                if is_threepid_reserved(self.hs.config, threepid):
                     return
             # Else if there is no room in the MAU bucket, bail
             current_mau = yield self.store.get_monthly_active_count()
             if current_mau >= self.hs.config.max_mau_value:
                 raise ResourceLimitError(
                     403, "Monthly Active User Limit Exceeded",
-
                     admin_contact=self.hs.config.admin_contact,
                     errcode=Codes.RESOURCE_LIMIT_EXCEEDED,
                     limit_type="monthly_active_user"
diff --git a/synapse/config/server.py b/synapse/config/server.py
index 2faf4721897b..c1c7c0105e41 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -404,6 +404,23 @@ def add_arguments(self, parser):
                                   " service on the given port.")
 
 
+def is_threepid_reserved(config, threepid):
+    """Check the threepid against the reserved threepid config
+    Args:
+        config(ServerConfig) - to access server config attributes
+        threepid(dict) - The threepid to test for
+
+    Returns:
+        boolean Is the threepid undertest reserved_user
+    """
+
+    for tp in config.mau_limits_reserved_threepids:
+        if (threepid['medium'] == tp['medium']
+                and threepid['address'] == tp['address']):
+            return True
+    return False
+
+
 def read_gc_thresholds(thresholds):
     """Reads the three integer thresholds for garbage collection. Ensures that
     the thresholds are integers if thresholds are supplied.
diff --git a/synapse/rest/client/v1_only/register.py b/synapse/rest/client/v1_only/register.py
index 95873e03d541..dadb376b02ca 100644
--- a/synapse/rest/client/v1_only/register.py
+++ b/synapse/rest/client/v1_only/register.py
@@ -23,6 +23,7 @@
 import synapse.util.stringutils as stringutils
 from synapse.api.constants import LoginType
 from synapse.api.errors import Codes, SynapseError
+from synapse.config.server import is_threepid_reserved
 from synapse.http.servlet import assert_params_in_dict, parse_json_object_from_request
 from synapse.rest.client.v1.base import ClientV1RestServlet
 from synapse.types import create_requester
@@ -282,7 +283,7 @@ def _do_password(self, request, register_json, session):
             if "user" in register_json else None
         )
         threepid = None
-        if session[LoginType.EMAIL_IDENTITY]:
+        if session.get(LoginType.EMAIL_IDENTITY):
             threepid = session["threepidCreds"]
 
         handler = self.handlers.registration_handler
@@ -293,8 +294,8 @@ def _do_password(self, request, register_json, session):
         )
         # Necessary due to auth checks prior to the threepid being
         # written to the db
-        if self.store.is_threepid_reserved(threepid):
-            self.store.upsert_monthly_active_user(registered_user_id)
+        if is_threepid_reserved(self.hs.config, threepid):
+            yield self.store.upsert_monthly_active_user(user_id)
 
         if session[LoginType.EMAIL_IDENTITY]:
             logger.debug("Binding emails %s to %s" % (
diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index f22b7577ea06..2fb4d43ccb18 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -26,6 +26,7 @@
 import synapse.types
 from synapse.api.constants import LoginType
 from synapse.api.errors import Codes, SynapseError, UnrecognizedRequestError
+from synapse.config.server import is_threepid_reserved
 from synapse.http.servlet import (
     RestServlet,
     assert_params_in_dict,
@@ -408,8 +409,8 @@ def on_POST(self, request):
             )
             # Necessary due to auth checks prior to the threepid being
             # written to the db
-            if self.store.is_threepid_reserved(threepid):
-                self.store.upsert_monthly_active_user(registered_user_id)
+            if is_threepid_reserved(self.hs.config, threepid):
+                yield self.store.upsert_monthly_active_user(registered_user_id)
 
             # remember that we've now registered that user account, and with
             #  what user ID (since the user may not have specified)
diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py
index 173867c4b173..c7899d7fd269 100644
--- a/synapse/storage/monthly_active_users.py
+++ b/synapse/storage/monthly_active_users.py
@@ -219,17 +219,3 @@ def populate_monthly_active_users(self, user_id):
                     yield self.upsert_monthly_active_user(user_id)
             elif now - last_seen_timestamp > LAST_SEEN_GRANULARITY:
                 yield self.upsert_monthly_active_user(user_id)
-
-    def is_threepid_reserved(self, threepid):
-        """Check the threepid against the reserved threepid config
-        Args:
-            threepid(dict) - The threepid to test for
-        Returns:
-            boolean Is the threepid undertest reserved_user
-        """
-        for tp in self.hs.config.mau_limits_reserved_threepids:
-            if (threepid['medium'] == tp['medium']
-                    and threepid['address'] == tp['address']):
-                return True
-            else:
-                return False
diff --git a/tests/utils.py b/tests/utils.py
index 179b592501ce..63e30dc6c033 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -26,6 +26,7 @@
 
 from synapse.api.constants import EventTypes
 from synapse.api.errors import CodeMessageException, cs_error
+from synapse.config.server import ServerConfig
 from synapse.federation.transport import server
 from synapse.http.server import HttpServer
 from synapse.server import HomeServer
@@ -158,6 +159,11 @@ def setup_test_homeserver(
         # background, which upsets the test runner.
         config.update_user_directory = False
 
+        def is_threepid_reserved(threepid):
+            return ServerConfig.is_threepid_reserved(config, threepid)
+
+        config.is_threepid_reserved.side_effect = is_threepid_reserved
+
     config.use_frozen_dicts = True
     config.ldap_enabled = False
 

From 301cb60d0b2de55f7feac24a043b2624ad3c8733 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Fri, 31 Aug 2018 17:29:35 +0100
Subject: [PATCH 532/830] assert rather than warn

---
 synapse/api/auth.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index a89687f42017..34382e4e3c8d 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -783,6 +783,7 @@ def check_auth_blocking(self, user_id=None, threepid=None):
         Args:
             user_id(str|None): If present, checks for presence against existing
             MAU cohort
+
             threepid(dict|None): If present, checks for presence against configured
             reserved threepid. Used in cases where the user is trying register
             with a MAU blocked server, normally they would be rejected but their
@@ -803,9 +804,7 @@ def check_auth_blocking(self, user_id=None, threepid=None):
                 limit_type=self.hs.config.hs_disabled_limit_type
             )
         if self.hs.config.limit_usage_by_mau is True:
-
-            if user_id and threepid:
-                logger.warn("Called with both user_id and threepid, this shoudn't happen")
+            assert not (user_id and threepid)
 
             # If the user is already part of the MAU cohort or a trial user
             if user_id:

From a395f1ddb3133134525ac5ae94ff03d94c95dfe6 Mon Sep 17 00:00:00 2001
From: Colin W <cwmke@users.noreply.github.com>
Date: Sun, 2 Sep 2018 17:30:07 -0500
Subject: [PATCH 533/830] Update readme on develop branch

---
 README.rst | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/README.rst b/README.rst
index 4c5971d043c8..46498a7ee0ab 100644
--- a/README.rst
+++ b/README.rst
@@ -747,6 +747,18 @@ so an example nginx configuration might look like::
       }
   }
 
+and an example apache configuration may look like::
+
+    <VirtualHost *:443>
+        SSLEngine on
+        ServerName matrix.example.com;
+
+        <Location /_matrix>
+            ProxyPass http://127.0.0.1:8008/_matrix/ nocanon
+            ProxyPassReverse http://127.0.0.1:8008/_matrix/
+        </Location>
+    </VirtualHost>
+
 You will also want to set ``bind_addresses: ['127.0.0.1']`` and ``x_forwarded: true``
 for port 8008 in ``homeserver.yaml`` to ensure that client IP addresses are
 recorded correctly.

From 81942c109db7a5c816a65d5931d7be7a6560b6d7 Mon Sep 17 00:00:00 2001
From: Colin W <cwmke@users.noreply.github.com>
Date: Sun, 2 Sep 2018 23:28:03 -0500
Subject: [PATCH 534/830] Remove end '/'s

---
 README.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.rst b/README.rst
index 46498a7ee0ab..0fa39a416d97 100644
--- a/README.rst
+++ b/README.rst
@@ -754,8 +754,8 @@ and an example apache configuration may look like::
         ServerName matrix.example.com;
 
         <Location /_matrix>
-            ProxyPass http://127.0.0.1:8008/_matrix/ nocanon
-            ProxyPassReverse http://127.0.0.1:8008/_matrix/
+            ProxyPass http://127.0.0.1:8008/_matrix nocanon
+            ProxyPassReverse http://127.0.0.1:8008/_matrix
         </Location>
     </VirtualHost>
 

From 905f8de6737c18a629de054c4be01fa70990eaae Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Mon, 3 Sep 2018 22:17:06 +1000
Subject: [PATCH 535/830] Create 3378.misc

---
 changelog.d/3378.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3378.misc

diff --git a/changelog.d/3378.misc b/changelog.d/3378.misc
new file mode 100644
index 000000000000..8f88f88e69c6
--- /dev/null
+++ b/changelog.d/3378.misc
@@ -0,0 +1 @@
+Removed the link to the unmaintained matrix-synapse-auto-deploy project from the readme.

From 06ee2b7cc5e792acfade45217d6aaf1b56a201b8 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 3 Sep 2018 15:43:01 +0100
Subject: [PATCH 536/830] Newsfile

---
 changelog.d/3737.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3737.misc

diff --git a/changelog.d/3737.misc b/changelog.d/3737.misc
new file mode 100644
index 000000000000..0361da4ebc3b
--- /dev/null
+++ b/changelog.d/3737.misc
@@ -0,0 +1 @@
+Remove redundant state resolution function

From 77055dba92af80cdfdffc30fd3084ecd24902c2e Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Tue, 4 Sep 2018 02:21:48 +1000
Subject: [PATCH 537/830] Fix tests on postgresql (#3740)

---
 .travis.yml                                   |   4 -
 changelog.d/3740.misc                         |   1 +
 tests/handlers/test_device.py                 | 143 +++++++++---------
 tests/replication/slave/storage/_base.py      |  95 ++++++------
 .../slave/storage/test_account_data.py        |  20 +--
 .../replication/slave/storage/test_events.py  | 110 +++++++-------
 .../slave/storage/test_receipts.py            |  15 +-
 tests/server.py                               |   1 +
 tests/storage/test_appservice.py              | 131 ++++++++--------
 tests/storage/test_directory.py               |   3 +-
 tests/storage/test_event_federation.py        |   6 +-
 tests/storage/test_monthly_active_users.py    | 138 +++++++++--------
 tests/storage/test_presence.py                |   7 +-
 tests/storage/test_profile.py                 |   2 +-
 tests/storage/test_user_directory.py          |   2 +-
 tests/test_visibility.py                      |   2 +-
 tests/unittest.py                             |   7 +-
 tests/utils.py                                |  10 +-
 18 files changed, 356 insertions(+), 341 deletions(-)
 create mode 100644 changelog.d/3740.misc

diff --git a/.travis.yml b/.travis.yml
index 318701c9f89f..11c76db2e5a4 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -35,10 +35,6 @@ matrix:
   - python: 3.6
     env: TOX_ENV=check-newsfragment
 
-  allow_failures:
-  - python: 2.7
-    env: TOX_ENV=py27-postgres TRIAL_FLAGS="-j 4"
-
 install:
   - pip install tox
 
diff --git a/changelog.d/3740.misc b/changelog.d/3740.misc
new file mode 100644
index 000000000000..4dcb7fb5de6e
--- /dev/null
+++ b/changelog.d/3740.misc
@@ -0,0 +1 @@
+The test suite now passes on PostgreSQL.
diff --git a/tests/handlers/test_device.py b/tests/handlers/test_device.py
index 56e7acd37cfb..a3aa0a1cf240 100644
--- a/tests/handlers/test_device.py
+++ b/tests/handlers/test_device.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,79 +14,79 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from twisted.internet import defer
-
 import synapse.api.errors
 import synapse.handlers.device
 import synapse.storage
 
-from tests import unittest, utils
+from tests import unittest
 
 user1 = "@boris:aaa"
 user2 = "@theresa:bbb"
 
 
-class DeviceTestCase(unittest.TestCase):
-    def __init__(self, *args, **kwargs):
-        super(DeviceTestCase, self).__init__(*args, **kwargs)
-        self.store = None  # type: synapse.storage.DataStore
-        self.handler = None  # type: synapse.handlers.device.DeviceHandler
-        self.clock = None  # type: utils.MockClock
-
-    @defer.inlineCallbacks
-    def setUp(self):
-        hs = yield utils.setup_test_homeserver(self.addCleanup)
+class DeviceTestCase(unittest.HomeserverTestCase):
+    def make_homeserver(self, reactor, clock):
+        hs = self.setup_test_homeserver("server", http_client=None)
         self.handler = hs.get_device_handler()
         self.store = hs.get_datastore()
-        self.clock = hs.get_clock()
+        return hs
+
+    def prepare(self, reactor, clock, hs):
+        # These tests assume that it starts 1000 seconds in.
+        self.reactor.advance(1000)
 
-    @defer.inlineCallbacks
     def test_device_is_created_if_doesnt_exist(self):
-        res = yield self.handler.check_device_registered(
-            user_id="@boris:foo",
-            device_id="fco",
-            initial_device_display_name="display name",
+        res = self.get_success(
+            self.handler.check_device_registered(
+                user_id="@boris:foo",
+                device_id="fco",
+                initial_device_display_name="display name",
+            )
         )
         self.assertEqual(res, "fco")
 
-        dev = yield self.handler.store.get_device("@boris:foo", "fco")
+        dev = self.get_success(self.handler.store.get_device("@boris:foo", "fco"))
         self.assertEqual(dev["display_name"], "display name")
 
-    @defer.inlineCallbacks
     def test_device_is_preserved_if_exists(self):
-        res1 = yield self.handler.check_device_registered(
-            user_id="@boris:foo",
-            device_id="fco",
-            initial_device_display_name="display name",
+        res1 = self.get_success(
+            self.handler.check_device_registered(
+                user_id="@boris:foo",
+                device_id="fco",
+                initial_device_display_name="display name",
+            )
         )
         self.assertEqual(res1, "fco")
 
-        res2 = yield self.handler.check_device_registered(
-            user_id="@boris:foo",
-            device_id="fco",
-            initial_device_display_name="new display name",
+        res2 = self.get_success(
+            self.handler.check_device_registered(
+                user_id="@boris:foo",
+                device_id="fco",
+                initial_device_display_name="new display name",
+            )
         )
         self.assertEqual(res2, "fco")
 
-        dev = yield self.handler.store.get_device("@boris:foo", "fco")
+        dev = self.get_success(self.handler.store.get_device("@boris:foo", "fco"))
         self.assertEqual(dev["display_name"], "display name")
 
-    @defer.inlineCallbacks
     def test_device_id_is_made_up_if_unspecified(self):
-        device_id = yield self.handler.check_device_registered(
-            user_id="@theresa:foo",
-            device_id=None,
-            initial_device_display_name="display",
+        device_id = self.get_success(
+            self.handler.check_device_registered(
+                user_id="@theresa:foo",
+                device_id=None,
+                initial_device_display_name="display",
+            )
         )
 
-        dev = yield self.handler.store.get_device("@theresa:foo", device_id)
+        dev = self.get_success(self.handler.store.get_device("@theresa:foo", device_id))
         self.assertEqual(dev["display_name"], "display")
 
-    @defer.inlineCallbacks
     def test_get_devices_by_user(self):
-        yield self._record_users()
+        self._record_users()
+
+        res = self.get_success(self.handler.get_devices_by_user(user1))
 
-        res = yield self.handler.get_devices_by_user(user1)
         self.assertEqual(3, len(res))
         device_map = {d["device_id"]: d for d in res}
         self.assertDictContainsSubset(
@@ -119,11 +120,10 @@ def test_get_devices_by_user(self):
             device_map["abc"],
         )
 
-    @defer.inlineCallbacks
     def test_get_device(self):
-        yield self._record_users()
+        self._record_users()
 
-        res = yield self.handler.get_device(user1, "abc")
+        res = self.get_success(self.handler.get_device(user1, "abc"))
         self.assertDictContainsSubset(
             {
                 "user_id": user1,
@@ -135,59 +135,66 @@ def test_get_device(self):
             res,
         )
 
-    @defer.inlineCallbacks
     def test_delete_device(self):
-        yield self._record_users()
+        self._record_users()
 
         # delete the device
-        yield self.handler.delete_device(user1, "abc")
+        self.get_success(self.handler.delete_device(user1, "abc"))
 
         # check the device was deleted
-        with self.assertRaises(synapse.api.errors.NotFoundError):
-            yield self.handler.get_device(user1, "abc")
+        res = self.handler.get_device(user1, "abc")
+        self.pump()
+        self.assertIsInstance(
+            self.failureResultOf(res).value, synapse.api.errors.NotFoundError
+        )
 
         # we'd like to check the access token was invalidated, but that's a
         # bit of a PITA.
 
-    @defer.inlineCallbacks
     def test_update_device(self):
-        yield self._record_users()
+        self._record_users()
 
         update = {"display_name": "new display"}
-        yield self.handler.update_device(user1, "abc", update)
+        self.get_success(self.handler.update_device(user1, "abc", update))
 
-        res = yield self.handler.get_device(user1, "abc")
+        res = self.get_success(self.handler.get_device(user1, "abc"))
         self.assertEqual(res["display_name"], "new display")
 
-    @defer.inlineCallbacks
     def test_update_unknown_device(self):
         update = {"display_name": "new_display"}
-        with self.assertRaises(synapse.api.errors.NotFoundError):
-            yield self.handler.update_device("user_id", "unknown_device_id", update)
+        res = self.handler.update_device("user_id", "unknown_device_id", update)
+        self.pump()
+        self.assertIsInstance(
+            self.failureResultOf(res).value, synapse.api.errors.NotFoundError
+        )
 
-    @defer.inlineCallbacks
     def _record_users(self):
         # check this works for both devices which have a recorded client_ip,
         # and those which don't.
-        yield self._record_user(user1, "xyz", "display 0")
-        yield self._record_user(user1, "fco", "display 1", "token1", "ip1")
-        yield self._record_user(user1, "abc", "display 2", "token2", "ip2")
-        yield self._record_user(user1, "abc", "display 2", "token3", "ip3")
+        self._record_user(user1, "xyz", "display 0")
+        self._record_user(user1, "fco", "display 1", "token1", "ip1")
+        self._record_user(user1, "abc", "display 2", "token2", "ip2")
+        self._record_user(user1, "abc", "display 2", "token3", "ip3")
+
+        self._record_user(user2, "def", "dispkay", "token4", "ip4")
 
-        yield self._record_user(user2, "def", "dispkay", "token4", "ip4")
+        self.reactor.advance(10000)
 
-    @defer.inlineCallbacks
     def _record_user(
         self, user_id, device_id, display_name, access_token=None, ip=None
     ):
-        device_id = yield self.handler.check_device_registered(
-            user_id=user_id,
-            device_id=device_id,
-            initial_device_display_name=display_name,
+        device_id = self.get_success(
+            self.handler.check_device_registered(
+                user_id=user_id,
+                device_id=device_id,
+                initial_device_display_name=display_name,
+            )
         )
 
         if ip is not None:
-            yield self.store.insert_client_ip(
-                user_id, access_token, ip, "user_agent", device_id
+            self.get_success(
+                self.store.insert_client_ip(
+                    user_id, access_token, ip, "user_agent", device_id
+                )
             )
-            self.clock.advance_time(1000)
+            self.reactor.advance(1000)
diff --git a/tests/replication/slave/storage/_base.py b/tests/replication/slave/storage/_base.py
index 65df116efc75..089cecfbee16 100644
--- a/tests/replication/slave/storage/_base.py
+++ b/tests/replication/slave/storage/_base.py
@@ -1,4 +1,5 @@
 # Copyright 2016 OpenMarket Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,89 +12,91 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import tempfile
 
 from mock import Mock, NonCallableMock
 
-from twisted.internet import defer, reactor
-from twisted.internet.defer import Deferred
+import attr
 
 from synapse.replication.tcp.client import (
     ReplicationClientFactory,
     ReplicationClientHandler,
 )
 from synapse.replication.tcp.resource import ReplicationStreamProtocolFactory
-from synapse.util.logcontext import PreserveLoggingContext, make_deferred_yieldable
 
 from tests import unittest
-from tests.utils import setup_test_homeserver
 
 
-class TestReplicationClientHandler(ReplicationClientHandler):
-    """Overrides on_rdata so that we can wait for it to happen"""
+class BaseSlavedStoreTestCase(unittest.HomeserverTestCase):
+    def make_homeserver(self, reactor, clock):
 
-    def __init__(self, store):
-        super(TestReplicationClientHandler, self).__init__(store)
-        self._rdata_awaiters = []
-
-    def await_replication(self):
-        d = Deferred()
-        self._rdata_awaiters.append(d)
-        return make_deferred_yieldable(d)
-
-    def on_rdata(self, stream_name, token, rows):
-        awaiters = self._rdata_awaiters
-        self._rdata_awaiters = []
-        super(TestReplicationClientHandler, self).on_rdata(stream_name, token, rows)
-        with PreserveLoggingContext():
-            for a in awaiters:
-                a.callback(None)
-
-
-class BaseSlavedStoreTestCase(unittest.TestCase):
-    @defer.inlineCallbacks
-    def setUp(self):
-        self.hs = yield setup_test_homeserver(
-            self.addCleanup,
+        hs = self.setup_test_homeserver(
             "blue",
-            http_client=None,
             federation_client=Mock(),
             ratelimiter=NonCallableMock(spec_set=["send_message"]),
         )
-        self.hs.get_ratelimiter().send_message.return_value = (True, 0)
+
+        hs.get_ratelimiter().send_message.return_value = (True, 0)
+
+        return hs
+
+    def prepare(self, reactor, clock, hs):
 
         self.master_store = self.hs.get_datastore()
         self.slaved_store = self.STORE_TYPE(self.hs.get_db_conn(), self.hs)
         self.event_id = 0
 
         server_factory = ReplicationStreamProtocolFactory(self.hs)
-        # XXX: mktemp is unsafe and should never be used. but we're just a test.
-        path = tempfile.mktemp(prefix="base_slaved_store_test_case_socket")
-        listener = reactor.listenUNIX(path, server_factory)
-        self.addCleanup(listener.stopListening)
         self.streamer = server_factory.streamer
 
-        self.replication_handler = TestReplicationClientHandler(self.slaved_store)
+        self.replication_handler = ReplicationClientHandler(self.slaved_store)
         client_factory = ReplicationClientFactory(
             self.hs, "client_name", self.replication_handler
         )
-        client_connector = reactor.connectUNIX(path, client_factory)
-        self.addCleanup(client_factory.stopTrying)
-        self.addCleanup(client_connector.disconnect)
+
+        server = server_factory.buildProtocol(None)
+        client = client_factory.buildProtocol(None)
+
+        @attr.s
+        class FakeTransport(object):
+
+            other = attr.ib()
+            disconnecting = False
+            buffer = attr.ib(default=b'')
+
+            def registerProducer(self, producer, streaming):
+
+                self.producer = producer
+
+                def _produce():
+                    self.producer.resumeProducing()
+                    reactor.callLater(0.1, _produce)
+
+                reactor.callLater(0.0, _produce)
+
+            def write(self, byt):
+                self.buffer = self.buffer + byt
+
+                if getattr(self.other, "transport") is not None:
+                    self.other.dataReceived(self.buffer)
+                    self.buffer = b""
+
+            def writeSequence(self, seq):
+                for x in seq:
+                    self.write(x)
+
+        client.makeConnection(FakeTransport(server))
+        server.makeConnection(FakeTransport(client))
 
     def replicate(self):
         """Tell the master side of replication that something has happened, and then
         wait for the replication to occur.
         """
-        # xxx: should we be more specific in what we wait for?
-        d = self.replication_handler.await_replication()
         self.streamer.on_notifier_poke()
-        return d
+        self.pump(0.1)
 
-    @defer.inlineCallbacks
     def check(self, method, args, expected_result=None):
-        master_result = yield getattr(self.master_store, method)(*args)
-        slaved_result = yield getattr(self.slaved_store, method)(*args)
+        master_result = self.get_success(getattr(self.master_store, method)(*args))
+        slaved_result = self.get_success(getattr(self.slaved_store, method)(*args))
         if expected_result is not None:
             self.assertEqual(master_result, expected_result)
             self.assertEqual(slaved_result, expected_result)
diff --git a/tests/replication/slave/storage/test_account_data.py b/tests/replication/slave/storage/test_account_data.py
index 87cc2b2fba5f..43e3248703c9 100644
--- a/tests/replication/slave/storage/test_account_data.py
+++ b/tests/replication/slave/storage/test_account_data.py
@@ -12,9 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
-from twisted.internet import defer
-
 from synapse.replication.slave.storage.account_data import SlavedAccountDataStore
 
 from ._base import BaseSlavedStoreTestCase
@@ -27,16 +24,19 @@ class SlavedAccountDataStoreTestCase(BaseSlavedStoreTestCase):
 
     STORE_TYPE = SlavedAccountDataStore
 
-    @defer.inlineCallbacks
     def test_user_account_data(self):
-        yield self.master_store.add_account_data_for_user(USER_ID, TYPE, {"a": 1})
-        yield self.replicate()
-        yield self.check(
+        self.get_success(
+            self.master_store.add_account_data_for_user(USER_ID, TYPE, {"a": 1})
+        )
+        self.replicate()
+        self.check(
             "get_global_account_data_by_type_for_user", [TYPE, USER_ID], {"a": 1}
         )
 
-        yield self.master_store.add_account_data_for_user(USER_ID, TYPE, {"a": 2})
-        yield self.replicate()
-        yield self.check(
+        self.get_success(
+            self.master_store.add_account_data_for_user(USER_ID, TYPE, {"a": 2})
+        )
+        self.replicate()
+        self.check(
             "get_global_account_data_by_type_for_user", [TYPE, USER_ID], {"a": 2}
         )
diff --git a/tests/replication/slave/storage/test_events.py b/tests/replication/slave/storage/test_events.py
index 2ba80ccdcfa9..db44d33c68cb 100644
--- a/tests/replication/slave/storage/test_events.py
+++ b/tests/replication/slave/storage/test_events.py
@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from twisted.internet import defer
-
 from synapse.events import FrozenEvent, _EventInternalMetadata
 from synapse.events.snapshot import EventContext
 from synapse.replication.slave.storage.events import SlavedEventStore
@@ -55,70 +53,66 @@ def setUp(self):
     def tearDown(self):
         [unpatch() for unpatch in self.unpatches]
 
-    @defer.inlineCallbacks
     def test_get_latest_event_ids_in_room(self):
-        create = yield self.persist(type="m.room.create", key="", creator=USER_ID)
-        yield self.replicate()
-        yield self.check("get_latest_event_ids_in_room", (ROOM_ID,), [create.event_id])
+        create = self.persist(type="m.room.create", key="", creator=USER_ID)
+        self.replicate()
+        self.check("get_latest_event_ids_in_room", (ROOM_ID,), [create.event_id])
 
-        join = yield self.persist(
+        join = self.persist(
             type="m.room.member",
             key=USER_ID,
             membership="join",
             prev_events=[(create.event_id, {})],
         )
-        yield self.replicate()
-        yield self.check("get_latest_event_ids_in_room", (ROOM_ID,), [join.event_id])
+        self.replicate()
+        self.check("get_latest_event_ids_in_room", (ROOM_ID,), [join.event_id])
 
-    @defer.inlineCallbacks
     def test_redactions(self):
-        yield self.persist(type="m.room.create", key="", creator=USER_ID)
-        yield self.persist(type="m.room.member", key=USER_ID, membership="join")
+        self.persist(type="m.room.create", key="", creator=USER_ID)
+        self.persist(type="m.room.member", key=USER_ID, membership="join")
 
-        msg = yield self.persist(type="m.room.message", msgtype="m.text", body="Hello")
-        yield self.replicate()
-        yield self.check("get_event", [msg.event_id], msg)
+        msg = self.persist(type="m.room.message", msgtype="m.text", body="Hello")
+        self.replicate()
+        self.check("get_event", [msg.event_id], msg)
 
-        redaction = yield self.persist(type="m.room.redaction", redacts=msg.event_id)
-        yield self.replicate()
+        redaction = self.persist(type="m.room.redaction", redacts=msg.event_id)
+        self.replicate()
 
         msg_dict = msg.get_dict()
         msg_dict["content"] = {}
         msg_dict["unsigned"]["redacted_by"] = redaction.event_id
         msg_dict["unsigned"]["redacted_because"] = redaction
         redacted = FrozenEvent(msg_dict, msg.internal_metadata.get_dict())
-        yield self.check("get_event", [msg.event_id], redacted)
+        self.check("get_event", [msg.event_id], redacted)
 
-    @defer.inlineCallbacks
     def test_backfilled_redactions(self):
-        yield self.persist(type="m.room.create", key="", creator=USER_ID)
-        yield self.persist(type="m.room.member", key=USER_ID, membership="join")
+        self.persist(type="m.room.create", key="", creator=USER_ID)
+        self.persist(type="m.room.member", key=USER_ID, membership="join")
 
-        msg = yield self.persist(type="m.room.message", msgtype="m.text", body="Hello")
-        yield self.replicate()
-        yield self.check("get_event", [msg.event_id], msg)
+        msg = self.persist(type="m.room.message", msgtype="m.text", body="Hello")
+        self.replicate()
+        self.check("get_event", [msg.event_id], msg)
 
-        redaction = yield self.persist(
+        redaction = self.persist(
             type="m.room.redaction", redacts=msg.event_id, backfill=True
         )
-        yield self.replicate()
+        self.replicate()
 
         msg_dict = msg.get_dict()
         msg_dict["content"] = {}
         msg_dict["unsigned"]["redacted_by"] = redaction.event_id
         msg_dict["unsigned"]["redacted_because"] = redaction
         redacted = FrozenEvent(msg_dict, msg.internal_metadata.get_dict())
-        yield self.check("get_event", [msg.event_id], redacted)
+        self.check("get_event", [msg.event_id], redacted)
 
-    @defer.inlineCallbacks
     def test_invites(self):
-        yield self.persist(type="m.room.create", key="", creator=USER_ID)
-        yield self.check("get_invited_rooms_for_user", [USER_ID_2], [])
-        event = yield self.persist(
-            type="m.room.member", key=USER_ID_2, membership="invite"
-        )
-        yield self.replicate()
-        yield self.check(
+        self.persist(type="m.room.create", key="", creator=USER_ID)
+        self.check("get_invited_rooms_for_user", [USER_ID_2], [])
+        event = self.persist(type="m.room.member", key=USER_ID_2, membership="invite")
+
+        self.replicate()
+
+        self.check(
             "get_invited_rooms_for_user",
             [USER_ID_2],
             [
@@ -132,37 +126,34 @@ def test_invites(self):
             ],
         )
 
-    @defer.inlineCallbacks
     def test_push_actions_for_user(self):
-        yield self.persist(type="m.room.create", key="", creator=USER_ID)
-        yield self.persist(type="m.room.join", key=USER_ID, membership="join")
-        yield self.persist(
+        self.persist(type="m.room.create", key="", creator=USER_ID)
+        self.persist(type="m.room.join", key=USER_ID, membership="join")
+        self.persist(
             type="m.room.join", sender=USER_ID, key=USER_ID_2, membership="join"
         )
-        event1 = yield self.persist(
-            type="m.room.message", msgtype="m.text", body="hello"
-        )
-        yield self.replicate()
-        yield self.check(
+        event1 = self.persist(type="m.room.message", msgtype="m.text", body="hello")
+        self.replicate()
+        self.check(
             "get_unread_event_push_actions_by_room_for_user",
             [ROOM_ID, USER_ID_2, event1.event_id],
             {"highlight_count": 0, "notify_count": 0},
         )
 
-        yield self.persist(
+        self.persist(
             type="m.room.message",
             msgtype="m.text",
             body="world",
             push_actions=[(USER_ID_2, ["notify"])],
         )
-        yield self.replicate()
-        yield self.check(
+        self.replicate()
+        self.check(
             "get_unread_event_push_actions_by_room_for_user",
             [ROOM_ID, USER_ID_2, event1.event_id],
             {"highlight_count": 0, "notify_count": 1},
         )
 
-        yield self.persist(
+        self.persist(
             type="m.room.message",
             msgtype="m.text",
             body="world",
@@ -170,8 +161,8 @@ def test_push_actions_for_user(self):
                 (USER_ID_2, ["notify", {"set_tweak": "highlight", "value": True}])
             ],
         )
-        yield self.replicate()
-        yield self.check(
+        self.replicate()
+        self.check(
             "get_unread_event_push_actions_by_room_for_user",
             [ROOM_ID, USER_ID_2, event1.event_id],
             {"highlight_count": 1, "notify_count": 2},
@@ -179,7 +170,6 @@ def test_push_actions_for_user(self):
 
     event_id = 0
 
-    @defer.inlineCallbacks
     def persist(
         self,
         sender=USER_ID,
@@ -206,8 +196,8 @@ def persist(
             depth = self.event_id
 
         if not prev_events:
-            latest_event_ids = yield self.master_store.get_latest_event_ids_in_room(
-                room_id
+            latest_event_ids = self.get_success(
+                self.master_store.get_latest_event_ids_in_room(room_id)
             )
             prev_events = [(ev_id, {}) for ev_id in latest_event_ids]
 
@@ -240,19 +230,23 @@ def persist(
             )
         else:
             state_handler = self.hs.get_state_handler()
-            context = yield state_handler.compute_event_context(event)
+            context = self.get_success(state_handler.compute_event_context(event))
 
-        yield self.master_store.add_push_actions_to_staging(
+        self.master_store.add_push_actions_to_staging(
             event.event_id, {user_id: actions for user_id, actions in push_actions}
         )
 
         ordering = None
         if backfill:
-            yield self.master_store.persist_events([(event, context)], backfilled=True)
+            self.get_success(
+                self.master_store.persist_events([(event, context)], backfilled=True)
+            )
         else:
-            ordering, _ = yield self.master_store.persist_event(event, context)
+            ordering, _ = self.get_success(
+                self.master_store.persist_event(event, context)
+            )
 
         if ordering:
             event.internal_metadata.stream_ordering = ordering
 
-        defer.returnValue(event)
+        return event
diff --git a/tests/replication/slave/storage/test_receipts.py b/tests/replication/slave/storage/test_receipts.py
index ae1adeded103..f47d94f690f9 100644
--- a/tests/replication/slave/storage/test_receipts.py
+++ b/tests/replication/slave/storage/test_receipts.py
@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from twisted.internet import defer
-
 from synapse.replication.slave.storage.receipts import SlavedReceiptsStore
 
 from ._base import BaseSlavedStoreTestCase
@@ -27,13 +25,10 @@ class SlavedReceiptTestCase(BaseSlavedStoreTestCase):
 
     STORE_TYPE = SlavedReceiptsStore
 
-    @defer.inlineCallbacks
     def test_receipt(self):
-        yield self.check("get_receipts_for_user", [USER_ID, "m.read"], {})
-        yield self.master_store.insert_receipt(
-            ROOM_ID, "m.read", USER_ID, [EVENT_ID], {}
-        )
-        yield self.replicate()
-        yield self.check(
-            "get_receipts_for_user", [USER_ID, "m.read"], {ROOM_ID: EVENT_ID}
+        self.check("get_receipts_for_user", [USER_ID, "m.read"], {})
+        self.get_success(
+            self.master_store.insert_receipt(ROOM_ID, "m.read", USER_ID, [EVENT_ID], {})
         )
+        self.replicate()
+        self.check("get_receipts_for_user", [USER_ID, "m.read"], {ROOM_ID: EVENT_ID})
diff --git a/tests/server.py b/tests/server.py
index 7dbdb7f8eaea..615bba1b5985 100644
--- a/tests/server.py
+++ b/tests/server.py
@@ -232,6 +232,7 @@ def _(res):
 
     clock.threadpool = ThreadPool()
     pool.threadpool = ThreadPool()
+    pool.running = True
     return d
 
 
diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py
index c893990454bb..3f0083831b51 100644
--- a/tests/storage/test_appservice.py
+++ b/tests/storage/test_appservice.py
@@ -37,18 +37,14 @@ class ApplicationServiceStoreTestCase(unittest.TestCase):
     @defer.inlineCallbacks
     def setUp(self):
         self.as_yaml_files = []
-        config = Mock(
-            app_service_config_files=self.as_yaml_files,
-            event_cache_size=1,
-            password_providers=[],
-        )
         hs = yield setup_test_homeserver(
-            self.addCleanup,
-            config=config,
-            federation_sender=Mock(),
-            federation_client=Mock(),
+            self.addCleanup, federation_sender=Mock(), federation_client=Mock()
         )
 
+        hs.config.app_service_config_files = self.as_yaml_files
+        hs.config.event_cache_size = 1
+        hs.config.password_providers = []
+
         self.as_token = "token1"
         self.as_url = "some_url"
         self.as_id = "as1"
@@ -58,7 +54,7 @@ def setUp(self):
         self._add_appservice("token2", "as2", "some_url", "some_hs_token", "bob")
         self._add_appservice("token3", "as3", "some_url", "some_hs_token", "bob")
         # must be done after inserts
-        self.store = ApplicationServiceStore(None, hs)
+        self.store = ApplicationServiceStore(hs.get_db_conn(), hs)
 
     def tearDown(self):
         # TODO: suboptimal that we need to create files for tests!
@@ -105,18 +101,16 @@ class ApplicationServiceTransactionStoreTestCase(unittest.TestCase):
     def setUp(self):
         self.as_yaml_files = []
 
-        config = Mock(
-            app_service_config_files=self.as_yaml_files,
-            event_cache_size=1,
-            password_providers=[],
-        )
         hs = yield setup_test_homeserver(
-            self.addCleanup,
-            config=config,
-            federation_sender=Mock(),
-            federation_client=Mock(),
+            self.addCleanup, federation_sender=Mock(), federation_client=Mock()
         )
+
+        hs.config.app_service_config_files = self.as_yaml_files
+        hs.config.event_cache_size = 1
+        hs.config.password_providers = []
+
         self.db_pool = hs.get_db_pool()
+        self.engine = hs.database_engine
 
         self.as_list = [
             {"token": "token1", "url": "https://matrix-as.org", "id": "id_1"},
@@ -129,7 +123,7 @@ def setUp(self):
 
         self.as_yaml_files = []
 
-        self.store = TestTransactionStore(None, hs)
+        self.store = TestTransactionStore(hs.get_db_conn(), hs)
 
     def _add_service(self, url, as_token, id):
         as_yaml = dict(
@@ -146,29 +140,35 @@ def _add_service(self, url, as_token, id):
             self.as_yaml_files.append(as_token)
 
     def _set_state(self, id, state, txn=None):
-        return self.db_pool.runQuery(
-            "INSERT INTO application_services_state(as_id, state, last_txn) "
-            "VALUES(?,?,?)",
+        return self.db_pool.runOperation(
+            self.engine.convert_param_style(
+                "INSERT INTO application_services_state(as_id, state, last_txn) "
+                "VALUES(?,?,?)"
+            ),
             (id, state, txn),
         )
 
     def _insert_txn(self, as_id, txn_id, events):
-        return self.db_pool.runQuery(
-            "INSERT INTO application_services_txns(as_id, txn_id, event_ids) "
-            "VALUES(?,?,?)",
+        return self.db_pool.runOperation(
+            self.engine.convert_param_style(
+                "INSERT INTO application_services_txns(as_id, txn_id, event_ids) "
+                "VALUES(?,?,?)"
+            ),
             (as_id, txn_id, json.dumps([e.event_id for e in events])),
         )
 
     def _set_last_txn(self, as_id, txn_id):
-        return self.db_pool.runQuery(
-            "INSERT INTO application_services_state(as_id, last_txn, state) "
-            "VALUES(?,?,?)",
+        return self.db_pool.runOperation(
+            self.engine.convert_param_style(
+                "INSERT INTO application_services_state(as_id, last_txn, state) "
+                "VALUES(?,?,?)"
+            ),
             (as_id, txn_id, ApplicationServiceState.UP),
         )
 
     @defer.inlineCallbacks
     def test_get_appservice_state_none(self):
-        service = Mock(id=999)
+        service = Mock(id="999")
         state = yield self.store.get_appservice_state(service)
         self.assertEquals(None, state)
 
@@ -200,7 +200,9 @@ def test_set_appservices_state_down(self):
         service = Mock(id=self.as_list[1]["id"])
         yield self.store.set_appservice_state(service, ApplicationServiceState.DOWN)
         rows = yield self.db_pool.runQuery(
-            "SELECT as_id FROM application_services_state WHERE state=?",
+            self.engine.convert_param_style(
+                "SELECT as_id FROM application_services_state WHERE state=?"
+            ),
             (ApplicationServiceState.DOWN,),
         )
         self.assertEquals(service.id, rows[0][0])
@@ -212,7 +214,9 @@ def test_set_appservices_state_multiple_up(self):
         yield self.store.set_appservice_state(service, ApplicationServiceState.DOWN)
         yield self.store.set_appservice_state(service, ApplicationServiceState.UP)
         rows = yield self.db_pool.runQuery(
-            "SELECT as_id FROM application_services_state WHERE state=?",
+            self.engine.convert_param_style(
+                "SELECT as_id FROM application_services_state WHERE state=?"
+            ),
             (ApplicationServiceState.UP,),
         )
         self.assertEquals(service.id, rows[0][0])
@@ -279,14 +283,19 @@ def test_complete_appservice_txn_first_txn(self):
         yield self.store.complete_appservice_txn(txn_id=txn_id, service=service)
 
         res = yield self.db_pool.runQuery(
-            "SELECT last_txn FROM application_services_state WHERE as_id=?",
+            self.engine.convert_param_style(
+                "SELECT last_txn FROM application_services_state WHERE as_id=?"
+            ),
             (service.id,),
         )
         self.assertEquals(1, len(res))
         self.assertEquals(txn_id, res[0][0])
 
         res = yield self.db_pool.runQuery(
-            "SELECT * FROM application_services_txns WHERE txn_id=?", (txn_id,)
+            self.engine.convert_param_style(
+                "SELECT * FROM application_services_txns WHERE txn_id=?"
+            ),
+            (txn_id,),
         )
         self.assertEquals(0, len(res))
 
@@ -300,7 +309,9 @@ def test_complete_appservice_txn_existing_in_state_table(self):
         yield self.store.complete_appservice_txn(txn_id=txn_id, service=service)
 
         res = yield self.db_pool.runQuery(
-            "SELECT last_txn, state FROM application_services_state WHERE " "as_id=?",
+            self.engine.convert_param_style(
+                "SELECT last_txn, state FROM application_services_state WHERE as_id=?"
+            ),
             (service.id,),
         )
         self.assertEquals(1, len(res))
@@ -308,7 +319,10 @@ def test_complete_appservice_txn_existing_in_state_table(self):
         self.assertEquals(ApplicationServiceState.UP, res[0][1])
 
         res = yield self.db_pool.runQuery(
-            "SELECT * FROM application_services_txns WHERE txn_id=?", (txn_id,)
+            self.engine.convert_param_style(
+                "SELECT * FROM application_services_txns WHERE txn_id=?"
+            ),
+            (txn_id,),
         )
         self.assertEquals(0, len(res))
 
@@ -394,37 +408,31 @@ def test_unique_works(self):
         f1 = self._write_config(suffix="1")
         f2 = self._write_config(suffix="2")
 
-        config = Mock(
-            app_service_config_files=[f1, f2], event_cache_size=1, password_providers=[]
-        )
         hs = yield setup_test_homeserver(
-            self.addCleanup,
-            config=config,
-            datastore=Mock(),
-            federation_sender=Mock(),
-            federation_client=Mock(),
+            self.addCleanup, federation_sender=Mock(), federation_client=Mock()
         )
 
-        ApplicationServiceStore(None, hs)
+        hs.config.app_service_config_files = [f1, f2]
+        hs.config.event_cache_size = 1
+        hs.config.password_providers = []
+
+        ApplicationServiceStore(hs.get_db_conn(), hs)
 
     @defer.inlineCallbacks
     def test_duplicate_ids(self):
         f1 = self._write_config(id="id", suffix="1")
         f2 = self._write_config(id="id", suffix="2")
 
-        config = Mock(
-            app_service_config_files=[f1, f2], event_cache_size=1, password_providers=[]
-        )
         hs = yield setup_test_homeserver(
-            self.addCleanup,
-            config=config,
-            datastore=Mock(),
-            federation_sender=Mock(),
-            federation_client=Mock(),
+            self.addCleanup, federation_sender=Mock(), federation_client=Mock()
         )
 
+        hs.config.app_service_config_files = [f1, f2]
+        hs.config.event_cache_size = 1
+        hs.config.password_providers = []
+
         with self.assertRaises(ConfigError) as cm:
-            ApplicationServiceStore(None, hs)
+            ApplicationServiceStore(hs.get_db_conn(), hs)
 
         e = cm.exception
         self.assertIn(f1, str(e))
@@ -436,19 +444,16 @@ def test_duplicate_as_tokens(self):
         f1 = self._write_config(as_token="as_token", suffix="1")
         f2 = self._write_config(as_token="as_token", suffix="2")
 
-        config = Mock(
-            app_service_config_files=[f1, f2], event_cache_size=1, password_providers=[]
-        )
         hs = yield setup_test_homeserver(
-            self.addCleanup,
-            config=config,
-            datastore=Mock(),
-            federation_sender=Mock(),
-            federation_client=Mock(),
+            self.addCleanup, federation_sender=Mock(), federation_client=Mock()
         )
 
+        hs.config.app_service_config_files = [f1, f2]
+        hs.config.event_cache_size = 1
+        hs.config.password_providers = []
+
         with self.assertRaises(ConfigError) as cm:
-            ApplicationServiceStore(None, hs)
+            ApplicationServiceStore(hs.get_db_conn(), hs)
 
         e = cm.exception
         self.assertIn(f1, str(e))
diff --git a/tests/storage/test_directory.py b/tests/storage/test_directory.py
index b4510c1c8d2b..4e128e10478e 100644
--- a/tests/storage/test_directory.py
+++ b/tests/storage/test_directory.py
@@ -16,7 +16,6 @@
 
 from twisted.internet import defer
 
-from synapse.storage.directory import DirectoryStore
 from synapse.types import RoomAlias, RoomID
 
 from tests import unittest
@@ -28,7 +27,7 @@ class DirectoryStoreTestCase(unittest.TestCase):
     def setUp(self):
         hs = yield setup_test_homeserver(self.addCleanup)
 
-        self.store = DirectoryStore(None, hs)
+        self.store = hs.get_datastore()
 
         self.room = RoomID.from_string("!abcde:test")
         self.alias = RoomAlias.from_string("#my-room:test")
diff --git a/tests/storage/test_event_federation.py b/tests/storage/test_event_federation.py
index 2fdf34fdf63a..0d4e74d63719 100644
--- a/tests/storage/test_event_federation.py
+++ b/tests/storage/test_event_federation.py
@@ -37,10 +37,10 @@ def insert_event(txn, i):
                 (
                     "INSERT INTO events ("
                     "   room_id, event_id, type, depth, topological_ordering,"
-                    "   content, processed, outlier) "
-                    "VALUES (?, ?, 'm.test', ?, ?, 'test', ?, ?)"
+                    "   content, processed, outlier, stream_ordering) "
+                    "VALUES (?, ?, 'm.test', ?, ?, 'test', ?, ?, ?)"
                 ),
-                (room_id, event_id, i, i, True, False),
+                (room_id, event_id, i, i, True, False, i),
             )
 
             txn.execute(
diff --git a/tests/storage/test_monthly_active_users.py b/tests/storage/test_monthly_active_users.py
index f2ed866ae7fc..2036287288ac 100644
--- a/tests/storage/test_monthly_active_users.py
+++ b/tests/storage/test_monthly_active_users.py
@@ -13,25 +13,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from twisted.internet import defer
-
-import tests.unittest
-import tests.utils
-from tests.utils import setup_test_homeserver
+from tests.unittest import HomeserverTestCase
 
 FORTY_DAYS = 40 * 24 * 60 * 60
 
 
-class MonthlyActiveUsersTestCase(tests.unittest.TestCase):
-    def __init__(self, *args, **kwargs):
-        super(MonthlyActiveUsersTestCase, self).__init__(*args, **kwargs)
+class MonthlyActiveUsersTestCase(HomeserverTestCase):
+    def make_homeserver(self, reactor, clock):
+
+        hs = self.setup_test_homeserver()
+        self.store = hs.get_datastore()
+
+        # Advance the clock a bit
+        reactor.advance(FORTY_DAYS)
 
-    @defer.inlineCallbacks
-    def setUp(self):
-        self.hs = yield setup_test_homeserver(self.addCleanup)
-        self.store = self.hs.get_datastore()
+        return hs
 
-    @defer.inlineCallbacks
     def test_initialise_reserved_users(self):
         self.hs.config.max_mau_value = 5
         user1 = "@user1:server"
@@ -44,88 +41,101 @@ def test_initialise_reserved_users(self):
         ]
         user_num = len(threepids)
 
-        yield self.store.register(user_id=user1, token="123", password_hash=None)
-
-        yield self.store.register(user_id=user2, token="456", password_hash=None)
+        self.store.register(user_id=user1, token="123", password_hash=None)
+        self.store.register(user_id=user2, token="456", password_hash=None)
+        self.pump()
 
         now = int(self.hs.get_clock().time_msec())
-        yield self.store.user_add_threepid(user1, "email", user1_email, now, now)
-        yield self.store.user_add_threepid(user2, "email", user2_email, now, now)
-        yield self.store.initialise_reserved_users(threepids)
+        self.store.user_add_threepid(user1, "email", user1_email, now, now)
+        self.store.user_add_threepid(user2, "email", user2_email, now, now)
+        self.store.initialise_reserved_users(threepids)
+        self.pump()
 
-        active_count = yield self.store.get_monthly_active_count()
+        active_count = self.store.get_monthly_active_count()
 
         # Test total counts
-        self.assertEquals(active_count, user_num)
+        self.assertEquals(self.get_success(active_count), user_num)
 
         # Test user is marked as active
-
-        timestamp = yield self.store.user_last_seen_monthly_active(user1)
-        self.assertTrue(timestamp)
-        timestamp = yield self.store.user_last_seen_monthly_active(user2)
-        self.assertTrue(timestamp)
+        timestamp = self.store.user_last_seen_monthly_active(user1)
+        self.assertTrue(self.get_success(timestamp))
+        timestamp = self.store.user_last_seen_monthly_active(user2)
+        self.assertTrue(self.get_success(timestamp))
 
         # Test that users are never removed from the db.
         self.hs.config.max_mau_value = 0
 
-        self.hs.get_clock().advance_time(FORTY_DAYS)
+        self.reactor.advance(FORTY_DAYS)
 
-        yield self.store.reap_monthly_active_users()
+        self.store.reap_monthly_active_users()
+        self.pump()
 
-        active_count = yield self.store.get_monthly_active_count()
-        self.assertEquals(active_count, user_num)
+        active_count = self.store.get_monthly_active_count()
+        self.assertEquals(self.get_success(active_count), user_num)
 
         # Test that regalar users are removed from the db
         ru_count = 2
-        yield self.store.upsert_monthly_active_user("@ru1:server")
-        yield self.store.upsert_monthly_active_user("@ru2:server")
-        active_count = yield self.store.get_monthly_active_count()
+        self.store.upsert_monthly_active_user("@ru1:server")
+        self.store.upsert_monthly_active_user("@ru2:server")
+        self.pump()
 
-        self.assertEqual(active_count, user_num + ru_count)
+        active_count = self.store.get_monthly_active_count()
+        self.assertEqual(self.get_success(active_count), user_num + ru_count)
         self.hs.config.max_mau_value = user_num
-        yield self.store.reap_monthly_active_users()
+        self.store.reap_monthly_active_users()
+        self.pump()
 
-        active_count = yield self.store.get_monthly_active_count()
-        self.assertEquals(active_count, user_num)
+        active_count = self.store.get_monthly_active_count()
+        self.assertEquals(self.get_success(active_count), user_num)
 
-    @defer.inlineCallbacks
     def test_can_insert_and_count_mau(self):
-        count = yield self.store.get_monthly_active_count()
-        self.assertEqual(0, count)
+        count = self.store.get_monthly_active_count()
+        self.assertEqual(0, self.get_success(count))
 
-        yield self.store.upsert_monthly_active_user("@user:server")
-        count = yield self.store.get_monthly_active_count()
+        self.store.upsert_monthly_active_user("@user:server")
+        self.pump()
 
-        self.assertEqual(1, count)
+        count = self.store.get_monthly_active_count()
+        self.assertEqual(1, self.get_success(count))
 
-    @defer.inlineCallbacks
     def test_user_last_seen_monthly_active(self):
         user_id1 = "@user1:server"
         user_id2 = "@user2:server"
         user_id3 = "@user3:server"
 
-        result = yield self.store.user_last_seen_monthly_active(user_id1)
-        self.assertFalse(result == 0)
-        yield self.store.upsert_monthly_active_user(user_id1)
-        yield self.store.upsert_monthly_active_user(user_id2)
-        result = yield self.store.user_last_seen_monthly_active(user_id1)
-        self.assertTrue(result > 0)
-        result = yield self.store.user_last_seen_monthly_active(user_id3)
-        self.assertFalse(result == 0)
+        result = self.store.user_last_seen_monthly_active(user_id1)
+        self.assertFalse(self.get_success(result) == 0)
+
+        self.store.upsert_monthly_active_user(user_id1)
+        self.store.upsert_monthly_active_user(user_id2)
+        self.pump()
+
+        result = self.store.user_last_seen_monthly_active(user_id1)
+        self.assertGreater(self.get_success(result), 0)
+
+        result = self.store.user_last_seen_monthly_active(user_id3)
+        self.assertNotEqual(self.get_success(result), 0)
 
-    @defer.inlineCallbacks
     def test_reap_monthly_active_users(self):
         self.hs.config.max_mau_value = 5
         initial_users = 10
         for i in range(initial_users):
-            yield self.store.upsert_monthly_active_user("@user%d:server" % i)
-        count = yield self.store.get_monthly_active_count()
-        self.assertTrue(count, initial_users)
-        yield self.store.reap_monthly_active_users()
-        count = yield self.store.get_monthly_active_count()
-        self.assertEquals(count, initial_users - self.hs.config.max_mau_value)
-
-        self.hs.get_clock().advance_time(FORTY_DAYS)
-        yield self.store.reap_monthly_active_users()
-        count = yield self.store.get_monthly_active_count()
-        self.assertEquals(count, 0)
+            self.store.upsert_monthly_active_user("@user%d:server" % i)
+        self.pump()
+
+        count = self.store.get_monthly_active_count()
+        self.assertTrue(self.get_success(count), initial_users)
+
+        self.store.reap_monthly_active_users()
+        self.pump()
+        count = self.store.get_monthly_active_count()
+        self.assertEquals(
+            self.get_success(count), initial_users - self.hs.config.max_mau_value
+        )
+
+        self.reactor.advance(FORTY_DAYS)
+        self.store.reap_monthly_active_users()
+        self.pump()
+
+        count = self.store.get_monthly_active_count()
+        self.assertEquals(self.get_success(count), 0)
diff --git a/tests/storage/test_presence.py b/tests/storage/test_presence.py
index b5b58ff660a3..c7a63f39b9a0 100644
--- a/tests/storage/test_presence.py
+++ b/tests/storage/test_presence.py
@@ -16,19 +16,18 @@
 
 from twisted.internet import defer
 
-from synapse.storage.presence import PresenceStore
 from synapse.types import UserID
 
 from tests import unittest
-from tests.utils import MockClock, setup_test_homeserver
+from tests.utils import setup_test_homeserver
 
 
 class PresenceStoreTestCase(unittest.TestCase):
     @defer.inlineCallbacks
     def setUp(self):
-        hs = yield setup_test_homeserver(self.addCleanup, clock=MockClock())
+        hs = yield setup_test_homeserver(self.addCleanup)
 
-        self.store = PresenceStore(None, hs)
+        self.store = hs.get_datastore()
 
         self.u_apple = UserID.from_string("@apple:test")
         self.u_banana = UserID.from_string("@banana:test")
diff --git a/tests/storage/test_profile.py b/tests/storage/test_profile.py
index a1f6618bf94b..45824bd3b2e5 100644
--- a/tests/storage/test_profile.py
+++ b/tests/storage/test_profile.py
@@ -28,7 +28,7 @@ class ProfileStoreTestCase(unittest.TestCase):
     def setUp(self):
         hs = yield setup_test_homeserver(self.addCleanup)
 
-        self.store = ProfileStore(None, hs)
+        self.store = ProfileStore(hs.get_db_conn(), hs)
 
         self.u_frank = UserID.from_string("@frank:test")
 
diff --git a/tests/storage/test_user_directory.py b/tests/storage/test_user_directory.py
index b46e0ea7e279..0dde1ab2fea3 100644
--- a/tests/storage/test_user_directory.py
+++ b/tests/storage/test_user_directory.py
@@ -30,7 +30,7 @@ class UserDirectoryStoreTestCase(unittest.TestCase):
     @defer.inlineCallbacks
     def setUp(self):
         self.hs = yield setup_test_homeserver(self.addCleanup)
-        self.store = UserDirectoryStore(None, self.hs)
+        self.store = UserDirectoryStore(self.hs.get_db_conn(), self.hs)
 
         # alice and bob are both in !room_id. bobby is not but shares
         # a homeserver with alice.
diff --git a/tests/test_visibility.py b/tests/test_visibility.py
index 8d8ce0cab976..2eea3b098b1f 100644
--- a/tests/test_visibility.py
+++ b/tests/test_visibility.py
@@ -96,7 +96,7 @@ def test_erased_user(self):
         events_to_filter.append(evt)
 
         # the erasey user gets erased
-        self.hs.get_datastore().mark_user_erased("@erased:local_hs")
+        yield self.hs.get_datastore().mark_user_erased("@erased:local_hs")
 
         # ... and the filtering happens.
         filtered = yield filter_events_for_server(
diff --git a/tests/unittest.py b/tests/unittest.py
index 8b513bb32b28..a3d39920db0a 100644
--- a/tests/unittest.py
+++ b/tests/unittest.py
@@ -22,6 +22,7 @@
 
 import twisted
 import twisted.logger
+from twisted.internet.defer import Deferred
 from twisted.trial import unittest
 
 from synapse.http.server import JsonResource
@@ -281,12 +282,14 @@ def setup_test_homeserver(self, *args, **kwargs):
         kwargs.update(self._hs_args)
         return setup_test_homeserver(self.addCleanup, *args, **kwargs)
 
-    def pump(self):
+    def pump(self, by=0.0):
         """
         Pump the reactor enough that Deferreds will fire.
         """
-        self.reactor.pump([0.0] * 100)
+        self.reactor.pump([by] * 100)
 
     def get_success(self, d):
+        if not isinstance(d, Deferred):
+            return d
         self.pump()
         return self.successResultOf(d)
diff --git a/tests/utils.py b/tests/utils.py
index 63e30dc6c033..114470d64118 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -30,8 +30,8 @@
 from synapse.federation.transport import server
 from synapse.http.server import HttpServer
 from synapse.server import HomeServer
-from synapse.storage import DataStore, PostgresEngine
-from synapse.storage.engines import create_engine
+from synapse.storage import DataStore
+from synapse.storage.engines import PostgresEngine, create_engine
 from synapse.storage.prepare_database import (
     _get_or_create_schema_state,
     _setup_new_database,
@@ -42,6 +42,7 @@
 
 # set this to True to run the tests against postgres instead of sqlite.
 USE_POSTGRES_FOR_TESTS = os.environ.get("SYNAPSE_POSTGRES", False)
+LEAVE_DB = os.environ.get("SYNAPSE_LEAVE_DB", False)
 POSTGRES_USER = os.environ.get("SYNAPSE_POSTGRES_USER", "postgres")
 POSTGRES_BASE_DB = "_synapse_unit_tests_base_%s" % (os.getpid(),)
 
@@ -244,8 +245,9 @@ def cleanup():
                 cur.close()
                 db_conn.close()
 
-            # Register the cleanup hook
-            cleanup_func(cleanup)
+            if not LEAVE_DB:
+                # Register the cleanup hook
+                cleanup_func(cleanup)
 
         hs.setup()
     else:

From b13836da7f5951bfe3973819b5affa12805185cf Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 3 Sep 2018 17:22:49 +0100
Subject: [PATCH 538/830] Remove conn_id from repl prometheus metrics

`conn_id` gets set to a random string, and so we end up filling up
prometheus with tonnes of data series, which is bad.
---
 synapse/replication/tcp/protocol.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/synapse/replication/tcp/protocol.py b/synapse/replication/tcp/protocol.py
index 74e892c10416..0d28152703bb 100644
--- a/synapse/replication/tcp/protocol.py
+++ b/synapse/replication/tcp/protocol.py
@@ -607,9 +607,9 @@ def transport_buffer_size(protocol):
 transport_send_buffer = LaterGauge(
     "synapse_replication_tcp_protocol_transport_send_buffer",
     "",
-    ["name", "conn_id"],
+    ["name"],
     lambda: {
-        (p.name, p.conn_id): transport_buffer_size(p) for p in connected_connections
+        (p.name,): transport_buffer_size(p) for p in connected_connections
     },
 )
 
@@ -632,9 +632,9 @@ def transport_kernel_read_buffer_size(protocol, read=True):
 tcp_transport_kernel_send_buffer = LaterGauge(
     "synapse_replication_tcp_protocol_transport_kernel_send_buffer",
     "",
-    ["name", "conn_id"],
+    ["name"],
     lambda: {
-        (p.name, p.conn_id): transport_kernel_read_buffer_size(p, False)
+        (p.name,): transport_kernel_read_buffer_size(p, False)
         for p in connected_connections
     },
 )
@@ -643,9 +643,9 @@ def transport_kernel_read_buffer_size(protocol, read=True):
 tcp_transport_kernel_read_buffer = LaterGauge(
     "synapse_replication_tcp_protocol_transport_kernel_read_buffer",
     "",
-    ["name", "conn_id"],
+    ["name"],
     lambda: {
-        (p.name, p.conn_id): transport_kernel_read_buffer_size(p, True)
+        (p.name,): transport_kernel_read_buffer_size(p, True)
         for p in connected_connections
     },
 )
@@ -654,9 +654,9 @@ def transport_kernel_read_buffer_size(protocol, read=True):
 tcp_inbound_commands = LaterGauge(
     "synapse_replication_tcp_protocol_inbound_commands",
     "",
-    ["command", "name", "conn_id"],
+    ["command", "name"],
     lambda: {
-        (k[0], p.name, p.conn_id): count
+        (k[0], p.name,): count
         for p in connected_connections
         for k, count in iteritems(p.inbound_commands_counter)
     },
@@ -665,9 +665,9 @@ def transport_kernel_read_buffer_size(protocol, read=True):
 tcp_outbound_commands = LaterGauge(
     "synapse_replication_tcp_protocol_outbound_commands",
     "",
-    ["command", "name", "conn_id"],
+    ["command", "name"],
     lambda: {
-        (k[0], p.name, p.conn_id): count
+        (k[0], p.name,): count
         for p in connected_connections
         for k, count in iteritems(p.outbound_commands_counter)
     },

From 87b111f96a244cd212e8fff7f1a0f661fdd8073e Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 3 Sep 2018 17:26:15 +0100
Subject: [PATCH 539/830] Newsfile

---
 changelog.d/3788.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3788.bugfix

diff --git a/changelog.d/3788.bugfix b/changelog.d/3788.bugfix
new file mode 100644
index 000000000000..72316fb881fa
--- /dev/null
+++ b/changelog.d/3788.bugfix
@@ -0,0 +1 @@
+Remove connection ID for replication prometheus metrics, as it creates a large number of new series.

From 3e242dc14976bf455717c5f631a997a2c0e5f2c4 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 4 Sep 2018 11:45:52 +0100
Subject: [PATCH 540/830] Remove conn_id

---
 synapse/replication/tcp/protocol.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/synapse/replication/tcp/protocol.py b/synapse/replication/tcp/protocol.py
index 0d28152703bb..5dc7b3fffcf3 100644
--- a/synapse/replication/tcp/protocol.py
+++ b/synapse/replication/tcp/protocol.py
@@ -590,9 +590,9 @@ def on_connection_closed(self):
 pending_commands = LaterGauge(
     "synapse_replication_tcp_protocol_pending_commands",
     "",
-    ["name", "conn_id"],
+    ["name"],
     lambda: {
-        (p.name, p.conn_id): len(p.pending_commands) for p in connected_connections
+        (p.name,): len(p.pending_commands) for p in connected_connections
     },
 )
 

From c42f7fd7b9f99923d71ad064fff7d75a5fc3cd93 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Tue, 4 Sep 2018 12:03:17 +0100
Subject: [PATCH 541/830] improve human readable error messages

---
 synapse/rest/client/v2_alpha/account.py  | 16 ++++++++++++----
 synapse/rest/client/v2_alpha/register.py | 11 ++++++++---
 2 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py
index 372648cafdab..37b32dd37b6a 100644
--- a/synapse/rest/client/v2_alpha/account.py
+++ b/synapse/rest/client/v2_alpha/account.py
@@ -53,7 +53,9 @@ def on_POST(self, request):
 
         if not check_3pid_allowed(self.hs, "email", body['email']):
             raise SynapseError(
-                403, "Third party identifier is not allowed", Codes.THREEPID_DENIED,
+                403,
+                "Your email domain is not authorized on this server",
+                Codes.THREEPID_DENIED,
             )
 
         existingUid = yield self.hs.get_datastore().get_user_id_by_threepid(
@@ -89,7 +91,9 @@ def on_POST(self, request):
 
         if not check_3pid_allowed(self.hs, "msisdn", msisdn):
             raise SynapseError(
-                403, "Third party identifier is not allowed", Codes.THREEPID_DENIED,
+                403,
+                "Account phone numbers are not authorized on this server",
+                Codes.THREEPID_DENIED,
             )
 
         existingUid = yield self.datastore.get_user_id_by_threepid(
@@ -241,7 +245,9 @@ def on_POST(self, request):
 
         if not check_3pid_allowed(self.hs, "email", body['email']):
             raise SynapseError(
-                403, "Third party identifier is not allowed", Codes.THREEPID_DENIED,
+                403,
+                "Your email domain is not authorized on this server",
+                Codes.THREEPID_DENIED,
             )
 
         existingUid = yield self.datastore.get_user_id_by_threepid(
@@ -276,7 +282,9 @@ def on_POST(self, request):
 
         if not check_3pid_allowed(self.hs, "msisdn", msisdn):
             raise SynapseError(
-                403, "Third party identifier is not allowed", Codes.THREEPID_DENIED,
+                403,
+                "Account phone numbers are not authorized on this server",
+                Codes.THREEPID_DENIED,
             )
 
         existingUid = yield self.datastore.get_user_id_by_threepid(
diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index 2fb4d43ccb18..200b29fa387b 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -75,7 +75,9 @@ def on_POST(self, request):
 
         if not check_3pid_allowed(self.hs, "email", body['email']):
             raise SynapseError(
-                403, "Third party identifier is not allowed", Codes.THREEPID_DENIED,
+                403,
+                "Your email domain is not authorized to register on this server",
+                Codes.THREEPID_DENIED,
             )
 
         existingUid = yield self.hs.get_datastore().get_user_id_by_threepid(
@@ -115,7 +117,9 @@ def on_POST(self, request):
 
         if not check_3pid_allowed(self.hs, "msisdn", msisdn):
             raise SynapseError(
-                403, "Third party identifier is not allowed", Codes.THREEPID_DENIED,
+                403,
+                "Phone numbers are not authorized to register on this server",
+                Codes.THREEPID_DENIED,
             )
 
         existingUid = yield self.hs.get_datastore().get_user_id_by_threepid(
@@ -373,7 +377,8 @@ def on_POST(self, request):
 
                     if not check_3pid_allowed(self.hs, medium, address):
                         raise SynapseError(
-                            403, "Third party identifier is not allowed",
+                            403,
+                            "Third party identifier (email/phone number) is not allowed",
                             Codes.THREEPID_DENIED,
                         )
 

From bae37cd811fd71c68d28eb300eb0b74a5b6a1c27 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Tue, 4 Sep 2018 12:07:00 +0100
Subject: [PATCH 542/830] improve human readable error message

---
 synapse/rest/client/v2_alpha/register.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index 200b29fa387b..192f52e46271 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -378,7 +378,8 @@ def on_POST(self, request):
                     if not check_3pid_allowed(self.hs, medium, address):
                         raise SynapseError(
                             403,
-                            "Third party identifier (email/phone number) is not allowed",
+                            "Third party identifiers (email/phone numbers)" +
+                            " are not authorized on this server",
                             Codes.THREEPID_DENIED,
                         )
 

From 70fc599ede85bdeb66ac94bea6f85f3f922f7ca5 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Tue, 4 Sep 2018 12:08:27 +0100
Subject: [PATCH 543/830] towncrier

---
 changelog.d/3789.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3789.misc

diff --git a/changelog.d/3789.misc b/changelog.d/3789.misc
new file mode 100644
index 000000000000..d2d5d91091a4
--- /dev/null
+++ b/changelog.d/3789.misc
@@ -0,0 +1 @@
+Improve human readable error messages for threepid registration/account update

From 7e9ced417846307d88075ed34d7764717a32def0 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Tue, 4 Sep 2018 21:12:04 +1000
Subject: [PATCH 544/830] version and towncrier

---
 CHANGES.md               | 44 ++++++++++++++++++++++++++++++++++++++++
 changelog.d/3378.misc    |  1 -
 changelog.d/3659.feature |  1 -
 changelog.d/3673.misc    |  1 -
 changelog.d/3680.feature |  1 -
 changelog.d/3722.bugfix  |  1 -
 changelog.d/3724.feature |  1 -
 changelog.d/3725.misc    |  1 -
 changelog.d/3726.misc    |  1 -
 changelog.d/3727.misc    |  1 -
 changelog.d/3730.misc    |  1 -
 changelog.d/3734.misc    |  1 -
 changelog.d/3735.misc    |  1 -
 changelog.d/3737.misc    |  1 -
 changelog.d/3740.misc    |  1 -
 changelog.d/3746.misc    |  1 -
 changelog.d/3747.bugfix  |  1 -
 changelog.d/3749.feature |  1 -
 changelog.d/3751.feature |  1 -
 changelog.d/3753.bugfix  |  1 -
 changelog.d/3754.bugfix  |  1 -
 changelog.d/3755.bugfix  |  1 -
 changelog.d/3756.bugfix  |  1 -
 changelog.d/3758.bugfix  |  1 -
 changelog.d/3760.bugfix  |  1 -
 changelog.d/3764.misc    |  1 -
 changelog.d/3768.bugfix  |  1 -
 changelog.d/3777.bugfix  |  1 -
 synapse/__init__.py      |  2 +-
 29 files changed, 45 insertions(+), 28 deletions(-)
 delete mode 100644 changelog.d/3378.misc
 delete mode 100644 changelog.d/3659.feature
 delete mode 100644 changelog.d/3673.misc
 delete mode 100644 changelog.d/3680.feature
 delete mode 100644 changelog.d/3722.bugfix
 delete mode 100644 changelog.d/3724.feature
 delete mode 100644 changelog.d/3725.misc
 delete mode 100644 changelog.d/3726.misc
 delete mode 100644 changelog.d/3727.misc
 delete mode 100644 changelog.d/3730.misc
 delete mode 100644 changelog.d/3734.misc
 delete mode 100644 changelog.d/3735.misc
 delete mode 100644 changelog.d/3737.misc
 delete mode 100644 changelog.d/3740.misc
 delete mode 100644 changelog.d/3746.misc
 delete mode 100644 changelog.d/3747.bugfix
 delete mode 100644 changelog.d/3749.feature
 delete mode 100644 changelog.d/3751.feature
 delete mode 100644 changelog.d/3753.bugfix
 delete mode 100644 changelog.d/3754.bugfix
 delete mode 100644 changelog.d/3755.bugfix
 delete mode 100644 changelog.d/3756.bugfix
 delete mode 100644 changelog.d/3758.bugfix
 delete mode 100644 changelog.d/3760.bugfix
 delete mode 100644 changelog.d/3764.misc
 delete mode 100644 changelog.d/3768.bugfix
 delete mode 100644 changelog.d/3777.bugfix

diff --git a/CHANGES.md b/CHANGES.md
index a35f5aebc795..c2b1e70488d3 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,47 @@
+Synapse 0.33.4rc1 (2018-09-04)
+==============================
+
+Features
+--------
+
+- Support profile API endpoints on workers ([\#3659](https://github.com/matrix-org/synapse/issues/3659))
+- Server notices for resource limit blocking ([\#3680](https://github.com/matrix-org/synapse/issues/3680))
+- Allow guests to use /rooms/:roomId/event/:eventId ([\#3724](https://github.com/matrix-org/synapse/issues/3724))
+- Add mau_trial_days config param, so that users only get counted as MAU after N days. ([\#3749](https://github.com/matrix-org/synapse/issues/3749))
+- Require twisted 17.1 or later (fixes [#3741](https://github.com/matrix-org/synapse/issues/3741)). ([\#3751](https://github.com/matrix-org/synapse/issues/3751))
+
+
+Bugfixes
+--------
+
+- Fix error collecting prometheus metrics when run on dedicated thread due to threading concurrency issues ([\#3722](https://github.com/matrix-org/synapse/issues/3722))
+- Fix bug where we resent "limit exceeded" server notices repeatedly ([\#3747](https://github.com/matrix-org/synapse/issues/3747))
+- Fix bug where we broke sync when using limit_usage_by_mau but hadn't configured server notices ([\#3753](https://github.com/matrix-org/synapse/issues/3753))
+- Fix 'federation_domain_whitelist' such that an empty list correctly blocks all outbound federation traffic ([\#3754](https://github.com/matrix-org/synapse/issues/3754))
+- Fix tagging of server notice rooms ([\#3755](https://github.com/matrix-org/synapse/issues/3755), [\#3756](https://github.com/matrix-org/synapse/issues/3756))
+- Fix 'admin_uri' config variable and error parameter to be 'admin_contact' to match the spec. ([\#3758](https://github.com/matrix-org/synapse/issues/3758))
+- Don't return non-LL-member state in incremental sync state blocks ([\#3760](https://github.com/matrix-org/synapse/issues/3760))
+- Fix bug in sending presence over federation ([\#3768](https://github.com/matrix-org/synapse/issues/3768))
+- Fix bug where preserved threepid user comes to sign up and server is mau blocked ([\#3777](https://github.com/matrix-org/synapse/issues/3777))
+
+
+Internal Changes
+----------------
+
+- Removed the link to the unmaintained matrix-synapse-auto-deploy project from the readme. ([\#3378](https://github.com/matrix-org/synapse/issues/3378))
+- Refactor state module to support multiple room versions ([\#3673](https://github.com/matrix-org/synapse/issues/3673))
+- The synapse.storage module has been ported to Python 3. ([\#3725](https://github.com/matrix-org/synapse/issues/3725))
+- Split the state_group_cache into member and non-member state events (and so speed up LL /sync) ([\#3726](https://github.com/matrix-org/synapse/issues/3726))
+- Log failure to authenticate remote servers as warnings (without stack traces) ([\#3727](https://github.com/matrix-org/synapse/issues/3727))
+- The CONTRIBUTING guidelines have been updated to mention our use of Markdown and that .misc files have content. ([\#3730](https://github.com/matrix-org/synapse/issues/3730))
+- Reference the need for an HTTP replication port when using the federation_reader worker ([\#3734](https://github.com/matrix-org/synapse/issues/3734))
+- Fix minor spelling error in federation client documentation. ([\#3735](https://github.com/matrix-org/synapse/issues/3735))
+- Remove redundant state resolution function ([\#3737](https://github.com/matrix-org/synapse/issues/3737))
+- The test suite now passes on PostgreSQL. ([\#3740](https://github.com/matrix-org/synapse/issues/3740))
+- Fix MAU cache invalidation due to missing yield ([\#3746](https://github.com/matrix-org/synapse/issues/3746))
+- Make sure that we close db connections opened during init ([\#3764](https://github.com/matrix-org/synapse/issues/3764))
+
+
 Synapse 0.33.3 (2018-08-22)
 ===========================
 
diff --git a/changelog.d/3378.misc b/changelog.d/3378.misc
deleted file mode 100644
index 8f88f88e69c6..000000000000
--- a/changelog.d/3378.misc
+++ /dev/null
@@ -1 +0,0 @@
-Removed the link to the unmaintained matrix-synapse-auto-deploy project from the readme.
diff --git a/changelog.d/3659.feature b/changelog.d/3659.feature
deleted file mode 100644
index a5b4821c0953..000000000000
--- a/changelog.d/3659.feature
+++ /dev/null
@@ -1 +0,0 @@
-Support profile API endpoints on workers
diff --git a/changelog.d/3673.misc b/changelog.d/3673.misc
deleted file mode 100644
index d672111fb9ae..000000000000
--- a/changelog.d/3673.misc
+++ /dev/null
@@ -1 +0,0 @@
-Refactor state module to support multiple room versions
diff --git a/changelog.d/3680.feature b/changelog.d/3680.feature
deleted file mode 100644
index 4edaaf76a83e..000000000000
--- a/changelog.d/3680.feature
+++ /dev/null
@@ -1 +0,0 @@
-Server notices for resource limit blocking
diff --git a/changelog.d/3722.bugfix b/changelog.d/3722.bugfix
deleted file mode 100644
index 16cbaf76cbf4..000000000000
--- a/changelog.d/3722.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix error collecting prometheus metrics when run on dedicated thread due to threading concurrency issues
diff --git a/changelog.d/3724.feature b/changelog.d/3724.feature
deleted file mode 100644
index 1b374ccf4760..000000000000
--- a/changelog.d/3724.feature
+++ /dev/null
@@ -1 +0,0 @@
-Allow guests to use /rooms/:roomId/event/:eventId
diff --git a/changelog.d/3725.misc b/changelog.d/3725.misc
deleted file mode 100644
index 91ab9d7137b3..000000000000
--- a/changelog.d/3725.misc
+++ /dev/null
@@ -1 +0,0 @@
-The synapse.storage module has been ported to Python 3.
diff --git a/changelog.d/3726.misc b/changelog.d/3726.misc
deleted file mode 100644
index c4f66ec998af..000000000000
--- a/changelog.d/3726.misc
+++ /dev/null
@@ -1 +0,0 @@
-Split the state_group_cache into member and non-member state events (and so speed up LL /sync)
diff --git a/changelog.d/3727.misc b/changelog.d/3727.misc
deleted file mode 100644
index 0b83220d900a..000000000000
--- a/changelog.d/3727.misc
+++ /dev/null
@@ -1 +0,0 @@
-Log failure to authenticate remote servers as warnings (without stack traces)
diff --git a/changelog.d/3730.misc b/changelog.d/3730.misc
deleted file mode 100644
index b1ea84f732a2..000000000000
--- a/changelog.d/3730.misc
+++ /dev/null
@@ -1 +0,0 @@
-The CONTRIBUTING guidelines have been updated to mention our use of Markdown and that .misc files have content.
diff --git a/changelog.d/3734.misc b/changelog.d/3734.misc
deleted file mode 100644
index 4f6e4b384808..000000000000
--- a/changelog.d/3734.misc
+++ /dev/null
@@ -1 +0,0 @@
-Reference the need for an HTTP replication port when using the federation_reader worker
diff --git a/changelog.d/3735.misc b/changelog.d/3735.misc
deleted file mode 100644
index f17004be71e3..000000000000
--- a/changelog.d/3735.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix minor spelling error in federation client documentation.
diff --git a/changelog.d/3737.misc b/changelog.d/3737.misc
deleted file mode 100644
index 0361da4ebc3b..000000000000
--- a/changelog.d/3737.misc
+++ /dev/null
@@ -1 +0,0 @@
-Remove redundant state resolution function
diff --git a/changelog.d/3740.misc b/changelog.d/3740.misc
deleted file mode 100644
index 4dcb7fb5de6e..000000000000
--- a/changelog.d/3740.misc
+++ /dev/null
@@ -1 +0,0 @@
-The test suite now passes on PostgreSQL.
diff --git a/changelog.d/3746.misc b/changelog.d/3746.misc
deleted file mode 100644
index fc00ee773a82..000000000000
--- a/changelog.d/3746.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix MAU cache invalidation due to missing yield
diff --git a/changelog.d/3747.bugfix b/changelog.d/3747.bugfix
deleted file mode 100644
index c41e2a1213af..000000000000
--- a/changelog.d/3747.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix bug where we resent "limit exceeded" server notices repeatedly
diff --git a/changelog.d/3749.feature b/changelog.d/3749.feature
deleted file mode 100644
index 9f8837b106de..000000000000
--- a/changelog.d/3749.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add mau_trial_days config param, so that users only get counted as MAU after N days.
diff --git a/changelog.d/3751.feature b/changelog.d/3751.feature
deleted file mode 100644
index dc9742b15bd6..000000000000
--- a/changelog.d/3751.feature
+++ /dev/null
@@ -1 +0,0 @@
-Require twisted 17.1 or later (fixes [#3741](https://github.com/matrix-org/synapse/issues/3741)).
diff --git a/changelog.d/3753.bugfix b/changelog.d/3753.bugfix
deleted file mode 100644
index b4301267df36..000000000000
--- a/changelog.d/3753.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix bug where we broke sync when using limit_usage_by_mau but hadn't configured server notices
diff --git a/changelog.d/3754.bugfix b/changelog.d/3754.bugfix
deleted file mode 100644
index 6e3ec8019418..000000000000
--- a/changelog.d/3754.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix 'federation_domain_whitelist' such that an empty list correctly blocks all outbound federation traffic
diff --git a/changelog.d/3755.bugfix b/changelog.d/3755.bugfix
deleted file mode 100644
index 6a1f83f0cee1..000000000000
--- a/changelog.d/3755.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix tagging of server notice rooms
diff --git a/changelog.d/3756.bugfix b/changelog.d/3756.bugfix
deleted file mode 100644
index 6a1f83f0cee1..000000000000
--- a/changelog.d/3756.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix tagging of server notice rooms
diff --git a/changelog.d/3758.bugfix b/changelog.d/3758.bugfix
deleted file mode 100644
index 862739bfe8e3..000000000000
--- a/changelog.d/3758.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix 'admin_uri' config variable and error parameter to be 'admin_contact' to match the spec.
diff --git a/changelog.d/3760.bugfix b/changelog.d/3760.bugfix
deleted file mode 100644
index ce61fb8a2bc8..000000000000
--- a/changelog.d/3760.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Don't return non-LL-member state in incremental sync state blocks
diff --git a/changelog.d/3764.misc b/changelog.d/3764.misc
deleted file mode 100644
index f3614f1982e1..000000000000
--- a/changelog.d/3764.misc
+++ /dev/null
@@ -1 +0,0 @@
-Make sure that we close db connections opened during init
\ No newline at end of file
diff --git a/changelog.d/3768.bugfix b/changelog.d/3768.bugfix
deleted file mode 100644
index a039a7fa685b..000000000000
--- a/changelog.d/3768.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix bug in sending presence over federation
diff --git a/changelog.d/3777.bugfix b/changelog.d/3777.bugfix
deleted file mode 100644
index 46efc543a7b4..000000000000
--- a/changelog.d/3777.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix bug where preserved threepid user comes to sign up and server is mau blocked
diff --git a/synapse/__init__.py b/synapse/__init__.py
index e62901b7614b..2c42893673e1 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -17,4 +17,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.33.3"
+__version__ = "0.33.4rc1"

From 87c18d12ee30c84a108ac686454720d6ac6a2f84 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 4 Sep 2018 15:18:25 +0100
Subject: [PATCH 545/830] Implement 'event_format' filter param in /sync

This has been specced and part-implemented; let's implement it for /sync (but
no other endpoints yet :/).
---
 synapse/api/filtering.py             |  1 +
 synapse/rest/client/v2_alpha/sync.py | 51 +++++++++++++++++++++-------
 2 files changed, 39 insertions(+), 13 deletions(-)

diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py
index 186831e1181f..a31a9a17e0b7 100644
--- a/synapse/api/filtering.py
+++ b/synapse/api/filtering.py
@@ -251,6 +251,7 @@ def __init__(self, filter_json):
             "include_leave", False
         )
         self.event_fields = filter_json.get("event_fields", [])
+        self.event_format = filter_json.get("event_format", "client")
 
     def __repr__(self):
         return "<FilterCollection %s>" % (json.dumps(self._filter_json),)
diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py
index 1275baa1ba9a..263d8eb73eb1 100644
--- a/synapse/rest/client/v2_alpha/sync.py
+++ b/synapse/rest/client/v2_alpha/sync.py
@@ -25,6 +25,7 @@
 from synapse.api.filtering import DEFAULT_FILTER_COLLECTION, FilterCollection
 from synapse.events.utils import (
     format_event_for_client_v2_without_room_id,
+    format_event_raw,
     serialize_event,
 )
 from synapse.handlers.presence import format_user_presence_state
@@ -175,17 +176,28 @@ def on_GET(self, request):
 
     @staticmethod
     def encode_response(time_now, sync_result, access_token_id, filter):
+        if filter.event_format == 'client':
+            event_formatter = format_event_for_client_v2_without_room_id
+        elif filter.event_format == 'federation':
+            event_formatter = format_event_raw
+        else:
+            raise Exception("Unknown event format %s" % (filter.event_format, ))
+
         joined = SyncRestServlet.encode_joined(
-            sync_result.joined, time_now, access_token_id, filter.event_fields
+            sync_result.joined, time_now, access_token_id,
+            filter.event_fields,
+            event_formatter,
         )
 
         invited = SyncRestServlet.encode_invited(
             sync_result.invited, time_now, access_token_id,
+            event_formatter,
         )
 
         archived = SyncRestServlet.encode_archived(
             sync_result.archived, time_now, access_token_id,
             filter.event_fields,
+            event_formatter,
         )
 
         return {
@@ -228,7 +240,7 @@ def encode_presence(events, time_now):
         }
 
     @staticmethod
-    def encode_joined(rooms, time_now, token_id, event_fields):
+    def encode_joined(rooms, time_now, token_id, event_fields, event_formatter):
         """
         Encode the joined rooms in a sync result
 
@@ -240,7 +252,9 @@ def encode_joined(rooms, time_now, token_id, event_fields):
             token_id(int): ID of the user's auth token - used for namespacing
                 of transaction IDs
             event_fields(list<str>): List of event fields to include. If empty,
-            all fields will be returned.
+                all fields will be returned.
+            event_formatter (func[dict]): function to convert from federation format
+                to client format
         Returns:
             dict[str, dict[str, object]]: the joined rooms list, in our
                 response format
@@ -248,13 +262,14 @@ def encode_joined(rooms, time_now, token_id, event_fields):
         joined = {}
         for room in rooms:
             joined[room.room_id] = SyncRestServlet.encode_room(
-                room, time_now, token_id, only_fields=event_fields
+                room, time_now, token_id, joined=True, only_fields=event_fields,
+                event_formatter=event_formatter,
             )
 
         return joined
 
     @staticmethod
-    def encode_invited(rooms, time_now, token_id):
+    def encode_invited(rooms, time_now, token_id, event_formatter):
         """
         Encode the invited rooms in a sync result
 
@@ -264,7 +279,9 @@ def encode_invited(rooms, time_now, token_id):
             time_now(int): current time - used as a baseline for age
                 calculations
             token_id(int): ID of the user's auth token - used for namespacing
-            of transaction IDs
+                of transaction IDs
+            event_formatter (func[dict]): function to convert from federation format
+                to client format
 
         Returns:
             dict[str, dict[str, object]]: the invited rooms list, in our
@@ -274,7 +291,7 @@ def encode_invited(rooms, time_now, token_id):
         for room in rooms:
             invite = serialize_event(
                 room.invite, time_now, token_id=token_id,
-                event_format=format_event_for_client_v2_without_room_id,
+                event_format=event_formatter,
                 is_invite=True,
             )
             unsigned = dict(invite.get("unsigned", {}))
@@ -288,7 +305,7 @@ def encode_invited(rooms, time_now, token_id):
         return invited
 
     @staticmethod
-    def encode_archived(rooms, time_now, token_id, event_fields):
+    def encode_archived(rooms, time_now, token_id, event_fields, event_formatter):
         """
         Encode the archived rooms in a sync result
 
@@ -300,7 +317,9 @@ def encode_archived(rooms, time_now, token_id, event_fields):
             token_id(int): ID of the user's auth token - used for namespacing
                 of transaction IDs
             event_fields(list<str>): List of event fields to include. If empty,
-            all fields will be returned.
+                all fields will be returned.
+            event_formatter (func[dict]): function to convert from federation format
+                to client format
         Returns:
             dict[str, dict[str, object]]: The invited rooms list, in our
                 response format
@@ -308,13 +327,18 @@ def encode_archived(rooms, time_now, token_id, event_fields):
         joined = {}
         for room in rooms:
             joined[room.room_id] = SyncRestServlet.encode_room(
-                room, time_now, token_id, joined=False, only_fields=event_fields
+                room, time_now, token_id, joined=False,
+                only_fields=event_fields,
+                event_formatter=event_formatter,
             )
 
         return joined
 
     @staticmethod
-    def encode_room(room, time_now, token_id, joined=True, only_fields=None):
+    def encode_room(
+            room, time_now, token_id, joined,
+            only_fields, event_formatter,
+    ):
         """
         Args:
             room (JoinedSyncResult|ArchivedSyncResult): sync result for a
@@ -326,14 +350,15 @@ def encode_room(room, time_now, token_id, joined=True, only_fields=None):
             joined (bool): True if the user is joined to this room - will mean
                 we handle ephemeral events
             only_fields(list<str>): Optional. The list of event fields to include.
+            event_formatter (func[dict]): function to convert from federation format
+                to client format
         Returns:
             dict[str, object]: the room, encoded in our response format
         """
         def serialize(event):
-            # TODO(mjark): Respect formatting requirements in the filter.
             return serialize_event(
                 event, time_now, token_id=token_id,
-                event_format=format_event_for_client_v2_without_room_id,
+                event_format=event_formatter,
                 only_event_fields=only_fields,
             )
 

From c91bd295f58947d1cded7dc985b7ae55b99b3a4f Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 4 Sep 2018 15:23:19 +0100
Subject: [PATCH 546/830] changelog

---
 changelog.d/3790.feature | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3790.feature

diff --git a/changelog.d/3790.feature b/changelog.d/3790.feature
new file mode 100644
index 000000000000..2c4ac62fb570
--- /dev/null
+++ b/changelog.d/3790.feature
@@ -0,0 +1 @@
+Implement `event_format` filter param in `/sync`

From 5f02017aead05adf127b89d7e203849982bd7fdc Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 5 Sep 2018 10:20:40 +0100
Subject: [PATCH 547/830] Improve performance of getting typing updates for
 replication

Fetching the list of all new typing notifications involved iterating
over all rooms and comparing their serial. Lets move to using a stream
change cache, like we do for other streams.
---
 synapse/handlers/typing.py | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py
index 2d2d3d5a0d91..65f475d639d8 100644
--- a/synapse/handlers/typing.py
+++ b/synapse/handlers/typing.py
@@ -20,6 +20,7 @@
 
 from synapse.api.errors import AuthError, SynapseError
 from synapse.types import UserID, get_domain_from_id
+from synapse.util.caches.stream_change_cache import StreamChangeCache
 from synapse.util.logcontext import run_in_background
 from synapse.util.metrics import Measure
 from synapse.util.wheel_timer import WheelTimer
@@ -68,6 +69,11 @@ def __init__(self, hs):
         # map room IDs to sets of users currently typing
         self._room_typing = {}
 
+        # caches which room_ids changed at which serials
+        self._typing_stream_change_cache = StreamChangeCache(
+            "TypingStreamChangeCache", self._latest_room_serial,
+        )
+
         self.clock.looping_call(
             self._handle_timeouts,
             5000,
@@ -274,19 +280,29 @@ def _push_update_local(self, member, typing):
 
         self._latest_room_serial += 1
         self._room_serials[member.room_id] = self._latest_room_serial
+        self._typing_stream_change_cache.entity_has_changed(
+            member.room_id, self._latest_room_serial,
+        )
 
         self.notifier.on_new_event(
             "typing_key", self._latest_room_serial, rooms=[member.room_id]
         )
 
     def get_all_typing_updates(self, last_id, current_id):
-        # TODO: Work out a way to do this without scanning the entire state.
         if last_id == current_id:
             return []
 
+        changed_rooms = self._typing_stream_change_cache.get_all_entities_changed(
+            last_id,
+        )
+
+        if changed_rooms is None:
+            changed_rooms = self._room_serials
+
         rows = []
-        for room_id, serial in self._room_serials.items():
-            if last_id < serial and serial <= current_id:
+        for room_id in changed_rooms:
+            serial = self._room_serials[room_id]
+            if last_id < serial <= current_id:
                 typing = self._room_typing[room_id]
                 rows.append((serial, room_id, list(typing)))
         rows.sort()

From 804dd41e18c449e711e443398b95c9f6c68b6fa2 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 4 Sep 2018 01:09:12 +0100
Subject: [PATCH 548/830] Check that signatures on events are valid

We should check that both the sender's server, and the server which created the
event_id (which may be different from whatever the remote server has told us
the origin is), have signed the event.
---
 synapse/federation/federation_base.py | 126 ++++++++++++++++++++++----
 1 file changed, 110 insertions(+), 16 deletions(-)

diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py
index c11798093d7a..5be8e66fb81d 100644
--- a/synapse/federation/federation_base.py
+++ b/synapse/federation/federation_base.py
@@ -13,17 +13,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
+from collections import namedtuple
 
 import six
 
 from twisted.internet import defer
+from twisted.internet.defer import DeferredList
 
-from synapse.api.constants import MAX_DEPTH
+from synapse.api.constants import MAX_DEPTH, EventTypes, Membership
 from synapse.api.errors import Codes, SynapseError
 from synapse.crypto.event_signing import check_event_content_hash
 from synapse.events import FrozenEvent
 from synapse.events.utils import prune_event
 from synapse.http.servlet import assert_params_in_dict
+from synapse.types import get_domain_from_id
 from synapse.util import logcontext, unwrapFirstError
 
 logger = logging.getLogger(__name__)
@@ -133,34 +136,25 @@ def _check_sigs_and_hashes(self, pdus):
               * throws a SynapseError if the signature check failed.
             The deferreds run their callbacks in the sentinel logcontext.
         """
-
-        redacted_pdus = [
-            prune_event(pdu)
-            for pdu in pdus
-        ]
-
-        deferreds = self.keyring.verify_json_objects_for_server([
-            (p.origin, p.get_pdu_json())
-            for p in redacted_pdus
-        ])
+        deferreds = _check_sigs_on_pdus(self.keyring, pdus)
 
         ctx = logcontext.LoggingContext.current_context()
 
-        def callback(_, pdu, redacted):
+        def callback(_, pdu):
             with logcontext.PreserveLoggingContext(ctx):
                 if not check_event_content_hash(pdu):
                     logger.warn(
                         "Event content has been tampered, redacting %s: %s",
                         pdu.event_id, pdu.get_pdu_json()
                     )
-                    return redacted
+                    return prune_event(pdu)
 
                 if self.spam_checker.check_event_for_spam(pdu):
                     logger.warn(
                         "Event contains spam, redacting %s: %s",
                         pdu.event_id, pdu.get_pdu_json()
                     )
-                    return redacted
+                    return prune_event(pdu)
 
                 return pdu
 
@@ -173,16 +167,116 @@ def errback(failure, pdu):
                 )
             return failure
 
-        for deferred, pdu, redacted in zip(deferreds, pdus, redacted_pdus):
+        for deferred, pdu in zip(deferreds, pdus):
             deferred.addCallbacks(
                 callback, errback,
-                callbackArgs=[pdu, redacted],
+                callbackArgs=[pdu],
                 errbackArgs=[pdu],
             )
 
         return deferreds
 
 
+class PduToCheckSig(namedtuple("PduToCheckSig", [
+    "pdu", "redacted_pdu_json", "event_id_domain", "sender_domain", "deferreds",
+])):
+    pass
+
+
+def _check_sigs_on_pdus(keyring, pdus):
+    """Check that the given events are correctly signed
+
+    Args:
+        keyring (synapse.crypto.Keyring): keyring object to do the checks
+        pdus (Collection[EventBase]): the events to be checked
+
+    Returns:
+        List[Deferred]: a Deferred for each event in pdus, which will either succeed if
+           the signatures are valid, or fail (with a SynapseError) if not.
+    """
+
+    # (currently this is written assuming the v1 room structure; we'll probably want a
+    # separate function for checking v2 rooms)
+
+    # we want to check that the event is signed by:
+    #
+    # (a) the server which created the event_id
+    #
+    # (b) the sender's server.
+    #
+    #     - except in the case of invites created from a 3pid invite, which are exempt
+    #     from this check, because the sender has to match that of the original 3pid
+    #     invite, but the event may come from a different HS, for reasons that I don't
+    #     entirely grok (why do the senders have to match? and if they do, why doesn't the
+    #     joining server ask the inviting server to do the switcheroo with
+    #     exchange_third_party_invite?).
+    #
+    #     That's pretty awful, since redacting such an invite will render it invalid
+    #     (because it will then look like a regular invite without a valid signature),
+    #     and signatures are *supposed* to be valid whether or not an event has been
+    #     redacted. But this isn't the worst of the ways that 3pid invites are broken.
+    #
+    # let's start by getting the domain for each pdu, and flattening the event back
+    # to JSON.
+    pdus_to_check = [
+        PduToCheckSig(
+            pdu=p,
+            redacted_pdu_json=prune_event(p).get_pdu_json(),
+            event_id_domain=get_domain_from_id(p.event_id),
+            sender_domain=get_domain_from_id(p.sender),
+            deferreds=[],
+        )
+        for p in pdus
+    ]
+
+    # first make sure that the event is signed by the event_id's domain
+    deferreds = keyring.verify_json_objects_for_server([
+        (p.event_id_domain, p.redacted_pdu_json)
+        for p in pdus_to_check
+    ])
+
+    for p, d in zip(pdus_to_check, deferreds):
+        p.deferreds.append(d)
+
+    # now let's look for events where the sender's domain is different to the
+    # event id's domain (normally only the case for joins/leaves), and add additional
+    # checks.
+    pdus_to_check_sender = [
+        p for p in pdus_to_check
+        if p.sender_domain != p.event_id_domain and not _is_invite_via_3pid(p.pdu)
+    ]
+
+    more_deferreds = keyring.verify_json_objects_for_server([
+        (p.sender_domain, p.redacted_pdu_json)
+        for p in pdus_to_check_sender
+    ])
+
+    for p, d in zip(pdus_to_check_sender, more_deferreds):
+        p.deferreds.append(d)
+
+    # replace lists of deferreds with single Deferreds
+    return [_flatten_deferred_list(p.deferreds) for p in pdus_to_check]
+
+
+def _flatten_deferred_list(deferreds):
+    """Given a list of one or more deferreds, either return the single deferred, or
+    combine into a DeferredList.
+    """
+    if len(deferreds) > 1:
+        return DeferredList(deferreds, fireOnOneErrback=True, consumeErrors=True)
+    else:
+        assert len(deferreds) == 1
+        return deferreds[0]
+
+
+def _is_invite_via_3pid(event):
+    return (
+        event.type == EventTypes.Member
+        and event.membership == Membership.INVITE
+        and "third_party_invite" in event.content
+    )
+
+
 def event_from_pdu_json(pdu_json, outlier=False):
     """Construct a FrozenEvent from an event json received over federation
 

From c127c8d0421f0228a46ebbe280c9537e8d8ea42b Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 4 Sep 2018 01:23:18 +0100
Subject: [PATCH 549/830] Fix origin handling for pushed transactions

Use the actual origin for push transactions, rather than whatever the remote
server claimed.
---
 synapse/federation/federation_server.py | 20 ++++++++++----------
 synapse/federation/persistence.py       |  8 ++++----
 synapse/federation/transport/server.py  |  2 +-
 tests/handlers/test_typing.py           | 19 ++++++++-----------
 tests/utils.py                          | 12 +++++++++---
 5 files changed, 32 insertions(+), 29 deletions(-)

diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index 3e0cd294a1b2..547c6aec80f8 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -99,7 +99,7 @@ def on_backfill_request(self, origin, room_id, versions, limit):
 
     @defer.inlineCallbacks
     @log_function
-    def on_incoming_transaction(self, transaction_data):
+    def on_incoming_transaction(self, origin, transaction_data):
         # keep this as early as possible to make the calculated origin ts as
         # accurate as possible.
         request_time = self._clock.time_msec()
@@ -108,34 +108,33 @@ def on_incoming_transaction(self, transaction_data):
 
         if not transaction.transaction_id:
             raise Exception("Transaction missing transaction_id")
-        if not transaction.origin:
-            raise Exception("Transaction missing origin")
 
         logger.debug("[%s] Got transaction", transaction.transaction_id)
 
         # use a linearizer to ensure that we don't process the same transaction
         # multiple times in parallel.
         with (yield self._transaction_linearizer.queue(
-                (transaction.origin, transaction.transaction_id),
+                (origin, transaction.transaction_id),
         )):
             result = yield self._handle_incoming_transaction(
-                transaction, request_time,
+                origin, transaction, request_time,
             )
 
         defer.returnValue(result)
 
     @defer.inlineCallbacks
-    def _handle_incoming_transaction(self, transaction, request_time):
+    def _handle_incoming_transaction(self, origin, transaction, request_time):
         """ Process an incoming transaction and return the HTTP response
 
         Args:
+            origin (unicode): the server making the request
             transaction (Transaction): incoming transaction
             request_time (int): timestamp that the HTTP request arrived at
 
         Returns:
             Deferred[(int, object)]: http response code and body
         """
-        response = yield self.transaction_actions.have_responded(transaction)
+        response = yield self.transaction_actions.have_responded(origin, transaction)
 
         if response:
             logger.debug(
@@ -149,7 +148,7 @@ def _handle_incoming_transaction(self, transaction, request_time):
 
         received_pdus_counter.inc(len(transaction.pdus))
 
-        origin_host, _ = parse_server_name(transaction.origin)
+        origin_host, _ = parse_server_name(origin)
 
         pdus_by_room = {}
 
@@ -190,7 +189,7 @@ def process_pdus_for_room(room_id):
                 event_id = pdu.event_id
                 try:
                     yield self._handle_received_pdu(
-                        transaction.origin, pdu
+                        origin, pdu
                     )
                     pdu_results[event_id] = {}
                 except FederationError as e:
@@ -212,7 +211,7 @@ def process_pdus_for_room(room_id):
         if hasattr(transaction, "edus"):
             for edu in (Edu(**x) for x in transaction.edus):
                 yield self.received_edu(
-                    transaction.origin,
+                    origin,
                     edu.edu_type,
                     edu.content
                 )
@@ -224,6 +223,7 @@ def process_pdus_for_room(room_id):
         logger.debug("Returning: %s", str(response))
 
         yield self.transaction_actions.set_response(
+            origin,
             transaction,
             200, response
         )
diff --git a/synapse/federation/persistence.py b/synapse/federation/persistence.py
index 9146215c2117..74ffd13b4f89 100644
--- a/synapse/federation/persistence.py
+++ b/synapse/federation/persistence.py
@@ -36,7 +36,7 @@ def __init__(self, datastore):
         self.store = datastore
 
     @log_function
-    def have_responded(self, transaction):
+    def have_responded(self, origin, transaction):
         """ Have we already responded to a transaction with the same id and
         origin?
 
@@ -50,11 +50,11 @@ def have_responded(self, transaction):
                                "transaction_id")
 
         return self.store.get_received_txn_response(
-            transaction.transaction_id, transaction.origin
+            transaction.transaction_id, origin
         )
 
     @log_function
-    def set_response(self, transaction, code, response):
+    def set_response(self, origin, transaction, code, response):
         """ Persist how we responded to a transaction.
 
         Returns:
@@ -66,7 +66,7 @@ def set_response(self, transaction, code, response):
 
         return self.store.set_received_txn_response(
             transaction.transaction_id,
-            transaction.origin,
+            origin,
             code,
             response,
         )
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 77969a4f3804..8cde9716ac80 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -353,7 +353,7 @@ def on_PUT(self, origin, content, query, transaction_id):
 
         try:
             code, response = yield self.handler.on_incoming_transaction(
-                transaction_data
+                origin, transaction_data,
             )
         except Exception:
             logger.exception("on_incoming_transaction failed")
diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py
index ad58073a140c..c2d951b45fe5 100644
--- a/tests/handlers/test_typing.py
+++ b/tests/handlers/test_typing.py
@@ -33,7 +33,7 @@
 )
 
 
-def _expect_edu(destination, edu_type, content, origin="test"):
+def _expect_edu_transaction(edu_type, content, origin="test"):
     return {
         "origin": origin,
         "origin_server_ts": 1000000,
@@ -42,8 +42,8 @@ def _expect_edu(destination, edu_type, content, origin="test"):
     }
 
 
-def _make_edu_json(origin, edu_type, content):
-    return json.dumps(_expect_edu("test", edu_type, content, origin=origin)).encode(
+def _make_edu_transaction_json(edu_type, content):
+    return json.dumps(_expect_edu_transaction(edu_type, content)).encode(
         'utf8'
     )
 
@@ -190,8 +190,7 @@ def test_started_typing_remote_send(self):
             call(
                 "farm",
                 path="/_matrix/federation/v1/send/1000000/",
-                data=_expect_edu(
-                    "farm",
+                data=_expect_edu_transaction(
                     "m.typing",
                     content={
                         "room_id": self.room_id,
@@ -221,11 +220,10 @@ def test_started_typing_remote_recv(self):
 
         self.assertEquals(self.event_source.get_current_key(), 0)
 
-        yield self.mock_federation_resource.trigger(
+        (code, response) = yield self.mock_federation_resource.trigger(
             "PUT",
             "/_matrix/federation/v1/send/1000000/",
-            _make_edu_json(
-                "farm",
+            _make_edu_transaction_json(
                 "m.typing",
                 content={
                     "room_id": self.room_id,
@@ -233,7 +231,7 @@ def test_started_typing_remote_recv(self):
                     "typing": True,
                 },
             ),
-            federation_auth=True,
+            federation_auth_origin=b'farm',
         )
 
         self.on_new_event.assert_has_calls(
@@ -264,8 +262,7 @@ def test_stopped_typing(self):
             call(
                 "farm",
                 path="/_matrix/federation/v1/send/1000000/",
-                data=_expect_edu(
-                    "farm",
+                data=_expect_edu_transaction(
                     "m.typing",
                     content={
                         "room_id": self.room_id,
diff --git a/tests/utils.py b/tests/utils.py
index bb0fc740540c..8de2898b2f4c 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -306,7 +306,10 @@ def trigger_get(self, path):
 
     @patch('twisted.web.http.Request')
     @defer.inlineCallbacks
-    def trigger(self, http_method, path, content, mock_request, federation_auth=False):
+    def trigger(
+        self, http_method, path, content, mock_request,
+        federation_auth_origin=None,
+    ):
         """ Fire an HTTP event.
 
         Args:
@@ -315,6 +318,7 @@ def trigger(self, http_method, path, content, mock_request, federation_auth=Fals
             content : The HTTP body
             mock_request : Mocked request to pass to the event so it can get
                            content.
+            federation_auth_origin (bytes|None): domain to authenticate as, for federation
         Returns:
             A tuple of (code, response)
         Raises:
@@ -335,8 +339,10 @@ def trigger(self, http_method, path, content, mock_request, federation_auth=Fals
         mock_request.getClientIP.return_value = "-"
 
         headers = {}
-        if federation_auth:
-            headers[b"Authorization"] = [b"X-Matrix origin=test,key=,sig="]
+        if federation_auth_origin is not None:
+            headers[b"Authorization"] = [
+                b"X-Matrix origin=%s,key=,sig=" % (federation_auth_origin, )
+            ]
         mock_request.requestHeaders.getRawHeaders = mock_getRawHeaders(headers)
 
         # return the right path if the event requires it

From 2d2828dcbc2c6360d28a64d3849cf849eb5348c4 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Thu, 6 Sep 2018 00:10:47 +1000
Subject: [PATCH 550/830] Port http/ to Python 3 (#3771)

---
 changelog.d/3771.misc                   |   1 +
 setup.cfg                               |   5 +-
 synapse/appservice/api.py               |  13 +-
 synapse/federation/federation_server.py |  10 +-
 synapse/http/client.py                  |  82 ++++------
 synapse/http/matrixfederationclient.py  | 204 ++++++++++--------------
 synapse/http/site.py                    |   4 +-
 synapse/python_dependencies.py          |   1 +
 8 files changed, 134 insertions(+), 186 deletions(-)
 create mode 100644 changelog.d/3771.misc

diff --git a/changelog.d/3771.misc b/changelog.d/3771.misc
new file mode 100644
index 000000000000..47aa34bc04b6
--- /dev/null
+++ b/changelog.d/3771.misc
@@ -0,0 +1 @@
+http/ is now ported to Python 3.
diff --git a/setup.cfg b/setup.cfg
index c2620be6c52c..52feaa9cc7b5 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -17,13 +17,14 @@ ignore =
 [pep8]
 max-line-length = 90
 #  W503 requires that binary operators be at the end, not start, of lines. Erik
-#  doesn't like it.  E203 is contrary to PEP8.
-ignore = W503,E203
+#  doesn't like it.  E203 is contrary to PEP8.  E731 is silly.
+ignore = W503,E203,E731
 
 [flake8]
 # note that flake8 inherits the "ignore" settings from "pep8" (because it uses
 # pep8 to do those checks), but not the "max-line-length" setting
 max-line-length = 90
+ignore=W503,E203,E731
 
 [isort]
 line_length = 89
diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py
index 6980e5890ec4..9ccc5a80fc59 100644
--- a/synapse/appservice/api.py
+++ b/synapse/appservice/api.py
@@ -13,7 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-import urllib
+
+from six.moves import urllib
 
 from prometheus_client import Counter
 
@@ -98,7 +99,7 @@ def __init__(self, hs):
     def query_user(self, service, user_id):
         if service.url is None:
             defer.returnValue(False)
-        uri = service.url + ("/users/%s" % urllib.quote(user_id))
+        uri = service.url + ("/users/%s" % urllib.parse.quote(user_id))
         response = None
         try:
             response = yield self.get_json(uri, {
@@ -119,7 +120,7 @@ def query_user(self, service, user_id):
     def query_alias(self, service, alias):
         if service.url is None:
             defer.returnValue(False)
-        uri = service.url + ("/rooms/%s" % urllib.quote(alias))
+        uri = service.url + ("/rooms/%s" % urllib.parse.quote(alias))
         response = None
         try:
             response = yield self.get_json(uri, {
@@ -153,7 +154,7 @@ def query_3pe(self, service, kind, protocol, fields):
             service.url,
             APP_SERVICE_PREFIX,
             kind,
-            urllib.quote(protocol)
+            urllib.parse.quote(protocol)
         )
         try:
             response = yield self.get_json(uri, fields)
@@ -188,7 +189,7 @@ def _get():
             uri = "%s%s/thirdparty/protocol/%s" % (
                 service.url,
                 APP_SERVICE_PREFIX,
-                urllib.quote(protocol)
+                urllib.parse.quote(protocol)
             )
             try:
                 info = yield self.get_json(uri, {})
@@ -228,7 +229,7 @@ def push_bulk(self, service, events, txn_id=None):
         txn_id = str(txn_id)
 
         uri = service.url + ("/transactions/%s" %
-                             urllib.quote(txn_id))
+                             urllib.parse.quote(txn_id))
         try:
             yield self.put_json(
                 uri=uri,
diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index 3e0cd294a1b2..6e52c4b6b558 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -838,9 +838,9 @@ def on_edu(self, edu_type, origin, content):
             )
 
         return self._send_edu(
-                edu_type=edu_type,
-                origin=origin,
-                content=content,
+            edu_type=edu_type,
+            origin=origin,
+            content=content,
         )
 
     def on_query(self, query_type, args):
@@ -851,6 +851,6 @@ def on_query(self, query_type, args):
             return handler(args)
 
         return self._get_query_client(
-                query_type=query_type,
-                args=args,
+            query_type=query_type,
+            args=args,
         )
diff --git a/synapse/http/client.py b/synapse/http/client.py
index ab4fbf59b28d..4ba54fed05ef 100644
--- a/synapse/http/client.py
+++ b/synapse/http/client.py
@@ -13,24 +13,25 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 import logging
-import urllib
 
-from six import StringIO
+from six import text_type
+from six.moves import urllib
 
+import treq
 from canonicaljson import encode_canonical_json, json
 from prometheus_client import Counter
 
 from OpenSSL import SSL
 from OpenSSL.SSL import VERIFY_NONE
-from twisted.internet import defer, protocol, reactor, ssl, task
+from twisted.internet import defer, protocol, reactor, ssl
 from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS
 from twisted.web._newclient import ResponseDone
 from twisted.web.client import (
     Agent,
     BrowserLikeRedirectAgent,
     ContentDecoderAgent,
-    FileBodyProducer as TwistedFileBodyProducer,
     GzipDecoder,
     HTTPConnectionPool,
     PartialDownloadError,
@@ -83,18 +84,20 @@ def __init__(self, hs):
         if hs.config.user_agent_suffix:
             self.user_agent = "%s %s" % (self.user_agent, hs.config.user_agent_suffix,)
 
+        self.user_agent = self.user_agent.encode('ascii')
+
     @defer.inlineCallbacks
-    def request(self, method, uri, *args, **kwargs):
+    def request(self, method, uri, data=b'', headers=None):
         # A small wrapper around self.agent.request() so we can easily attach
         # counters to it
         outgoing_requests_counter.labels(method).inc()
 
         # log request but strip `access_token` (AS requests for example include this)
-        logger.info("Sending request %s %s", method, redact_uri(uri))
+        logger.info("Sending request %s %s", method, redact_uri(uri.encode('ascii')))
 
         try:
-            request_deferred = self.agent.request(
-                method, uri, *args, **kwargs
+            request_deferred = treq.request(
+                method, uri, agent=self.agent, data=data, headers=headers
             )
             add_timeout_to_deferred(
                 request_deferred, 60, self.hs.get_reactor(),
@@ -105,14 +108,14 @@ def request(self, method, uri, *args, **kwargs):
             incoming_responses_counter.labels(method, response.code).inc()
             logger.info(
                 "Received response to  %s %s: %s",
-                method, redact_uri(uri), response.code
+                method, redact_uri(uri.encode('ascii')), response.code
             )
             defer.returnValue(response)
         except Exception as e:
             incoming_responses_counter.labels(method, "ERR").inc()
             logger.info(
                 "Error sending request to  %s %s: %s %s",
-                method, redact_uri(uri), type(e).__name__, e.message
+                method, redact_uri(uri.encode('ascii')), type(e).__name__, e.args[0]
             )
             raise
 
@@ -137,7 +140,8 @@ def post_urlencoded_get_json(self, uri, args={}, headers=None):
         # TODO: Do we ever want to log message contents?
         logger.debug("post_urlencoded_get_json args: %s", args)
 
-        query_bytes = urllib.urlencode(encode_urlencode_args(args), True)
+        query_bytes = urllib.parse.urlencode(
+            encode_urlencode_args(args), True).encode("utf8")
 
         actual_headers = {
             b"Content-Type": [b"application/x-www-form-urlencoded"],
@@ -148,15 +152,14 @@ def post_urlencoded_get_json(self, uri, args={}, headers=None):
 
         response = yield self.request(
             "POST",
-            uri.encode("ascii"),
+            uri,
             headers=Headers(actual_headers),
-            bodyProducer=FileBodyProducer(StringIO(query_bytes))
+            data=query_bytes
         )
 
-        body = yield make_deferred_yieldable(readBody(response))
-
         if 200 <= response.code < 300:
-            defer.returnValue(json.loads(body))
+            body = yield make_deferred_yieldable(treq.json_content(response))
+            defer.returnValue(body)
         else:
             raise HttpResponseException(response.code, response.phrase, body)
 
@@ -191,9 +194,9 @@ def post_json_get_json(self, uri, post_json, headers=None):
 
         response = yield self.request(
             "POST",
-            uri.encode("ascii"),
+            uri,
             headers=Headers(actual_headers),
-            bodyProducer=FileBodyProducer(StringIO(json_str))
+            data=json_str
         )
 
         body = yield make_deferred_yieldable(readBody(response))
@@ -248,7 +251,7 @@ def put_json(self, uri, json_body, args={}, headers=None):
             ValueError: if the response was not JSON
         """
         if len(args):
-            query_bytes = urllib.urlencode(args, True)
+            query_bytes = urllib.parse.urlencode(args, True)
             uri = "%s?%s" % (uri, query_bytes)
 
         json_str = encode_canonical_json(json_body)
@@ -262,9 +265,9 @@ def put_json(self, uri, json_body, args={}, headers=None):
 
         response = yield self.request(
             "PUT",
-            uri.encode("ascii"),
+            uri,
             headers=Headers(actual_headers),
-            bodyProducer=FileBodyProducer(StringIO(json_str))
+            data=json_str
         )
 
         body = yield make_deferred_yieldable(readBody(response))
@@ -293,7 +296,7 @@ def get_raw(self, uri, args={}, headers=None):
             HttpResponseException on a non-2xx HTTP response.
         """
         if len(args):
-            query_bytes = urllib.urlencode(args, True)
+            query_bytes = urllib.parse.urlencode(args, True)
             uri = "%s?%s" % (uri, query_bytes)
 
         actual_headers = {
@@ -304,7 +307,7 @@ def get_raw(self, uri, args={}, headers=None):
 
         response = yield self.request(
             "GET",
-            uri.encode("ascii"),
+            uri,
             headers=Headers(actual_headers),
         )
 
@@ -339,7 +342,7 @@ def get_file(self, url, output_stream, max_size=None, headers=None):
 
         response = yield self.request(
             "GET",
-            url.encode("ascii"),
+            url,
             headers=Headers(actual_headers),
         )
 
@@ -434,12 +437,12 @@ class CaptchaServerHttpClient(SimpleHttpClient):
 
     @defer.inlineCallbacks
     def post_urlencoded_get_raw(self, url, args={}):
-        query_bytes = urllib.urlencode(encode_urlencode_args(args), True)
+        query_bytes = urllib.parse.urlencode(encode_urlencode_args(args), True)
 
         response = yield self.request(
             "POST",
-            url.encode("ascii"),
-            bodyProducer=FileBodyProducer(StringIO(query_bytes)),
+            url,
+            data=query_bytes,
             headers=Headers({
                 b"Content-Type": [b"application/x-www-form-urlencoded"],
                 b"User-Agent": [self.user_agent],
@@ -510,7 +513,7 @@ def encode_urlencode_args(args):
 
 
 def encode_urlencode_arg(arg):
-    if isinstance(arg, unicode):
+    if isinstance(arg, text_type):
         return arg.encode('utf-8')
     elif isinstance(arg, list):
         return [encode_urlencode_arg(i) for i in arg]
@@ -542,26 +545,3 @@ def getContext(self, hostname=None, port=None):
 
     def creatorForNetloc(self, hostname, port):
         return self
-
-
-class FileBodyProducer(TwistedFileBodyProducer):
-    """Workaround for https://twistedmatrix.com/trac/ticket/8473
-
-    We override the pauseProducing and resumeProducing methods in twisted's
-    FileBodyProducer so that they do not raise exceptions if the task has
-    already completed.
-    """
-
-    def pauseProducing(self):
-        try:
-            super(FileBodyProducer, self).pauseProducing()
-        except task.TaskDone:
-            # task has already completed
-            pass
-
-    def resumeProducing(self):
-        try:
-            super(FileBodyProducer, self).resumeProducing()
-        except task.NotPaused:
-            # task was not paused (probably because it had already completed)
-            pass
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index b34bb8e31ab4..6a1fc8ca553f 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -17,19 +17,19 @@
 import logging
 import random
 import sys
-import urllib
 
-from six import string_types
-from six.moves.urllib import parse as urlparse
+from six import PY3, string_types
+from six.moves import urllib
 
-from canonicaljson import encode_canonical_json, json
+import treq
+from canonicaljson import encode_canonical_json
 from prometheus_client import Counter
 from signedjson.sign import sign_json
 
 from twisted.internet import defer, protocol, reactor
 from twisted.internet.error import DNSLookupError
 from twisted.web._newclient import ResponseDone
-from twisted.web.client import Agent, HTTPConnectionPool, readBody
+from twisted.web.client import Agent, HTTPConnectionPool
 from twisted.web.http_headers import Headers
 
 import synapse.metrics
@@ -58,13 +58,18 @@
 MAX_LONG_RETRIES = 10
 MAX_SHORT_RETRIES = 3
 
+if PY3:
+    MAXINT = sys.maxsize
+else:
+    MAXINT = sys.maxint
+
 
 class MatrixFederationEndpointFactory(object):
     def __init__(self, hs):
         self.tls_client_options_factory = hs.tls_client_options_factory
 
     def endpointForURI(self, uri):
-        destination = uri.netloc
+        destination = uri.netloc.decode('ascii')
 
         return matrix_federation_endpoint(
             reactor, destination, timeout=10,
@@ -93,26 +98,32 @@ def __init__(self, hs):
         )
         self.clock = hs.get_clock()
         self._store = hs.get_datastore()
-        self.version_string = hs.version_string
+        self.version_string = hs.version_string.encode('ascii')
         self._next_id = 1
 
     def _create_url(self, destination, path_bytes, param_bytes, query_bytes):
-        return urlparse.urlunparse(
-            ("matrix", destination, path_bytes, param_bytes, query_bytes, "")
+        return urllib.parse.urlunparse(
+            (b"matrix", destination, path_bytes, param_bytes, query_bytes, b"")
         )
 
     @defer.inlineCallbacks
     def _request(self, destination, method, path,
-                 body_callback, headers_dict={}, param_bytes=b"",
-                 query_bytes=b"", retry_on_dns_fail=True,
+                 json=None, json_callback=None,
+                 param_bytes=b"",
+                 query=None, retry_on_dns_fail=True,
                  timeout=None, long_retries=False,
                  ignore_backoff=False,
                  backoff_on_404=False):
-        """ Creates and sends a request to the given server
+        """
+        Creates and sends a request to the given server.
+
         Args:
             destination (str): The remote server to send the HTTP request to.
             method (str): HTTP method
             path (str): The HTTP path
+            json (dict or None): JSON to send in the body.
+            json_callback (func or None): A callback to generate the JSON.
+            query (dict or None): Query arguments.
             ignore_backoff (bool): true to ignore the historical backoff data
                 and try the request anyway.
             backoff_on_404 (bool): Back off if we get a 404
@@ -146,22 +157,29 @@ def _request(self, destination, method, path,
             ignore_backoff=ignore_backoff,
         )
 
-        destination = destination.encode("ascii")
+        headers_dict = {}
         path_bytes = path.encode("ascii")
-        with limiter:
-            headers_dict[b"User-Agent"] = [self.version_string]
-            headers_dict[b"Host"] = [destination]
+        if query:
+            query_bytes = encode_query_args(query)
+        else:
+            query_bytes = b""
 
-            url_bytes = self._create_url(
-                destination, path_bytes, param_bytes, query_bytes
-            )
+        headers_dict = {
+            "User-Agent": [self.version_string],
+            "Host": [destination],
+        }
+
+        with limiter:
+            url = self._create_url(
+                destination.encode("ascii"), path_bytes, param_bytes, query_bytes
+            ).decode('ascii')
 
             txn_id = "%s-O-%s" % (method, self._next_id)
-            self._next_id = (self._next_id + 1) % (sys.maxint - 1)
+            self._next_id = (self._next_id + 1) % (MAXINT - 1)
 
             outbound_logger.info(
                 "{%s} [%s] Sending request: %s %s",
-                txn_id, destination, method, url_bytes
+                txn_id, destination, method, url
             )
 
             # XXX: Would be much nicer to retry only at the transaction-layer
@@ -171,23 +189,33 @@ def _request(self, destination, method, path,
             else:
                 retries_left = MAX_SHORT_RETRIES
 
-            http_url_bytes = urlparse.urlunparse(
-                ("", "", path_bytes, param_bytes, query_bytes, "")
-            )
+            http_url = urllib.parse.urlunparse(
+                (b"", b"", path_bytes, param_bytes, query_bytes, b"")
+            ).decode('ascii')
 
             log_result = None
             try:
                 while True:
-                    producer = None
-                    if body_callback:
-                        producer = body_callback(method, http_url_bytes, headers_dict)
-
                     try:
-                        request_deferred = self.agent.request(
+                        if json_callback:
+                            json = json_callback()
+
+                        if json:
+                            data = encode_canonical_json(json)
+                            headers_dict["Content-Type"] = ["application/json"]
+                            self.sign_request(
+                                destination, method, http_url, headers_dict, json
+                            )
+                        else:
+                            data = None
+                            self.sign_request(destination, method, http_url, headers_dict)
+
+                        request_deferred = treq.request(
                             method,
-                            url_bytes,
-                            Headers(headers_dict),
-                            producer
+                            url,
+                            headers=Headers(headers_dict),
+                            data=data,
+                            agent=self.agent,
                         )
                         add_timeout_to_deferred(
                             request_deferred,
@@ -218,7 +246,7 @@ def _request(self, destination, method, path,
                             txn_id,
                             destination,
                             method,
-                            url_bytes,
+                            url,
                             _flatten_response_never_received(e),
                         )
 
@@ -252,7 +280,7 @@ def _request(self, destination, method, path,
                 # :'(
                 # Update transactions table?
                 with logcontext.PreserveLoggingContext():
-                    body = yield readBody(response)
+                    body = yield treq.content(response)
                 raise HttpResponseException(
                     response.code, response.phrase, body
                 )
@@ -297,11 +325,11 @@ def sign_request(self, destination, method, url_bytes, headers_dict,
         auth_headers = []
 
         for key, sig in request["signatures"][self.server_name].items():
-            auth_headers.append(bytes(
+            auth_headers.append((
                 "X-Matrix origin=%s,key=\"%s\",sig=\"%s\"" % (
                     self.server_name, key, sig,
-                )
-            ))
+                )).encode('ascii')
+            )
 
         headers_dict[b"Authorization"] = auth_headers
 
@@ -347,24 +375,14 @@ def put_json(self, destination, path, args={}, data={},
         """
 
         if not json_data_callback:
-            def json_data_callback():
-                return data
-
-        def body_callback(method, url_bytes, headers_dict):
-            json_data = json_data_callback()
-            self.sign_request(
-                destination, method, url_bytes, headers_dict, json_data
-            )
-            producer = _JsonProducer(json_data)
-            return producer
+            json_data_callback = lambda: data
 
         response = yield self._request(
             destination,
             "PUT",
             path,
-            body_callback=body_callback,
-            headers_dict={"Content-Type": ["application/json"]},
-            query_bytes=encode_query_args(args),
+            json_callback=json_data_callback,
+            query=args,
             long_retries=long_retries,
             timeout=timeout,
             ignore_backoff=ignore_backoff,
@@ -376,8 +394,8 @@ def body_callback(method, url_bytes, headers_dict):
             check_content_type_is_json(response.headers)
 
         with logcontext.PreserveLoggingContext():
-            body = yield readBody(response)
-        defer.returnValue(json.loads(body))
+            body = yield treq.json_content(response)
+        defer.returnValue(body)
 
     @defer.inlineCallbacks
     def post_json(self, destination, path, data={}, long_retries=False,
@@ -410,20 +428,12 @@ def post_json(self, destination, path, data={}, long_retries=False,
             Fails with ``FederationDeniedError`` if this destination
             is not on our federation whitelist
         """
-
-        def body_callback(method, url_bytes, headers_dict):
-            self.sign_request(
-                destination, method, url_bytes, headers_dict, data
-            )
-            return _JsonProducer(data)
-
         response = yield self._request(
             destination,
             "POST",
             path,
-            query_bytes=encode_query_args(args),
-            body_callback=body_callback,
-            headers_dict={"Content-Type": ["application/json"]},
+            query=args,
+            json=data,
             long_retries=long_retries,
             timeout=timeout,
             ignore_backoff=ignore_backoff,
@@ -434,9 +444,9 @@ def body_callback(method, url_bytes, headers_dict):
             check_content_type_is_json(response.headers)
 
         with logcontext.PreserveLoggingContext():
-            body = yield readBody(response)
+            body = yield treq.json_content(response)
 
-        defer.returnValue(json.loads(body))
+        defer.returnValue(body)
 
     @defer.inlineCallbacks
     def get_json(self, destination, path, args=None, retry_on_dns_fail=True,
@@ -471,16 +481,11 @@ def get_json(self, destination, path, args=None, retry_on_dns_fail=True,
 
         logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail)
 
-        def body_callback(method, url_bytes, headers_dict):
-            self.sign_request(destination, method, url_bytes, headers_dict)
-            return None
-
         response = yield self._request(
             destination,
             "GET",
             path,
-            query_bytes=encode_query_args(args),
-            body_callback=body_callback,
+            query=args,
             retry_on_dns_fail=retry_on_dns_fail,
             timeout=timeout,
             ignore_backoff=ignore_backoff,
@@ -491,9 +496,9 @@ def body_callback(method, url_bytes, headers_dict):
             check_content_type_is_json(response.headers)
 
         with logcontext.PreserveLoggingContext():
-            body = yield readBody(response)
+            body = yield treq.json_content(response)
 
-        defer.returnValue(json.loads(body))
+        defer.returnValue(body)
 
     @defer.inlineCallbacks
     def delete_json(self, destination, path, long_retries=False,
@@ -523,13 +528,11 @@ def delete_json(self, destination, path, long_retries=False,
             Fails with ``FederationDeniedError`` if this destination
             is not on our federation whitelist
         """
-
         response = yield self._request(
             destination,
             "DELETE",
             path,
-            query_bytes=encode_query_args(args),
-            headers_dict={"Content-Type": ["application/json"]},
+            query=args,
             long_retries=long_retries,
             timeout=timeout,
             ignore_backoff=ignore_backoff,
@@ -540,9 +543,9 @@ def delete_json(self, destination, path, long_retries=False,
             check_content_type_is_json(response.headers)
 
         with logcontext.PreserveLoggingContext():
-            body = yield readBody(response)
+            body = yield treq.json_content(response)
 
-        defer.returnValue(json.loads(body))
+        defer.returnValue(body)
 
     @defer.inlineCallbacks
     def get_file(self, destination, path, output_stream, args={},
@@ -569,26 +572,11 @@ def get_file(self, destination, path, output_stream, args={},
             Fails with ``FederationDeniedError`` if this destination
             is not on our federation whitelist
         """
-
-        encoded_args = {}
-        for k, vs in args.items():
-            if isinstance(vs, string_types):
-                vs = [vs]
-            encoded_args[k] = [v.encode("UTF-8") for v in vs]
-
-        query_bytes = urllib.urlencode(encoded_args, True)
-        logger.debug("Query bytes: %s Retry DNS: %s", query_bytes, retry_on_dns_fail)
-
-        def body_callback(method, url_bytes, headers_dict):
-            self.sign_request(destination, method, url_bytes, headers_dict)
-            return None
-
         response = yield self._request(
             destination,
             "GET",
             path,
-            query_bytes=query_bytes,
-            body_callback=body_callback,
+            query=args,
             retry_on_dns_fail=retry_on_dns_fail,
             ignore_backoff=ignore_backoff,
         )
@@ -639,30 +627,6 @@ def _readBodyToFile(response, stream, max_size):
     return d
 
 
-class _JsonProducer(object):
-    """ Used by the twisted http client to create the HTTP body from json
-    """
-    def __init__(self, jsn):
-        self.reset(jsn)
-
-    def reset(self, jsn):
-        self.body = encode_canonical_json(jsn)
-        self.length = len(self.body)
-
-    def startProducing(self, consumer):
-        consumer.write(self.body)
-        return defer.succeed(None)
-
-    def pauseProducing(self):
-        pass
-
-    def stopProducing(self):
-        pass
-
-    def resumeProducing(self):
-        pass
-
-
 def _flatten_response_never_received(e):
     if hasattr(e, "reasons"):
         reasons = ", ".join(
@@ -693,7 +657,7 @@ def check_content_type_is_json(headers):
             "No Content-Type header"
         )
 
-    c_type = c_type[0]  # only the first header
+    c_type = c_type[0].decode('ascii')  # only the first header
     val, options = cgi.parse_header(c_type)
     if val != "application/json":
         raise RuntimeError(
@@ -711,6 +675,6 @@ def encode_query_args(args):
             vs = [vs]
         encoded_args[k] = [v.encode("UTF-8") for v in vs]
 
-    query_bytes = urllib.urlencode(encoded_args, True)
+    query_bytes = urllib.parse.urlencode(encoded_args, True)
 
-    return query_bytes
+    return query_bytes.encode('utf8')
diff --git a/synapse/http/site.py b/synapse/http/site.py
index 88ed3714f9be..f0828c654243 100644
--- a/synapse/http/site.py
+++ b/synapse/http/site.py
@@ -204,14 +204,14 @@ def _started_processing(self, servlet_name):
         self.start_time = time.time()
         self.request_metrics = RequestMetrics()
         self.request_metrics.start(
-            self.start_time, name=servlet_name, method=self.method,
+            self.start_time, name=servlet_name, method=self.method.decode('ascii'),
         )
 
         self.site.access_logger.info(
             "%s - %s - Received request: %s %s",
             self.getClientIP(),
             self.site.site_tag,
-            self.method,
+            self.method.decode('ascii'),
             self.get_redacted_uri()
         )
 
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index 942d7c721f0e..6dd517932078 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -40,6 +40,7 @@
     "pynacl>=1.2.1": ["nacl>=1.2.1", "nacl.bindings"],
     "service_identity>=1.0.0": ["service_identity>=1.0.0"],
     "Twisted>=17.1.0": ["twisted>=17.1.0"],
+    "treq>=15.1": ["treq>=15.1"],
 
     # We use crypto.get_elliptic_curve which is only supported in >=0.15
     "pyopenssl>=0.15": ["OpenSSL>=0.15"],

From 7419764351aa2b2f00f363beaee6ec53c02c492e Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 5 Sep 2018 16:19:50 +0100
Subject: [PATCH 551/830] User iter* during sync state calculations

---
 synapse/handlers/sync.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index ef20c2296c18..0965cf2fc550 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -1729,17 +1729,17 @@ def _calculate_state(
     event_id_to_key = {
         e: key
         for key, e in itertools.chain(
-            timeline_contains.items(),
-            previous.items(),
-            timeline_start.items(),
-            current.items(),
+            iteritems(timeline_contains),
+            iteritems(previous),
+            iteritems(timeline_start),
+            iteritems(current),
         )
     }
 
-    c_ids = set(e for e in current.values())
-    ts_ids = set(e for e in timeline_start.values())
-    p_ids = set(e for e in previous.values())
-    tc_ids = set(e for e in timeline_contains.values())
+    c_ids = set(e for e in itervalues(current))
+    ts_ids = set(e for e in itervalues(timeline_start))
+    p_ids = set(e for e in itervalues(previous))
+    tc_ids = set(e for e in itervalues(timeline_contains))
 
     # If we are lazyloading room members, we explicitly add the membership events
     # for the senders in the timeline into the state block returned by /sync,

From 2254790ae458160117959b364378f1c6cf338005 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 5 Sep 2018 16:21:18 +0100
Subject: [PATCH 552/830] Newsfile

---
 changelog.d/3795.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3795.misc

diff --git a/changelog.d/3795.misc b/changelog.d/3795.misc
new file mode 100644
index 000000000000..9f64ee5e2bb2
--- /dev/null
+++ b/changelog.d/3795.misc
@@ -0,0 +1 @@
+Make /sync slightly faster by avoiding needless copies

From 61b05727fa6ba879f507d13ff40c0197916ef795 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 5 Sep 2018 22:30:36 +0100
Subject: [PATCH 553/830] guest users should not be part of mau total

---
 synapse/storage/monthly_active_users.py    |  7 +++-
 tests/storage/test_monthly_active_users.py | 44 +++++++++++++++++++++-
 2 files changed, 48 insertions(+), 3 deletions(-)

diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py
index c7899d7fd269..e34e16c07979 100644
--- a/synapse/storage/monthly_active_users.py
+++ b/synapse/storage/monthly_active_users.py
@@ -192,14 +192,19 @@ def user_last_seen_monthly_active(self, user_id):
         ))
 
     @defer.inlineCallbacks
-    def populate_monthly_active_users(self, user_id):
+    def populate_monthly_active_users(self, user_id, is_guest=False):
         """Checks on the state of monthly active user limits and optionally
         add the user to the monthly active tables
 
         Args:
             user_id(str): the user_id to query
         """
+
         if self.hs.config.limit_usage_by_mau:
+            # Guests should not be included as part of MAU group
+            if is_guest:
+                return
+
             is_trial = yield self.is_trial_user(user_id)
             if is_trial:
                 # we don't track trial users in the MAU table.
diff --git a/tests/storage/test_monthly_active_users.py b/tests/storage/test_monthly_active_users.py
index 2036287288ac..cfe0a7c77d25 100644
--- a/tests/storage/test_monthly_active_users.py
+++ b/tests/storage/test_monthly_active_users.py
@@ -12,6 +12,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from mock import Mock
+
+from twisted.internet import defer
 
 from tests.unittest import HomeserverTestCase
 
@@ -23,7 +26,8 @@ def make_homeserver(self, reactor, clock):
 
         hs = self.setup_test_homeserver()
         self.store = hs.get_datastore()
-
+        hs.config.limit_usage_by_mau = True
+        hs.config.max_mau_value = 50
         # Advance the clock a bit
         reactor.advance(FORTY_DAYS)
 
@@ -73,7 +77,7 @@ def test_initialise_reserved_users(self):
         active_count = self.store.get_monthly_active_count()
         self.assertEquals(self.get_success(active_count), user_num)
 
-        # Test that regalar users are removed from the db
+        # Test that regular users are removed from the db
         ru_count = 2
         self.store.upsert_monthly_active_user("@ru1:server")
         self.store.upsert_monthly_active_user("@ru2:server")
@@ -139,3 +143,39 @@ def test_reap_monthly_active_users(self):
 
         count = self.store.get_monthly_active_count()
         self.assertEquals(self.get_success(count), 0)
+
+    def test_populate_monthly_users_is_guest(self):
+        # Test that guest users are not added to mau list
+        self.store.upsert_monthly_active_user = Mock()
+        self.store.populate_monthly_active_users('user_id', True)
+        self.pump()
+        self.store.upsert_monthly_active_user.assert_not_called()
+
+    def test_populate_monthly_users_should_update(self):
+        self.store.upsert_monthly_active_user = Mock()
+
+        self.store.is_trial_user = Mock(
+            return_value=defer.succeed(False)
+        )
+
+        self.store.user_last_seen_monthly_active = Mock(
+            return_value=defer.succeed(None)
+        )
+        self.store.populate_monthly_active_users('user_id')
+        self.pump()
+        self.store.upsert_monthly_active_user.assert_called_once()
+
+    def test_populate_monthly_users_should_not_update(self):
+        self.store.upsert_monthly_active_user = Mock()
+
+        self.store.is_trial_user = Mock(
+            return_value=defer.succeed(False)
+        )
+        self.store.user_last_seen_monthly_active = Mock(
+            return_value=defer.succeed(
+                self.hs.get_clock().time_msec()
+            )
+        )
+        self.store.populate_monthly_active_users('user_id')
+        self.pump()
+        self.store.upsert_monthly_active_user.assert_not_called()

From 92657be7d0e108f11f5390222df9efc9f1e55908 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Wed, 5 Sep 2018 22:33:45 +0100
Subject: [PATCH 554/830] towncrier

---
 changelog.d/3800.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3800.bugfix

diff --git a/changelog.d/3800.bugfix b/changelog.d/3800.bugfix
new file mode 100644
index 000000000000..6b2e18b4a6e4
--- /dev/null
+++ b/changelog.d/3800.bugfix
@@ -0,0 +1 @@
+guest users should not be part of mau total

From 9c8cd855da4c32dc9219655d13cb86d6f9933381 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Christian=20Gr=C3=BCnhage?=
 <jan.christian@gruenhage.xyz>
Date: Thu, 6 Sep 2018 10:39:55 +0200
Subject: [PATCH 555/830] remove synctl from .dockerignore

---
 .dockerignore | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.dockerignore b/.dockerignore
index 6cdb8532d332..0180602e56fc 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -3,6 +3,5 @@ Dockerfile
 .gitignore
 demo/etc
 tox.ini
-synctl
 .git/*
 .tox/*

From af3125226d04f5dc9f74168ae2654a062537f075 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Christian=20Gr=C3=BCnhage?=
 <jan.christian@gruenhage.xyz>
Date: Thu, 6 Sep 2018 10:46:00 +0200
Subject: [PATCH 556/830] Create 3802.misc

---
 changelog.d/3802.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3802.misc

diff --git a/changelog.d/3802.misc b/changelog.d/3802.misc
new file mode 100644
index 000000000000..a00eccb28329
--- /dev/null
+++ b/changelog.d/3802.misc
@@ -0,0 +1 @@
+Unignore synctl in .dockerignore to fix docker builds

From 78d1042c1047133e837ded583b5289096526544f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Christian=20Gr=C3=BCnhage?=
 <jan.christian@gruenhage.xyz>
Date: Thu, 6 Sep 2018 10:39:55 +0200
Subject: [PATCH 557/830] remove synctl from .dockerignore

---
 .dockerignore | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.dockerignore b/.dockerignore
index 6cdb8532d332..0180602e56fc 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -3,6 +3,5 @@ Dockerfile
 .gitignore
 demo/etc
 tox.ini
-synctl
 .git/*
 .tox/*

From 0cd7b209e2540b35e2471ac8abe3a8a4e6a74725 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Christian=20Gr=C3=BCnhage?=
 <jan.christian@gruenhage.xyz>
Date: Thu, 6 Sep 2018 10:46:00 +0200
Subject: [PATCH 558/830] Create 3802.misc

---
 changelog.d/3802.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3802.misc

diff --git a/changelog.d/3802.misc b/changelog.d/3802.misc
new file mode 100644
index 000000000000..a00eccb28329
--- /dev/null
+++ b/changelog.d/3802.misc
@@ -0,0 +1 @@
+Unignore synctl in .dockerignore to fix docker builds

From 80189ed27c5c1b390cc9783b2931e71cccdb3c74 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 5 Sep 2018 17:03:49 +0100
Subject: [PATCH 559/830] prepare v0.33.3.1

---
 CHANGES.md            | 14 ++++++++++++++
 changelog.d/3802.misc |  1 -
 synapse/__init__.py   |  2 +-
 3 files changed, 15 insertions(+), 2 deletions(-)
 delete mode 100644 changelog.d/3802.misc

diff --git a/CHANGES.md b/CHANGES.md
index a35f5aebc795..59feb6b1a7d9 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,17 @@
+Synapse 0.33.3.1 (2018-09-06)
+=============================
+
+SECURITY FIXES
+--------------
+
+- Fix an issue where event signatures were not always correctly validated ([\#3796](https://github.com/matrix-org/synapse/issues/3796))
+- Fix an issue where server_acls could be circumvented for incoming events ([\#3796](https://github.com/matrix-org/synapse/issues/3796))
+
+Internal Changes
+----------------
+
+- Unignore synctl in .dockerignore to fix docker builds ([\#3802](https://github.com/matrix-org/synapse/issues/3802))
+
 Synapse 0.33.3 (2018-08-22)
 ===========================
 
diff --git a/changelog.d/3802.misc b/changelog.d/3802.misc
deleted file mode 100644
index a00eccb28329..000000000000
--- a/changelog.d/3802.misc
+++ /dev/null
@@ -1 +0,0 @@
-Unignore synctl in .dockerignore to fix docker builds
diff --git a/synapse/__init__.py b/synapse/__init__.py
index e62901b7614b..e395a0a9eeda 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -17,4 +17,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.33.3"
+__version__ = "0.33.3.1"

From 625542878de2049a7a0b8381a75a77d09c931a82 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 6 Sep 2018 12:53:15 +0100
Subject: [PATCH 560/830] bump dep on pyopenssl to 16.x

---
 synapse/python_dependencies.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index 6dd517932078..0d8de600cfa7 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -42,8 +42,8 @@
     "Twisted>=17.1.0": ["twisted>=17.1.0"],
     "treq>=15.1": ["treq>=15.1"],
 
-    # We use crypto.get_elliptic_curve which is only supported in >=0.15
-    "pyopenssl>=0.15": ["OpenSSL>=0.15"],
+    # Twisted has required pyopenssl 16.0 since about Twisted 16.6.
+    "pyopenssl>=16.0.0": ["OpenSSL>=16.0.0"],
 
     "pyyaml": ["yaml"],
     "pyasn1": ["pyasn1"],

From b3c2ebba3231255c9a2b4607e8344d7dcd3a5791 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 6 Sep 2018 12:55:17 +0100
Subject: [PATCH 561/830] changelog

---
 changelog.d/3804.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3804.bugfix

diff --git a/changelog.d/3804.bugfix b/changelog.d/3804.bugfix
new file mode 100644
index 000000000000..a0cef20e3fdf
--- /dev/null
+++ b/changelog.d/3804.bugfix
@@ -0,0 +1 @@
+Bump dependency on pyopenssl 16.x, to avoid incompatibility with recent Twisted.

From 417e7077aadb9f9832dabb0fa93e39e58df92a8a Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 6 Sep 2018 14:08:55 +0100
Subject: [PATCH 562/830] Bump version and changelog

---
 CHANGES.md          | 6 ++++++
 synapse/__init__.py | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/CHANGES.md b/CHANGES.md
index 4db47da3e387..fecfa7feeca4 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,9 @@
+Synapse 0.33.4rc2 (2018-09-06)
+==============================
+
+Pull in security fixes from v0.33.3.1
+
+
 Synapse 0.33.3.1 (2018-09-06)
 =============================
 
diff --git a/synapse/__init__.py b/synapse/__init__.py
index 2c42893673e1..063c30453400 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -17,4 +17,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.33.4rc1"
+__version__ = "0.33.4rc2"

From 2608ebc04c8b7ffb3417eeb720801d70ddc7dccd Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Fri, 7 Sep 2018 00:22:23 +1000
Subject: [PATCH 563/830] Port handlers/ to Python 3 (#3803)

---
 .gitignore                     |  1 +
 changelog.d/3803.misc          |  1 +
 synapse/handlers/auth.py       |  8 +++++---
 synapse/handlers/e2e_keys.py   |  5 +++--
 synapse/handlers/federation.py |  6 +++---
 synapse/handlers/room_list.py  |  2 +-
 synapse/handlers/search.py     | 14 +++++++-------
 synapse/handlers/sync.py       |  6 +++---
 8 files changed, 24 insertions(+), 19 deletions(-)
 create mode 100644 changelog.d/3803.misc

diff --git a/.gitignore b/.gitignore
index 9f42a7568f95..1718185384a4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -44,6 +44,7 @@ media_store/
 build/
 venv/
 venv*/
+*venv/
 
 localhost-800*/
 static/client/register/register_config.js
diff --git a/changelog.d/3803.misc b/changelog.d/3803.misc
new file mode 100644
index 000000000000..2b60653c298d
--- /dev/null
+++ b/changelog.d/3803.misc
@@ -0,0 +1 @@
+handlers/ is now ported to Python 3.
diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py
index 4a81bd2ba9c6..2a5eab124f72 100644
--- a/synapse/handlers/auth.py
+++ b/synapse/handlers/auth.py
@@ -895,22 +895,24 @@ def validate_hash(self, password, stored_hash):
 
         Args:
             password (unicode): Password to hash.
-            stored_hash (unicode): Expected hash value.
+            stored_hash (bytes): Expected hash value.
 
         Returns:
             Deferred(bool): Whether self.hash(password) == stored_hash.
         """
-
         def _do_validate_hash():
             # Normalise the Unicode in the password
             pw = unicodedata.normalize("NFKC", password)
 
             return bcrypt.checkpw(
                 pw.encode('utf8') + self.hs.config.password_pepper.encode("utf8"),
-                stored_hash.encode('utf8')
+                stored_hash
             )
 
         if stored_hash:
+            if not isinstance(stored_hash, bytes):
+                stored_hash = stored_hash.encode('ascii')
+
             return make_deferred_yieldable(
                 threads.deferToThreadPool(
                     self.hs.get_reactor(),
diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py
index 5816bf8b4fab..578e9250fba8 100644
--- a/synapse/handlers/e2e_keys.py
+++ b/synapse/handlers/e2e_keys.py
@@ -330,7 +330,8 @@ def _upload_one_time_keys_for_user(self, user_id, device_id, time_now,
                         (algorithm, key_id, ex_json, key)
                     )
             else:
-                new_keys.append((algorithm, key_id, encode_canonical_json(key)))
+                new_keys.append((
+                    algorithm, key_id, encode_canonical_json(key).decode('ascii')))
 
         yield self.store.add_e2e_one_time_keys(
             user_id, device_id, time_now, new_keys
@@ -358,7 +359,7 @@ def _exception_to_failure(e):
     # Note that some Exceptions (notably twisted's ResponseFailed etc) don't
     # give a string for e.message, which json then fails to serialize.
     return {
-        "status": 503, "message": str(e.message),
+        "status": 503, "message": str(e),
     }
 
 
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 3fa7a98445ce..0c68e8a472de 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -594,7 +594,7 @@ def backfill(self, dest, room_id, limit, extremities):
 
         required_auth = set(
             a_id
-            for event in events + state_events.values() + auth_events.values()
+            for event in events + list(state_events.values()) + list(auth_events.values())
             for a_id, _ in event.auth_events
         )
         auth_events.update({
@@ -802,7 +802,7 @@ def try_backfill(domains):
                     )
                     continue
                 except NotRetryingDestination as e:
-                    logger.info(e.message)
+                    logger.info(str(e))
                     continue
                 except FederationDeniedError as e:
                     logger.info(e)
@@ -1358,7 +1358,7 @@ def get_state_ids_for_pdu(self, room_id, event_id):
         )
 
         if state_groups:
-            _, state = state_groups.items().pop()
+            _, state = list(state_groups.items()).pop()
             results = state
 
             if event.is_state():
diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index 37e41afd61db..38e1737ec965 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -162,7 +162,7 @@ def get_order_for_room(room_id):
         # Filter out rooms that we don't want to return
         rooms_to_scan = [
             r for r in sorted_rooms
-            if r not in newly_unpublished and rooms_to_num_joined[room_id] > 0
+            if r not in newly_unpublished and rooms_to_num_joined[r] > 0
         ]
 
         total_room_count = len(rooms_to_scan)
diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py
index c464adbd0b9c..0c1d52fd1187 100644
--- a/synapse/handlers/search.py
+++ b/synapse/handlers/search.py
@@ -54,7 +54,7 @@ def search(self, user, content, batch=None):
         batch_token = None
         if batch:
             try:
-                b = decode_base64(batch)
+                b = decode_base64(batch).decode('ascii')
                 batch_group, batch_group_key, batch_token = b.split("\n")
 
                 assert batch_group is not None
@@ -258,18 +258,18 @@ def search(self, user, content, batch=None):
                 # it returns more from the same group (if applicable) rather
                 # than reverting to searching all results again.
                 if batch_group and batch_group_key:
-                    global_next_batch = encode_base64("%s\n%s\n%s" % (
+                    global_next_batch = encode_base64(("%s\n%s\n%s" % (
                         batch_group, batch_group_key, pagination_token
-                    ))
+                    )).encode('ascii'))
                 else:
-                    global_next_batch = encode_base64("%s\n%s\n%s" % (
+                    global_next_batch = encode_base64(("%s\n%s\n%s" % (
                         "all", "", pagination_token
-                    ))
+                    )).encode('ascii'))
 
                 for room_id, group in room_groups.items():
-                    group["next_batch"] = encode_base64("%s\n%s\n%s" % (
+                    group["next_batch"] = encode_base64(("%s\n%s\n%s" % (
                         "room_id", room_id, pagination_token
-                    ))
+                    )).encode('ascii'))
 
             allowed_events.extend(room_events)
 
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index ef20c2296c18..0091ceb80e4d 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -545,7 +545,7 @@ def compute_summary(self, room_id, sync_config, batch, state, now_token):
 
         member_ids = {
             state_key: event_id
-            for (t, state_key), event_id in state_ids.iteritems()
+            for (t, state_key), event_id in iteritems(state_ids)
             if t == EventTypes.Member
         }
         name_id = state_ids.get((EventTypes.Name, ''))
@@ -774,7 +774,7 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                     logger.debug("filtering state from %r...", state_ids)
                     state_ids = {
                         t: event_id
-                        for t, event_id in state_ids.iteritems()
+                        for t, event_id in iteritems(state_ids)
                         if cache.get(t[1]) != event_id
                     }
                     logger.debug("...to %r", state_ids)
@@ -1753,7 +1753,7 @@ def _calculate_state(
 
     if lazy_load_members:
         p_ids.difference_update(
-            e for t, e in timeline_start.iteritems()
+            e for t, e in iteritems(timeline_start)
             if t[0] == EventTypes.Member
         )
 

From 6707a3212cea6849d1c16b70cc1927b16d06814a Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 6 Sep 2018 15:23:55 +0100
Subject: [PATCH 564/830] Limit the number of PDUs/EDUs per fedreation
 transaction

---
 synapse/federation/transaction_queue.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py
index 94d7423d016a..8cbf8c4f7fbf 100644
--- a/synapse/federation/transaction_queue.py
+++ b/synapse/federation/transaction_queue.py
@@ -463,7 +463,19 @@ def _transaction_transmission_loop(self, destination):
                 # pending_transactions flag.
 
                 pending_pdus = self.pending_pdus_by_dest.pop(destination, [])
+
+                # We can only include at most 50 PDUs per transactions
+                pending_pdus, leftover_pdus = pending_pdus[:50], pending_pdus[50:]
+                if leftover_pdus:
+                    self.pending_pdus_by_dest[destination] = leftover_pdus
+
                 pending_edus = self.pending_edus_by_dest.pop(destination, [])
+
+                # We can only include at most 100 EDUs per transactions
+                pending_edus, leftover_edus = pending_edus[:100], pending_edus[100:]
+                if leftover_edus:
+                    self.pending_edus_by_dest[destination] = leftover_edus
+
                 pending_presence = self.pending_presence_by_dest.pop(destination, {})
 
                 pending_edus.extend(

From 28f5bfdcf7d806c958c850f8ac733557d485ddf6 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 6 Sep 2018 15:25:58 +0100
Subject: [PATCH 565/830] Newsfile

---
 changelog.d/3805.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3805.misc

diff --git a/changelog.d/3805.misc b/changelog.d/3805.misc
new file mode 100644
index 000000000000..8173f6da3166
--- /dev/null
+++ b/changelog.d/3805.misc
@@ -0,0 +1 @@
+Limit the number of PDUs/EDUs per fedreation transaction

From 3b4223aa234ec9f87d29a747af3e0b723d1183b1 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 6 Sep 2018 15:27:54 +0100
Subject: [PATCH 566/830] Only start postgres instance for postgres tests on
 Travis CI

---
 .travis.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 11c76db2e5a4..ebc972ed2492 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,9 +8,6 @@ before_script:
   - git remote set-branches --add origin develop
   - git fetch origin develop
 
-services:
-  - postgresql
-
 matrix:
   fast_finish: true
   include:
@@ -25,6 +22,8 @@ matrix:
 
   - python: 2.7
     env: TOX_ENV=py27-postgres TRIAL_FLAGS="-j 4"
+    services:
+      - postgresql
 
   - python: 3.6
     env: TOX_ENV=py36

From 5d848992bf986df3d7f1edd1be1a3680a9d19fb2 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 6 Sep 2018 15:28:46 +0100
Subject: [PATCH 567/830] Newsfile

---
 changelog.d/3806.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3806.misc

diff --git a/changelog.d/3806.misc b/changelog.d/3806.misc
new file mode 100644
index 000000000000..3c722eef2dc7
--- /dev/null
+++ b/changelog.d/3806.misc
@@ -0,0 +1 @@
+Only start postgres instance for postgres tests on Travis CI

From b07a2cbee959d84b0f9404c4e54fda21dc0cc4e4 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 6 Sep 2018 15:53:02 +0100
Subject: [PATCH 568/830] Spelling

---
 changelog.d/3805.misc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/changelog.d/3805.misc b/changelog.d/3805.misc
index 8173f6da3166..257feeb0718f 100644
--- a/changelog.d/3805.misc
+++ b/changelog.d/3805.misc
@@ -1 +1 @@
-Limit the number of PDUs/EDUs per fedreation transaction
+Limit the number of PDUs/EDUs per federation transaction

From 3801b8aa035594972c400c8bd036894a388c4ab3 Mon Sep 17 00:00:00 2001
From: Hubert Chathi <hubert@uhoreg.ca>
Date: Thu, 6 Sep 2018 11:23:16 -0400
Subject: [PATCH 569/830] try to make flake8 and isort happy

---
 synapse/api/errors.py                     | 1 +
 synapse/handlers/e2e_room_keys.py         | 2 +-
 synapse/rest/client/v2_alpha/room_keys.py | 5 ++++-
 synapse/storage/e2e_room_keys.py          | 6 ++++--
 tests/handlers/test_e2e_room_keys.py      | 9 +++++----
 5 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index 3002c95dd1cd..140dbfe8b864 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -326,6 +326,7 @@ def __init__(self, current_version):
         )
         self.current_version = current_version
 
+
 class IncompatibleRoomVersionError(SynapseError):
     """A server is trying to join a room whose version it does not support."""
 
diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py
index c09816b37274..2c330382cfc6 100644
--- a/synapse/handlers/e2e_room_keys.py
+++ b/synapse/handlers/e2e_room_keys.py
@@ -17,7 +17,7 @@
 
 from twisted.internet import defer
 
-from synapse.api.errors import StoreError, SynapseError, RoomKeysVersionError
+from synapse.api.errors import RoomKeysVersionError, StoreError, SynapseError
 from synapse.util.async_helpers import Linearizer
 
 logger = logging.getLogger(__name__)
diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py
index 9f0172e6f5ee..1ed18e986fa4 100644
--- a/synapse/rest/client/v2_alpha/room_keys.py
+++ b/synapse/rest/client/v2_alpha/room_keys.py
@@ -19,8 +19,11 @@
 
 from synapse.api.errors import SynapseError
 from synapse.http.servlet import (
-    RestServlet, parse_json_object_from_request, parse_string
+    RestServlet,
+    parse_json_object_from_request,
+    parse_string,
 )
+
 from ._base import client_v2_patterns
 
 logger = logging.getLogger(__name__)
diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py
index b695570a7b70..969f4aef9ca3 100644
--- a/synapse/storage/e2e_room_keys.py
+++ b/synapse/storage/e2e_room_keys.py
@@ -13,9 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import simplejson as json
+
 from twisted.internet import defer
+
 from synapse.api.errors import StoreError
-import simplejson as json
 
 from ._base import SQLBaseStore
 
@@ -58,7 +60,7 @@ def get_e2e_room_key(self, user_id, version, room_id, session_id):
             desc="get_e2e_room_key",
         )
 
-        row["session_data"] = json.loads(row["session_data"]);
+        row["session_data"] = json.loads(row["session_data"])
 
         defer.returnValue(row)
 
diff --git a/tests/handlers/test_e2e_room_keys.py b/tests/handlers/test_e2e_room_keys.py
index 7fa426444152..9e08eac0a5bf 100644
--- a/tests/handlers/test_e2e_room_keys.py
+++ b/tests/handlers/test_e2e_room_keys.py
@@ -14,17 +14,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import copy
+
 import mock
-from synapse.api import errors
+
 from twisted.internet import defer
-import copy
 
 import synapse.api.errors
 import synapse.handlers.e2e_room_keys
-
 import synapse.storage
-from tests import unittest, utils
+from synapse.api import errors
 
+from tests import unittest, utils
 
 # sample room_key data for use in the tests
 room_keys = {

From 7baf66ef5d1575b40b638d4f21204e1646792612 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 6 Sep 2018 16:46:51 +0100
Subject: [PATCH 570/830] Send existing room tags down sync on join

When a user joined a room any existing tags were not sent down the sync
stream. Ordinarily this isn't a problem because the user needs to be in
the room to have set tags in it, however synapse will sometimes add tags
for a user to a room, e.g. for server notices, which need to come down
sync.
---
 synapse/handlers/sync.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 9f133ded3f84..150d492cb021 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -1575,6 +1575,14 @@ def _generate_room_entry(self, sync_result_builder, ignored_users,
             newly_joined_room=newly_joined,
         )
 
+        # When we join the room (or the client requests full_state), we should
+        # send down any existing tags. Usually the user won't have tags in a
+        # newly joined room, unless either a) they've joined before or b) the
+        # tag was added by synapse e.g. for server notice rooms.
+        if full_state:
+            user_id = sync_result_builder.sync_config.user.to_string()
+            tags = yield self.store.get_tags_for_room(user_id, room_id)
+
         account_data_events = []
         if tags is not None:
             account_data_events.append({

From f60c9e2a01598c1900ab785096aa379d1fb6ac5b Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 6 Sep 2018 17:01:41 +0100
Subject: [PATCH 571/830] Don't send empty tags list down sync

---
 synapse/handlers/sync.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 150d492cb021..7eed2fcc9bb4 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -1583,6 +1583,11 @@ def _generate_room_entry(self, sync_result_builder, ignored_users,
             user_id = sync_result_builder.sync_config.user.to_string()
             tags = yield self.store.get_tags_for_room(user_id, room_id)
 
+            # If there aren't any tags, don't send the empty tags list down
+            # sync
+            if not tags:
+                tags = None
+
         account_data_events = []
         if tags is not None:
             account_data_events.append({

From 7298efd3619110adbf35bd1bc048d8c9a3a05949 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 6 Sep 2018 17:13:33 +0100
Subject: [PATCH 572/830] Newsfile

---
 changelog.d/3810.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3810.bugfix

diff --git a/changelog.d/3810.bugfix b/changelog.d/3810.bugfix
new file mode 100644
index 000000000000..2b938a81aef0
--- /dev/null
+++ b/changelog.d/3810.bugfix
@@ -0,0 +1 @@
+Fix existing room tags not coming down sync when joining a room

From 84a750e0c36f653b71a06c564154f257c1b7dee3 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Thu, 6 Sep 2018 17:22:53 +0100
Subject: [PATCH 573/830] ensure guests never enter mau list

---
 synapse/storage/monthly_active_users.py    | 7 +++----
 tests/storage/test_client_ips.py           | 4 +---
 tests/storage/test_monthly_active_users.py | 6 +++++-
 tests/utils.py                             | 1 +
 4 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py
index e34e16c07979..b890c152dbec 100644
--- a/synapse/storage/monthly_active_users.py
+++ b/synapse/storage/monthly_active_users.py
@@ -192,7 +192,7 @@ def user_last_seen_monthly_active(self, user_id):
         ))
 
     @defer.inlineCallbacks
-    def populate_monthly_active_users(self, user_id, is_guest=False):
+    def populate_monthly_active_users(self, user_id):
         """Checks on the state of monthly active user limits and optionally
         add the user to the monthly active tables
 
@@ -201,13 +201,12 @@ def populate_monthly_active_users(self, user_id, is_guest=False):
         """
 
         if self.hs.config.limit_usage_by_mau:
-            # Guests should not be included as part of MAU group
+            # Trial users and guests should not be included as part of MAU group
+            is_guest = yield self.is_guest(user_id)
             if is_guest:
                 return
-
             is_trial = yield self.is_trial_user(user_id)
             if is_trial:
-                # we don't track trial users in the MAU table.
                 return
 
             last_seen_timestamp = yield self.user_last_seen_monthly_active(user_id)
diff --git a/tests/storage/test_client_ips.py b/tests/storage/test_client_ips.py
index c2e88bdbafeb..c9b02a062be5 100644
--- a/tests/storage/test_client_ips.py
+++ b/tests/storage/test_client_ips.py
@@ -101,13 +101,11 @@ def test_updating_monthly_active_user_when_space(self):
         self.hs.config.limit_usage_by_mau = True
         self.hs.config.max_mau_value = 50
         user_id = "@user:server"
+        yield self.store.register(user_id=user_id, token="123", password_hash=None)
 
         active = yield self.store.user_last_seen_monthly_active(user_id)
         self.assertFalse(active)
 
-        yield self.store.insert_client_ip(
-            user_id, "access_token", "ip", "user_agent", "device_id"
-        )
         yield self.store.insert_client_ip(
             user_id, "access_token", "ip", "user_agent", "device_id"
         )
diff --git a/tests/storage/test_monthly_active_users.py b/tests/storage/test_monthly_active_users.py
index cfe0a7c77d25..ccfc21b7fcdf 100644
--- a/tests/storage/test_monthly_active_users.py
+++ b/tests/storage/test_monthly_active_users.py
@@ -146,8 +146,12 @@ def test_reap_monthly_active_users(self):
 
     def test_populate_monthly_users_is_guest(self):
         # Test that guest users are not added to mau list
+        user_id = "user_id"
+        self.store.register(
+            user_id=user_id, token="123", password_hash=None, make_guest=True
+        )
         self.store.upsert_monthly_active_user = Mock()
-        self.store.populate_monthly_active_users('user_id', True)
+        self.store.populate_monthly_active_users(user_id)
         self.pump()
         self.store.upsert_monthly_active_user.assert_not_called()
 
diff --git a/tests/utils.py b/tests/utils.py
index 114470d64118..f24bd81bf64f 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -147,6 +147,7 @@ def setup_test_homeserver(
         config.hs_disabled_message = ""
         config.hs_disabled_limit_type = ""
         config.max_mau_value = 50
+        config.mau_trial_days = 0
         config.mau_limits_reserved_threepids = []
         config.admin_contact = None
         config.rc_messages_per_second = 10000

From 52ec6e9dfad5e088f000d20cbb4a1bbc8008c00d Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Fri, 7 Sep 2018 02:58:18 +1000
Subject: [PATCH 574/830] Port tests/ to Python 3 (#3808)

---
 changelog.d/3808.misc                         |   1 +
 tests/api/test_auth.py                        |   1 +
 tests/app/test_frontend_proxy.py              |   4 +-
 tests/handlers/test_typing.py                 |   4 +-
 tests/rest/client/v1/test_rooms.py            | 561 ++++++++----------
 tests/rest/client/v2_alpha/test_sync.py       |   8 +-
 tests/server.py                               |   2 +-
 .../test_resource_limits_server_notices.py    |  30 +-
 tests/storage/test_state.py                   | 102 ++--
 tests/test_mau.py                             |  30 +-
 tests/test_state.py                           |   2 +-
 tests/utils.py                                |  34 +-
 12 files changed, 349 insertions(+), 430 deletions(-)
 create mode 100644 changelog.d/3808.misc

diff --git a/changelog.d/3808.misc b/changelog.d/3808.misc
new file mode 100644
index 000000000000..e5e1cd9e0e1a
--- /dev/null
+++ b/changelog.d/3808.misc
@@ -0,0 +1 @@
+tests/ is now ported to Python 3.
diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py
index f65a27e5f1ff..379e9c4ab198 100644
--- a/tests/api/test_auth.py
+++ b/tests/api/test_auth.py
@@ -471,6 +471,7 @@ def test_blocking_mau(self):
     def test_reserved_threepid(self):
         self.hs.config.limit_usage_by_mau = True
         self.hs.config.max_mau_value = 1
+        self.store.get_monthly_active_count = lambda: defer.succeed(2)
         threepid = {'medium': 'email', 'address': 'reserved@server.com'}
         unknown_threepid = {'medium': 'email', 'address': 'unreserved@server.com'}
         self.hs.config.mau_limits_reserved_threepids = [threepid]
diff --git a/tests/app/test_frontend_proxy.py b/tests/app/test_frontend_proxy.py
index 76b5090fffd1..a83f567ebd7e 100644
--- a/tests/app/test_frontend_proxy.py
+++ b/tests/app/test_frontend_proxy.py
@@ -47,7 +47,7 @@ def test_listen_http_with_presence_enabled(self):
         self.assertEqual(len(self.reactor.tcpServers), 1)
         site = self.reactor.tcpServers[0][1]
         self.resource = (
-            site.resource.children["_matrix"].children["client"].children["r0"]
+            site.resource.children[b"_matrix"].children[b"client"].children[b"r0"]
         )
 
         request, channel = self.make_request("PUT", "presence/a/status")
@@ -77,7 +77,7 @@ def test_listen_http_with_presence_disabled(self):
         self.assertEqual(len(self.reactor.tcpServers), 1)
         site = self.reactor.tcpServers[0][1]
         self.resource = (
-            site.resource.children["_matrix"].children["client"].children["r0"]
+            site.resource.children[b"_matrix"].children[b"client"].children[b"r0"]
         )
 
         request, channel = self.make_request("PUT", "presence/a/status")
diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py
index c2d951b45fe5..36e136cdeddf 100644
--- a/tests/handlers/test_typing.py
+++ b/tests/handlers/test_typing.py
@@ -43,9 +43,7 @@ def _expect_edu_transaction(edu_type, content, origin="test"):
 
 
 def _make_edu_transaction_json(edu_type, content):
-    return json.dumps(_expect_edu_transaction(edu_type, content)).encode(
-        'utf8'
-    )
+    return json.dumps(_expect_edu_transaction(edu_type, content)).encode('utf8')
 
 
 class TypingNotificationsTestCase(unittest.TestCase):
diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py
index 9fe0760496de..359f7777ffd6 100644
--- a/tests/rest/client/v1/test_rooms.py
+++ b/tests/rest/client/v1/test_rooms.py
@@ -22,39 +22,24 @@
 
 from twisted.internet import defer
 
-import synapse.rest.client.v1.room
 from synapse.api.constants import Membership
-from synapse.http.server import JsonResource
-from synapse.types import UserID
-from synapse.util import Clock
+from synapse.rest.client.v1 import room
 
 from tests import unittest
-from tests.server import (
-    ThreadedMemoryReactorClock,
-    make_request,
-    render,
-    setup_test_homeserver,
-)
-
-from .utils import RestHelper
 
 PATH_PREFIX = b"/_matrix/client/api/v1"
 
 
-class RoomBase(unittest.TestCase):
+class RoomBase(unittest.HomeserverTestCase):
     rmcreator_id = None
 
-    def setUp(self):
+    servlets = [room.register_servlets, room.register_deprecated_servlets]
 
-        self.clock = ThreadedMemoryReactorClock()
-        self.hs_clock = Clock(self.clock)
+    def make_homeserver(self, reactor, clock):
 
-        self.hs = setup_test_homeserver(
-            self.addCleanup,
+        self.hs = self.setup_test_homeserver(
             "red",
             http_client=None,
-            clock=self.hs_clock,
-            reactor=self.clock,
             federation_client=Mock(),
             ratelimiter=NonCallableMock(spec_set=["send_message"]),
         )
@@ -63,42 +48,21 @@ def setUp(self):
 
         self.hs.get_federation_handler = Mock(return_value=Mock())
 
-        def get_user_by_access_token(token=None, allow_guest=False):
-            return {
-                "user": UserID.from_string(self.helper.auth_user_id),
-                "token_id": 1,
-                "is_guest": False,
-            }
-
-        def get_user_by_req(request, allow_guest=False, rights="access"):
-            return synapse.types.create_requester(
-                UserID.from_string(self.helper.auth_user_id), 1, False, None
-            )
-
-        self.hs.get_auth().get_user_by_req = get_user_by_req
-        self.hs.get_auth().get_user_by_access_token = get_user_by_access_token
-        self.hs.get_auth().get_access_token_from_request = Mock(return_value=b"1234")
-
         def _insert_client_ip(*args, **kwargs):
             return defer.succeed(None)
 
         self.hs.get_datastore().insert_client_ip = _insert_client_ip
 
-        self.resource = JsonResource(self.hs)
-        synapse.rest.client.v1.room.register_servlets(self.hs, self.resource)
-        synapse.rest.client.v1.room.register_deprecated_servlets(self.hs, self.resource)
-        self.helper = RestHelper(self.hs, self.resource, self.user_id)
+        return self.hs
 
 
 class RoomPermissionsTestCase(RoomBase):
     """ Tests room permissions. """
 
-    user_id = b"@sid1:red"
-    rmcreator_id = b"@notme:red"
-
-    def setUp(self):
+    user_id = "@sid1:red"
+    rmcreator_id = "@notme:red"
 
-        super(RoomPermissionsTestCase, self).setUp()
+    def prepare(self, reactor, clock, hs):
 
         self.helper.auth_user_id = self.rmcreator_id
         # create some rooms under the name rmcreator_id
@@ -114,22 +78,20 @@ def setUp(self):
         self.created_rmid_msg_path = (
             "rooms/%s/send/m.room.message/a1" % (self.created_rmid)
         ).encode('ascii')
-        request, channel = make_request(
-            b"PUT",
-            self.created_rmid_msg_path,
-            b'{"msgtype":"m.text","body":"test msg"}',
+        request, channel = self.make_request(
+            "PUT", self.created_rmid_msg_path, b'{"msgtype":"m.text","body":"test msg"}'
         )
-        render(request, self.resource, self.clock)
-        self.assertEquals(channel.result["code"], b"200", channel.result)
+        self.render(request)
+        self.assertEquals(200, channel.code, channel.result)
 
         # set topic for public room
-        request, channel = make_request(
-            b"PUT",
+        request, channel = self.make_request(
+            "PUT",
             ("rooms/%s/state/m.room.topic" % self.created_public_rmid).encode('ascii'),
             b'{"topic":"Public Room Topic"}',
         )
-        render(request, self.resource, self.clock)
-        self.assertEquals(channel.result["code"], b"200", channel.result)
+        self.render(request)
+        self.assertEquals(200, channel.code, channel.result)
 
         # auth as user_id now
         self.helper.auth_user_id = self.user_id
@@ -140,128 +102,128 @@ def test_send_message(self):
         seq = iter(range(100))
 
         def send_msg_path():
-            return b"/rooms/%s/send/m.room.message/mid%s" % (
+            return "/rooms/%s/send/m.room.message/mid%s" % (
                 self.created_rmid,
-                str(next(seq)).encode('ascii'),
+                str(next(seq)),
             )
 
         # send message in uncreated room, expect 403
-        request, channel = make_request(
-            b"PUT",
-            b"/rooms/%s/send/m.room.message/mid2" % (self.uncreated_rmid,),
+        request, channel = self.make_request(
+            "PUT",
+            "/rooms/%s/send/m.room.message/mid2" % (self.uncreated_rmid,),
             msg_content,
         )
-        render(request, self.resource, self.clock)
-        self.assertEquals(403, int(channel.result["code"]), msg=channel.result["body"])
+        self.render(request)
+        self.assertEquals(403, channel.code, msg=channel.result["body"])
 
         # send message in created room not joined (no state), expect 403
-        request, channel = make_request(b"PUT", send_msg_path(), msg_content)
-        render(request, self.resource, self.clock)
-        self.assertEquals(403, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("PUT", send_msg_path(), msg_content)
+        self.render(request)
+        self.assertEquals(403, channel.code, msg=channel.result["body"])
 
         # send message in created room and invited, expect 403
         self.helper.invite(
             room=self.created_rmid, src=self.rmcreator_id, targ=self.user_id
         )
-        request, channel = make_request(b"PUT", send_msg_path(), msg_content)
-        render(request, self.resource, self.clock)
-        self.assertEquals(403, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("PUT", send_msg_path(), msg_content)
+        self.render(request)
+        self.assertEquals(403, channel.code, msg=channel.result["body"])
 
         # send message in created room and joined, expect 200
         self.helper.join(room=self.created_rmid, user=self.user_id)
-        request, channel = make_request(b"PUT", send_msg_path(), msg_content)
-        render(request, self.resource, self.clock)
-        self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("PUT", send_msg_path(), msg_content)
+        self.render(request)
+        self.assertEquals(200, channel.code, msg=channel.result["body"])
 
         # send message in created room and left, expect 403
         self.helper.leave(room=self.created_rmid, user=self.user_id)
-        request, channel = make_request(b"PUT", send_msg_path(), msg_content)
-        render(request, self.resource, self.clock)
-        self.assertEquals(403, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("PUT", send_msg_path(), msg_content)
+        self.render(request)
+        self.assertEquals(403, channel.code, msg=channel.result["body"])
 
     def test_topic_perms(self):
         topic_content = b'{"topic":"My Topic Name"}'
-        topic_path = b"/rooms/%s/state/m.room.topic" % self.created_rmid
+        topic_path = "/rooms/%s/state/m.room.topic" % self.created_rmid
 
         # set/get topic in uncreated room, expect 403
-        request, channel = make_request(
-            b"PUT", b"/rooms/%s/state/m.room.topic" % self.uncreated_rmid, topic_content
+        request, channel = self.make_request(
+            "PUT", "/rooms/%s/state/m.room.topic" % self.uncreated_rmid, topic_content
         )
-        render(request, self.resource, self.clock)
-        self.assertEquals(403, int(channel.result["code"]), msg=channel.result["body"])
-        request, channel = make_request(
-            b"GET", "/rooms/%s/state/m.room.topic" % self.uncreated_rmid
+        self.render(request)
+        self.assertEquals(403, channel.code, msg=channel.result["body"])
+        request, channel = self.make_request(
+            "GET", "/rooms/%s/state/m.room.topic" % self.uncreated_rmid
         )
-        render(request, self.resource, self.clock)
-        self.assertEquals(403, int(channel.result["code"]), msg=channel.result["body"])
+        self.render(request)
+        self.assertEquals(403, channel.code, msg=channel.result["body"])
 
         # set/get topic in created PRIVATE room not joined, expect 403
-        request, channel = make_request(b"PUT", topic_path, topic_content)
-        render(request, self.resource, self.clock)
-        self.assertEquals(403, int(channel.result["code"]), msg=channel.result["body"])
-        request, channel = make_request(b"GET", topic_path)
-        render(request, self.resource, self.clock)
-        self.assertEquals(403, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("PUT", topic_path, topic_content)
+        self.render(request)
+        self.assertEquals(403, channel.code, msg=channel.result["body"])
+        request, channel = self.make_request("GET", topic_path)
+        self.render(request)
+        self.assertEquals(403, channel.code, msg=channel.result["body"])
 
         # set topic in created PRIVATE room and invited, expect 403
         self.helper.invite(
             room=self.created_rmid, src=self.rmcreator_id, targ=self.user_id
         )
-        request, channel = make_request(b"PUT", topic_path, topic_content)
-        render(request, self.resource, self.clock)
-        self.assertEquals(403, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("PUT", topic_path, topic_content)
+        self.render(request)
+        self.assertEquals(403, channel.code, msg=channel.result["body"])
 
         # get topic in created PRIVATE room and invited, expect 403
-        request, channel = make_request(b"GET", topic_path)
-        render(request, self.resource, self.clock)
-        self.assertEquals(403, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("GET", topic_path)
+        self.render(request)
+        self.assertEquals(403, channel.code, msg=channel.result["body"])
 
         # set/get topic in created PRIVATE room and joined, expect 200
         self.helper.join(room=self.created_rmid, user=self.user_id)
 
         # Only room ops can set topic by default
         self.helper.auth_user_id = self.rmcreator_id
-        request, channel = make_request(b"PUT", topic_path, topic_content)
-        render(request, self.resource, self.clock)
-        self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("PUT", topic_path, topic_content)
+        self.render(request)
+        self.assertEquals(200, channel.code, msg=channel.result["body"])
         self.helper.auth_user_id = self.user_id
 
-        request, channel = make_request(b"GET", topic_path)
-        render(request, self.resource, self.clock)
-        self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
-        self.assert_dict(json.loads(topic_content), channel.json_body)
+        request, channel = self.make_request("GET", topic_path)
+        self.render(request)
+        self.assertEquals(200, channel.code, msg=channel.result["body"])
+        self.assert_dict(json.loads(topic_content.decode('utf8')), channel.json_body)
 
         # set/get topic in created PRIVATE room and left, expect 403
         self.helper.leave(room=self.created_rmid, user=self.user_id)
-        request, channel = make_request(b"PUT", topic_path, topic_content)
-        render(request, self.resource, self.clock)
-        self.assertEquals(403, int(channel.result["code"]), msg=channel.result["body"])
-        request, channel = make_request(b"GET", topic_path)
-        render(request, self.resource, self.clock)
-        self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("PUT", topic_path, topic_content)
+        self.render(request)
+        self.assertEquals(403, channel.code, msg=channel.result["body"])
+        request, channel = self.make_request("GET", topic_path)
+        self.render(request)
+        self.assertEquals(200, channel.code, msg=channel.result["body"])
 
         # get topic in PUBLIC room, not joined, expect 403
-        request, channel = make_request(
-            b"GET", b"/rooms/%s/state/m.room.topic" % self.created_public_rmid
+        request, channel = self.make_request(
+            "GET", "/rooms/%s/state/m.room.topic" % self.created_public_rmid
         )
-        render(request, self.resource, self.clock)
-        self.assertEquals(403, int(channel.result["code"]), msg=channel.result["body"])
+        self.render(request)
+        self.assertEquals(403, channel.code, msg=channel.result["body"])
 
         # set topic in PUBLIC room, not joined, expect 403
-        request, channel = make_request(
-            b"PUT",
-            b"/rooms/%s/state/m.room.topic" % self.created_public_rmid,
+        request, channel = self.make_request(
+            "PUT",
+            "/rooms/%s/state/m.room.topic" % self.created_public_rmid,
             topic_content,
         )
-        render(request, self.resource, self.clock)
-        self.assertEquals(403, int(channel.result["code"]), msg=channel.result["body"])
+        self.render(request)
+        self.assertEquals(403, channel.code, msg=channel.result["body"])
 
     def _test_get_membership(self, room=None, members=[], expect_code=None):
         for member in members:
-            path = b"/rooms/%s/state/m.room.member/%s" % (room, member)
-            request, channel = make_request(b"GET", path)
-            render(request, self.resource, self.clock)
-            self.assertEquals(expect_code, int(channel.result["code"]))
+            path = "/rooms/%s/state/m.room.member/%s" % (room, member)
+            request, channel = self.make_request("GET", path)
+            self.render(request)
+            self.assertEquals(expect_code, channel.code)
 
     def test_membership_basic_room_perms(self):
         # === room does not exist ===
@@ -428,217 +390,211 @@ def test_leave_permissions(self):
 class RoomsMemberListTestCase(RoomBase):
     """ Tests /rooms/$room_id/members/list REST events."""
 
-    user_id = b"@sid1:red"
+    user_id = "@sid1:red"
 
     def test_get_member_list(self):
         room_id = self.helper.create_room_as(self.user_id)
-        request, channel = make_request(b"GET", b"/rooms/%s/members" % room_id)
-        render(request, self.resource, self.clock)
-        self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("GET", "/rooms/%s/members" % room_id)
+        self.render(request)
+        self.assertEquals(200, channel.code, msg=channel.result["body"])
 
     def test_get_member_list_no_room(self):
-        request, channel = make_request(b"GET", b"/rooms/roomdoesnotexist/members")
-        render(request, self.resource, self.clock)
-        self.assertEquals(403, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("GET", "/rooms/roomdoesnotexist/members")
+        self.render(request)
+        self.assertEquals(403, channel.code, msg=channel.result["body"])
 
     def test_get_member_list_no_permission(self):
-        room_id = self.helper.create_room_as(b"@some_other_guy:red")
-        request, channel = make_request(b"GET", b"/rooms/%s/members" % room_id)
-        render(request, self.resource, self.clock)
-        self.assertEquals(403, int(channel.result["code"]), msg=channel.result["body"])
+        room_id = self.helper.create_room_as("@some_other_guy:red")
+        request, channel = self.make_request("GET", "/rooms/%s/members" % room_id)
+        self.render(request)
+        self.assertEquals(403, channel.code, msg=channel.result["body"])
 
     def test_get_member_list_mixed_memberships(self):
-        room_creator = b"@some_other_guy:red"
+        room_creator = "@some_other_guy:red"
         room_id = self.helper.create_room_as(room_creator)
-        room_path = b"/rooms/%s/members" % room_id
+        room_path = "/rooms/%s/members" % room_id
         self.helper.invite(room=room_id, src=room_creator, targ=self.user_id)
         # can't see list if you're just invited.
-        request, channel = make_request(b"GET", room_path)
-        render(request, self.resource, self.clock)
-        self.assertEquals(403, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("GET", room_path)
+        self.render(request)
+        self.assertEquals(403, channel.code, msg=channel.result["body"])
 
         self.helper.join(room=room_id, user=self.user_id)
         # can see list now joined
-        request, channel = make_request(b"GET", room_path)
-        render(request, self.resource, self.clock)
-        self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("GET", room_path)
+        self.render(request)
+        self.assertEquals(200, channel.code, msg=channel.result["body"])
 
         self.helper.leave(room=room_id, user=self.user_id)
         # can see old list once left
-        request, channel = make_request(b"GET", room_path)
-        render(request, self.resource, self.clock)
-        self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("GET", room_path)
+        self.render(request)
+        self.assertEquals(200, channel.code, msg=channel.result["body"])
 
 
 class RoomsCreateTestCase(RoomBase):
     """ Tests /rooms and /rooms/$room_id REST events. """
 
-    user_id = b"@sid1:red"
+    user_id = "@sid1:red"
 
     def test_post_room_no_keys(self):
         # POST with no config keys, expect new room id
-        request, channel = make_request(b"POST", b"/createRoom", b"{}")
+        request, channel = self.make_request("POST", "/createRoom", "{}")
 
-        render(request, self.resource, self.clock)
-        self.assertEquals(200, int(channel.result["code"]), channel.result)
+        self.render(request)
+        self.assertEquals(200, channel.code, channel.result)
         self.assertTrue("room_id" in channel.json_body)
 
     def test_post_room_visibility_key(self):
         # POST with visibility config key, expect new room id
-        request, channel = make_request(
-            b"POST", b"/createRoom", b'{"visibility":"private"}'
+        request, channel = self.make_request(
+            "POST", "/createRoom", b'{"visibility":"private"}'
         )
-        render(request, self.resource, self.clock)
-        self.assertEquals(200, int(channel.result["code"]))
+        self.render(request)
+        self.assertEquals(200, channel.code)
         self.assertTrue("room_id" in channel.json_body)
 
     def test_post_room_custom_key(self):
         # POST with custom config keys, expect new room id
-        request, channel = make_request(b"POST", b"/createRoom", b'{"custom":"stuff"}')
-        render(request, self.resource, self.clock)
-        self.assertEquals(200, int(channel.result["code"]))
+        request, channel = self.make_request(
+            "POST", "/createRoom", b'{"custom":"stuff"}'
+        )
+        self.render(request)
+        self.assertEquals(200, channel.code)
         self.assertTrue("room_id" in channel.json_body)
 
     def test_post_room_known_and_unknown_keys(self):
         # POST with custom + known config keys, expect new room id
-        request, channel = make_request(
-            b"POST", b"/createRoom", b'{"visibility":"private","custom":"things"}'
+        request, channel = self.make_request(
+            "POST", "/createRoom", b'{"visibility":"private","custom":"things"}'
         )
-        render(request, self.resource, self.clock)
-        self.assertEquals(200, int(channel.result["code"]))
+        self.render(request)
+        self.assertEquals(200, channel.code)
         self.assertTrue("room_id" in channel.json_body)
 
     def test_post_room_invalid_content(self):
         # POST with invalid content / paths, expect 400
-        request, channel = make_request(b"POST", b"/createRoom", b'{"visibili')
-        render(request, self.resource, self.clock)
-        self.assertEquals(400, int(channel.result["code"]))
+        request, channel = self.make_request("POST", "/createRoom", b'{"visibili')
+        self.render(request)
+        self.assertEquals(400, channel.code)
 
-        request, channel = make_request(b"POST", b"/createRoom", b'["hello"]')
-        render(request, self.resource, self.clock)
-        self.assertEquals(400, int(channel.result["code"]))
+        request, channel = self.make_request("POST", "/createRoom", b'["hello"]')
+        self.render(request)
+        self.assertEquals(400, channel.code)
 
 
 class RoomTopicTestCase(RoomBase):
     """ Tests /rooms/$room_id/topic REST events. """
 
-    user_id = b"@sid1:red"
-
-    def setUp(self):
-
-        super(RoomTopicTestCase, self).setUp()
+    user_id = "@sid1:red"
 
+    def prepare(self, reactor, clock, hs):
         # create the room
         self.room_id = self.helper.create_room_as(self.user_id)
-        self.path = b"/rooms/%s/state/m.room.topic" % (self.room_id,)
+        self.path = "/rooms/%s/state/m.room.topic" % (self.room_id,)
 
     def test_invalid_puts(self):
         # missing keys or invalid json
-        request, channel = make_request(b"PUT", self.path, '{}')
-        render(request, self.resource, self.clock)
-        self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("PUT", self.path, '{}')
+        self.render(request)
+        self.assertEquals(400, channel.code, msg=channel.result["body"])
 
-        request, channel = make_request(b"PUT", self.path, '{"_name":"bob"}')
-        render(request, self.resource, self.clock)
-        self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("PUT", self.path, '{"_name":"bo"}')
+        self.render(request)
+        self.assertEquals(400, channel.code, msg=channel.result["body"])
 
-        request, channel = make_request(b"PUT", self.path, '{"nao')
-        render(request, self.resource, self.clock)
-        self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("PUT", self.path, '{"nao')
+        self.render(request)
+        self.assertEquals(400, channel.code, msg=channel.result["body"])
 
-        request, channel = make_request(
-            b"PUT", self.path, '[{"_name":"bob"},{"_name":"jill"}]'
+        request, channel = self.make_request(
+            "PUT", self.path, '[{"_name":"bo"},{"_name":"jill"}]'
         )
-        render(request, self.resource, self.clock)
-        self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.render(request)
+        self.assertEquals(400, channel.code, msg=channel.result["body"])
 
-        request, channel = make_request(b"PUT", self.path, 'text only')
-        render(request, self.resource, self.clock)
-        self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("PUT", self.path, 'text only')
+        self.render(request)
+        self.assertEquals(400, channel.code, msg=channel.result["body"])
 
-        request, channel = make_request(b"PUT", self.path, '')
-        render(request, self.resource, self.clock)
-        self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("PUT", self.path, '')
+        self.render(request)
+        self.assertEquals(400, channel.code, msg=channel.result["body"])
 
         # valid key, wrong type
         content = '{"topic":["Topic name"]}'
-        request, channel = make_request(b"PUT", self.path, content)
-        render(request, self.resource, self.clock)
-        self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("PUT", self.path, content)
+        self.render(request)
+        self.assertEquals(400, channel.code, msg=channel.result["body"])
 
     def test_rooms_topic(self):
         # nothing should be there
-        request, channel = make_request(b"GET", self.path)
-        render(request, self.resource, self.clock)
-        self.assertEquals(404, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("GET", self.path)
+        self.render(request)
+        self.assertEquals(404, channel.code, msg=channel.result["body"])
 
         # valid put
         content = '{"topic":"Topic name"}'
-        request, channel = make_request(b"PUT", self.path, content)
-        render(request, self.resource, self.clock)
-        self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("PUT", self.path, content)
+        self.render(request)
+        self.assertEquals(200, channel.code, msg=channel.result["body"])
 
         # valid get
-        request, channel = make_request(b"GET", self.path)
-        render(request, self.resource, self.clock)
-        self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("GET", self.path)
+        self.render(request)
+        self.assertEquals(200, channel.code, msg=channel.result["body"])
         self.assert_dict(json.loads(content), channel.json_body)
 
     def test_rooms_topic_with_extra_keys(self):
         # valid put with extra keys
         content = '{"topic":"Seasons","subtopic":"Summer"}'
-        request, channel = make_request(b"PUT", self.path, content)
-        render(request, self.resource, self.clock)
-        self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("PUT", self.path, content)
+        self.render(request)
+        self.assertEquals(200, channel.code, msg=channel.result["body"])
 
         # valid get
-        request, channel = make_request(b"GET", self.path)
-        render(request, self.resource, self.clock)
-        self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("GET", self.path)
+        self.render(request)
+        self.assertEquals(200, channel.code, msg=channel.result["body"])
         self.assert_dict(json.loads(content), channel.json_body)
 
 
 class RoomMemberStateTestCase(RoomBase):
     """ Tests /rooms/$room_id/members/$user_id/state REST events. """
 
-    user_id = b"@sid1:red"
-
-    def setUp(self):
+    user_id = "@sid1:red"
 
-        super(RoomMemberStateTestCase, self).setUp()
+    def prepare(self, reactor, clock, hs):
         self.room_id = self.helper.create_room_as(self.user_id)
 
-    def tearDown(self):
-        pass
-
     def test_invalid_puts(self):
         path = "/rooms/%s/state/m.room.member/%s" % (self.room_id, self.user_id)
         # missing keys or invalid json
-        request, channel = make_request(b"PUT", path, '{}')
-        render(request, self.resource, self.clock)
-        self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("PUT", path, '{}')
+        self.render(request)
+        self.assertEquals(400, channel.code, msg=channel.result["body"])
 
-        request, channel = make_request(b"PUT", path, '{"_name":"bob"}')
-        render(request, self.resource, self.clock)
-        self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("PUT", path, '{"_name":"bo"}')
+        self.render(request)
+        self.assertEquals(400, channel.code, msg=channel.result["body"])
 
-        request, channel = make_request(b"PUT", path, '{"nao')
-        render(request, self.resource, self.clock)
-        self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("PUT", path, '{"nao')
+        self.render(request)
+        self.assertEquals(400, channel.code, msg=channel.result["body"])
 
-        request, channel = make_request(
-            b"PUT", path, b'[{"_name":"bob"},{"_name":"jill"}]'
+        request, channel = self.make_request(
+            "PUT", path, b'[{"_name":"bo"},{"_name":"jill"}]'
         )
-        render(request, self.resource, self.clock)
-        self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.render(request)
+        self.assertEquals(400, channel.code, msg=channel.result["body"])
 
-        request, channel = make_request(b"PUT", path, 'text only')
-        render(request, self.resource, self.clock)
-        self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("PUT", path, 'text only')
+        self.render(request)
+        self.assertEquals(400, channel.code, msg=channel.result["body"])
 
-        request, channel = make_request(b"PUT", path, '')
-        render(request, self.resource, self.clock)
-        self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("PUT", path, '')
+        self.render(request)
+        self.assertEquals(400, channel.code, msg=channel.result["body"])
 
         # valid keys, wrong types
         content = '{"membership":["%s","%s","%s"]}' % (
@@ -646,9 +602,9 @@ def test_invalid_puts(self):
             Membership.JOIN,
             Membership.LEAVE,
         )
-        request, channel = make_request(b"PUT", path, content.encode('ascii'))
-        render(request, self.resource, self.clock)
-        self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("PUT", path, content.encode('ascii'))
+        self.render(request)
+        self.assertEquals(400, channel.code, msg=channel.result["body"])
 
     def test_rooms_members_self(self):
         path = "/rooms/%s/state/m.room.member/%s" % (
@@ -658,13 +614,13 @@ def test_rooms_members_self(self):
 
         # valid join message (NOOP since we made the room)
         content = '{"membership":"%s"}' % Membership.JOIN
-        request, channel = make_request(b"PUT", path, content.encode('ascii'))
-        render(request, self.resource, self.clock)
-        self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("PUT", path, content.encode('ascii'))
+        self.render(request)
+        self.assertEquals(200, channel.code, msg=channel.result["body"])
 
-        request, channel = make_request(b"GET", path, None)
-        render(request, self.resource, self.clock)
-        self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("GET", path, None)
+        self.render(request)
+        self.assertEquals(200, channel.code, msg=channel.result["body"])
 
         expected_response = {"membership": Membership.JOIN}
         self.assertEquals(expected_response, channel.json_body)
@@ -678,13 +634,13 @@ def test_rooms_members_other(self):
 
         # valid invite message
         content = '{"membership":"%s"}' % Membership.INVITE
-        request, channel = make_request(b"PUT", path, content)
-        render(request, self.resource, self.clock)
-        self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("PUT", path, content)
+        self.render(request)
+        self.assertEquals(200, channel.code, msg=channel.result["body"])
 
-        request, channel = make_request(b"GET", path, None)
-        render(request, self.resource, self.clock)
-        self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("GET", path, None)
+        self.render(request)
+        self.assertEquals(200, channel.code, msg=channel.result["body"])
         self.assertEquals(json.loads(content), channel.json_body)
 
     def test_rooms_members_other_custom_keys(self):
@@ -699,13 +655,13 @@ def test_rooms_members_other_custom_keys(self):
             Membership.INVITE,
             "Join us!",
         )
-        request, channel = make_request(b"PUT", path, content)
-        render(request, self.resource, self.clock)
-        self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("PUT", path, content)
+        self.render(request)
+        self.assertEquals(200, channel.code, msg=channel.result["body"])
 
-        request, channel = make_request(b"GET", path, None)
-        render(request, self.resource, self.clock)
-        self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("GET", path, None)
+        self.render(request)
+        self.assertEquals(200, channel.code, msg=channel.result["body"])
         self.assertEquals(json.loads(content), channel.json_body)
 
 
@@ -714,60 +670,58 @@ class RoomMessagesTestCase(RoomBase):
 
     user_id = "@sid1:red"
 
-    def setUp(self):
-        super(RoomMessagesTestCase, self).setUp()
-
+    def prepare(self, reactor, clock, hs):
         self.room_id = self.helper.create_room_as(self.user_id)
 
     def test_invalid_puts(self):
         path = "/rooms/%s/send/m.room.message/mid1" % (urlparse.quote(self.room_id))
         # missing keys or invalid json
-        request, channel = make_request(b"PUT", path, '{}')
-        render(request, self.resource, self.clock)
-        self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("PUT", path, b'{}')
+        self.render(request)
+        self.assertEquals(400, channel.code, msg=channel.result["body"])
 
-        request, channel = make_request(b"PUT", path, '{"_name":"bob"}')
-        render(request, self.resource, self.clock)
-        self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("PUT", path, b'{"_name":"bo"}')
+        self.render(request)
+        self.assertEquals(400, channel.code, msg=channel.result["body"])
 
-        request, channel = make_request(b"PUT", path, '{"nao')
-        render(request, self.resource, self.clock)
-        self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("PUT", path, b'{"nao')
+        self.render(request)
+        self.assertEquals(400, channel.code, msg=channel.result["body"])
 
-        request, channel = make_request(
-            b"PUT", path, '[{"_name":"bob"},{"_name":"jill"}]'
+        request, channel = self.make_request(
+            "PUT", path, b'[{"_name":"bo"},{"_name":"jill"}]'
         )
-        render(request, self.resource, self.clock)
-        self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.render(request)
+        self.assertEquals(400, channel.code, msg=channel.result["body"])
 
-        request, channel = make_request(b"PUT", path, 'text only')
-        render(request, self.resource, self.clock)
-        self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("PUT", path, b'text only')
+        self.render(request)
+        self.assertEquals(400, channel.code, msg=channel.result["body"])
 
-        request, channel = make_request(b"PUT", path, '')
-        render(request, self.resource, self.clock)
-        self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+        request, channel = self.make_request("PUT", path, b'')
+        self.render(request)
+        self.assertEquals(400, channel.code, msg=channel.result["body"])
 
     def test_rooms_messages_sent(self):
         path = "/rooms/%s/send/m.room.message/mid1" % (urlparse.quote(self.room_id))
 
-        content = '{"body":"test","msgtype":{"type":"a"}}'
-        request, channel = make_request(b"PUT", path, content)
-        render(request, self.resource, self.clock)
-        self.assertEquals(400, int(channel.result["code"]), msg=channel.result["body"])
+        content = b'{"body":"test","msgtype":{"type":"a"}}'
+        request, channel = self.make_request("PUT", path, content)
+        self.render(request)
+        self.assertEquals(400, channel.code, msg=channel.result["body"])
 
         # custom message types
-        content = '{"body":"test","msgtype":"test.custom.text"}'
-        request, channel = make_request(b"PUT", path, content)
-        render(request, self.resource, self.clock)
-        self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+        content = b'{"body":"test","msgtype":"test.custom.text"}'
+        request, channel = self.make_request("PUT", path, content)
+        self.render(request)
+        self.assertEquals(200, channel.code, msg=channel.result["body"])
 
         # m.text message type
         path = "/rooms/%s/send/m.room.message/mid2" % (urlparse.quote(self.room_id))
-        content = '{"body":"test2","msgtype":"m.text"}'
-        request, channel = make_request(b"PUT", path, content)
-        render(request, self.resource, self.clock)
-        self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"])
+        content = b'{"body":"test2","msgtype":"m.text"}'
+        request, channel = self.make_request("PUT", path, content)
+        self.render(request)
+        self.assertEquals(200, channel.code, msg=channel.result["body"])
 
 
 class RoomInitialSyncTestCase(RoomBase):
@@ -775,16 +729,16 @@ class RoomInitialSyncTestCase(RoomBase):
 
     user_id = "@sid1:red"
 
-    def setUp(self):
-        super(RoomInitialSyncTestCase, self).setUp()
-
+    def prepare(self, reactor, clock, hs):
         # create the room
         self.room_id = self.helper.create_room_as(self.user_id)
 
     def test_initial_sync(self):
-        request, channel = make_request(b"GET", "/rooms/%s/initialSync" % self.room_id)
-        render(request, self.resource, self.clock)
-        self.assertEquals(200, int(channel.result["code"]))
+        request, channel = self.make_request(
+            "GET", "/rooms/%s/initialSync" % self.room_id
+        )
+        self.render(request)
+        self.assertEquals(200, channel.code)
 
         self.assertEquals(self.room_id, channel.json_body["room_id"])
         self.assertEquals("join", channel.json_body["membership"])
@@ -819,17 +773,16 @@ class RoomMessageListTestCase(RoomBase):
 
     user_id = "@sid1:red"
 
-    def setUp(self):
-        super(RoomMessageListTestCase, self).setUp()
+    def prepare(self, reactor, clock, hs):
         self.room_id = self.helper.create_room_as(self.user_id)
 
     def test_topo_token_is_accepted(self):
         token = "t1-0_0_0_0_0_0_0_0_0"
-        request, channel = make_request(
-            b"GET", "/rooms/%s/messages?access_token=x&from=%s" % (self.room_id, token)
+        request, channel = self.make_request(
+            "GET", "/rooms/%s/messages?access_token=x&from=%s" % (self.room_id, token)
         )
-        render(request, self.resource, self.clock)
-        self.assertEquals(200, int(channel.result["code"]))
+        self.render(request)
+        self.assertEquals(200, channel.code)
         self.assertTrue("start" in channel.json_body)
         self.assertEquals(token, channel.json_body['start'])
         self.assertTrue("chunk" in channel.json_body)
@@ -837,11 +790,11 @@ def test_topo_token_is_accepted(self):
 
     def test_stream_token_is_accepted_for_fwd_pagianation(self):
         token = "s0_0_0_0_0_0_0_0_0"
-        request, channel = make_request(
-            b"GET", "/rooms/%s/messages?access_token=x&from=%s" % (self.room_id, token)
+        request, channel = self.make_request(
+            "GET", "/rooms/%s/messages?access_token=x&from=%s" % (self.room_id, token)
         )
-        render(request, self.resource, self.clock)
-        self.assertEquals(200, int(channel.result["code"]))
+        self.render(request)
+        self.assertEquals(200, channel.code)
         self.assertTrue("start" in channel.json_body)
         self.assertEquals(token, channel.json_body['start'])
         self.assertTrue("chunk" in channel.json_body)
diff --git a/tests/rest/client/v2_alpha/test_sync.py b/tests/rest/client/v2_alpha/test_sync.py
index 560b1fba967a..4c30c5f258f0 100644
--- a/tests/rest/client/v2_alpha/test_sync.py
+++ b/tests/rest/client/v2_alpha/test_sync.py
@@ -62,12 +62,6 @@ def test_sync_presence_disabled(self):
         self.assertEqual(channel.code, 200)
         self.assertTrue(
             set(
-                [
-                    "next_batch",
-                    "rooms",
-                    "account_data",
-                    "to_device",
-                    "device_lists",
-                ]
+                ["next_batch", "rooms", "account_data", "to_device", "device_lists"]
             ).issubset(set(channel.json_body.keys()))
         )
diff --git a/tests/server.py b/tests/server.py
index 615bba1b5985..a2c3ca61f66d 100644
--- a/tests/server.py
+++ b/tests/server.py
@@ -65,7 +65,7 @@ def requestDone(self, _self):
     def getPeer(self):
         # We give an address so that getClientIP returns a non null entry,
         # causing us to record the MAU
-        return address.IPv4Address(b"TCP", "127.0.0.1", 3423)
+        return address.IPv4Address("TCP", "127.0.0.1", 3423)
 
     def getHost(self):
         return None
diff --git a/tests/server_notices/test_resource_limits_server_notices.py b/tests/server_notices/test_resource_limits_server_notices.py
index 5cc7fff39b17..4701eedd4507 100644
--- a/tests/server_notices/test_resource_limits_server_notices.py
+++ b/tests/server_notices/test_resource_limits_server_notices.py
@@ -80,12 +80,11 @@ def test_maybe_send_server_notice_to_user_remove_blocked_notice(self):
 
         self._rlsn._auth.check_auth_blocking = Mock()
         mock_event = Mock(
-            type=EventTypes.Message,
-            content={"msgtype": ServerNoticeMsgType},
+            type=EventTypes.Message, content={"msgtype": ServerNoticeMsgType}
+        )
+        self._rlsn._store.get_events = Mock(
+            return_value=defer.succeed({"123": mock_event})
         )
-        self._rlsn._store.get_events = Mock(return_value=defer.succeed(
-            {"123": mock_event}
-        ))
 
         yield self._rlsn.maybe_send_server_notice_to_user(self.user_id)
         # Would be better to check the content, but once == remove blocking event
@@ -99,12 +98,11 @@ def test_maybe_send_server_notice_to_user_remove_blocked_notice_noop(self):
         )
 
         mock_event = Mock(
-            type=EventTypes.Message,
-            content={"msgtype": ServerNoticeMsgType},
+            type=EventTypes.Message, content={"msgtype": ServerNoticeMsgType}
+        )
+        self._rlsn._store.get_events = Mock(
+            return_value=defer.succeed({"123": mock_event})
         )
-        self._rlsn._store.get_events = Mock(return_value=defer.succeed(
-            {"123": mock_event}
-        ))
         yield self._rlsn.maybe_send_server_notice_to_user(self.user_id)
 
         self._send_notice.assert_not_called()
@@ -177,13 +175,9 @@ def setUp(self):
 
     @defer.inlineCallbacks
     def test_server_notice_only_sent_once(self):
-        self.store.get_monthly_active_count = Mock(
-            return_value=1000,
-        )
+        self.store.get_monthly_active_count = Mock(return_value=1000)
 
-        self.store.user_last_seen_monthly_active = Mock(
-            return_value=1000,
-        )
+        self.store.user_last_seen_monthly_active = Mock(return_value=1000)
 
         # Call the function multiple times to ensure we only send the notice once
         yield self._rlsn.maybe_send_server_notice_to_user(self.user_id)
@@ -193,12 +187,12 @@ def test_server_notice_only_sent_once(self):
         # Now lets get the last load of messages in the service notice room and
         # check that there is only one server notice
         room_id = yield self.server_notices_manager.get_notice_room_for_user(
-            self.user_id,
+            self.user_id
         )
 
         token = yield self.event_source.get_current_token()
         events, _ = yield self.store.get_recent_events_for_room(
-            room_id, limit=100, end_token=token.room_key,
+            room_id, limit=100, end_token=token.room_key
         )
 
         count = 0
diff --git a/tests/storage/test_state.py b/tests/storage/test_state.py
index d717b9f94e27..b91096593216 100644
--- a/tests/storage/test_state.py
+++ b/tests/storage/test_state.py
@@ -185,8 +185,7 @@ def test_get_state_for_event(self):
 
         # test _get_some_state_from_cache correctly filters out members with types=[]
         (state_dict, is_all) = yield self.store._get_some_state_from_cache(
-            self.store._state_group_cache,
-            group, [], filtered_types=[EventTypes.Member]
+            self.store._state_group_cache, group, [], filtered_types=[EventTypes.Member]
         )
 
         self.assertEqual(is_all, True)
@@ -200,19 +199,20 @@ def test_get_state_for_event(self):
 
         (state_dict, is_all) = yield self.store._get_some_state_from_cache(
             self.store._state_group_members_cache,
-            group, [], filtered_types=[EventTypes.Member]
+            group,
+            [],
+            filtered_types=[EventTypes.Member],
         )
 
         self.assertEqual(is_all, True)
-        self.assertDictEqual(
-            {},
-            state_dict,
-        )
+        self.assertDictEqual({}, state_dict)
 
         # test _get_some_state_from_cache correctly filters in members with wildcard types
         (state_dict, is_all) = yield self.store._get_some_state_from_cache(
             self.store._state_group_cache,
-            group, [(EventTypes.Member, None)], filtered_types=[EventTypes.Member]
+            group,
+            [(EventTypes.Member, None)],
+            filtered_types=[EventTypes.Member],
         )
 
         self.assertEqual(is_all, True)
@@ -226,7 +226,9 @@ def test_get_state_for_event(self):
 
         (state_dict, is_all) = yield self.store._get_some_state_from_cache(
             self.store._state_group_members_cache,
-            group, [(EventTypes.Member, None)], filtered_types=[EventTypes.Member]
+            group,
+            [(EventTypes.Member, None)],
+            filtered_types=[EventTypes.Member],
         )
 
         self.assertEqual(is_all, True)
@@ -264,18 +266,15 @@ def test_get_state_for_event(self):
         )
 
         self.assertEqual(is_all, True)
-        self.assertDictEqual(
-            {
-                (e5.type, e5.state_key): e5.event_id,
-            },
-            state_dict,
-        )
+        self.assertDictEqual({(e5.type, e5.state_key): e5.event_id}, state_dict)
 
         # test _get_some_state_from_cache correctly filters in members with specific types
         # and no filtered_types
         (state_dict, is_all) = yield self.store._get_some_state_from_cache(
             self.store._state_group_members_cache,
-            group, [(EventTypes.Member, e5.state_key)], filtered_types=None
+            group,
+            [(EventTypes.Member, e5.state_key)],
+            filtered_types=None,
         )
 
         self.assertEqual(is_all, True)
@@ -305,9 +304,7 @@ def test_get_state_for_event(self):
             key=group,
             value=state_dict_ids,
             # list fetched keys so it knows it's partial
-            fetched_keys=(
-                (e1.type, e1.state_key),
-            ),
+            fetched_keys=((e1.type, e1.state_key),),
         )
 
         (is_all, known_absent, state_dict_ids) = self.store._state_group_cache.get(
@@ -315,20 +312,8 @@ def test_get_state_for_event(self):
         )
 
         self.assertEqual(is_all, False)
-        self.assertEqual(
-            known_absent,
-            set(
-                [
-                    (e1.type, e1.state_key),
-                ]
-            ),
-        )
-        self.assertDictEqual(
-            state_dict_ids,
-            {
-                (e1.type, e1.state_key): e1.event_id,
-            },
-        )
+        self.assertEqual(known_absent, set([(e1.type, e1.state_key)]))
+        self.assertDictEqual(state_dict_ids, {(e1.type, e1.state_key): e1.event_id})
 
         ############################################
         # test that things work with a partial cache
@@ -336,8 +321,7 @@ def test_get_state_for_event(self):
         # test _get_some_state_from_cache correctly filters out members with types=[]
         room_id = self.room.to_string()
         (state_dict, is_all) = yield self.store._get_some_state_from_cache(
-            self.store._state_group_cache,
-            group, [], filtered_types=[EventTypes.Member]
+            self.store._state_group_cache, group, [], filtered_types=[EventTypes.Member]
         )
 
         self.assertEqual(is_all, False)
@@ -346,7 +330,9 @@ def test_get_state_for_event(self):
         room_id = self.room.to_string()
         (state_dict, is_all) = yield self.store._get_some_state_from_cache(
             self.store._state_group_members_cache,
-            group, [], filtered_types=[EventTypes.Member]
+            group,
+            [],
+            filtered_types=[EventTypes.Member],
         )
 
         self.assertEqual(is_all, True)
@@ -355,20 +341,19 @@ def test_get_state_for_event(self):
         # test _get_some_state_from_cache correctly filters in members wildcard types
         (state_dict, is_all) = yield self.store._get_some_state_from_cache(
             self.store._state_group_cache,
-            group, [(EventTypes.Member, None)], filtered_types=[EventTypes.Member]
+            group,
+            [(EventTypes.Member, None)],
+            filtered_types=[EventTypes.Member],
         )
 
         self.assertEqual(is_all, False)
-        self.assertDictEqual(
-            {
-                (e1.type, e1.state_key): e1.event_id,
-            },
-            state_dict,
-        )
+        self.assertDictEqual({(e1.type, e1.state_key): e1.event_id}, state_dict)
 
         (state_dict, is_all) = yield self.store._get_some_state_from_cache(
             self.store._state_group_members_cache,
-            group, [(EventTypes.Member, None)], filtered_types=[EventTypes.Member]
+            group,
+            [(EventTypes.Member, None)],
+            filtered_types=[EventTypes.Member],
         )
 
         self.assertEqual(is_all, True)
@@ -389,12 +374,7 @@ def test_get_state_for_event(self):
         )
 
         self.assertEqual(is_all, False)
-        self.assertDictEqual(
-            {
-                (e1.type, e1.state_key): e1.event_id,
-            },
-            state_dict,
-        )
+        self.assertDictEqual({(e1.type, e1.state_key): e1.event_id}, state_dict)
 
         (state_dict, is_all) = yield self.store._get_some_state_from_cache(
             self.store._state_group_members_cache,
@@ -404,18 +384,15 @@ def test_get_state_for_event(self):
         )
 
         self.assertEqual(is_all, True)
-        self.assertDictEqual(
-            {
-                (e5.type, e5.state_key): e5.event_id,
-            },
-            state_dict,
-        )
+        self.assertDictEqual({(e5.type, e5.state_key): e5.event_id}, state_dict)
 
         # test _get_some_state_from_cache correctly filters in members with specific types
         # and no filtered_types
         (state_dict, is_all) = yield self.store._get_some_state_from_cache(
             self.store._state_group_cache,
-            group, [(EventTypes.Member, e5.state_key)], filtered_types=None
+            group,
+            [(EventTypes.Member, e5.state_key)],
+            filtered_types=None,
         )
 
         self.assertEqual(is_all, False)
@@ -423,13 +400,10 @@ def test_get_state_for_event(self):
 
         (state_dict, is_all) = yield self.store._get_some_state_from_cache(
             self.store._state_group_members_cache,
-            group, [(EventTypes.Member, e5.state_key)], filtered_types=None
+            group,
+            [(EventTypes.Member, e5.state_key)],
+            filtered_types=None,
         )
 
         self.assertEqual(is_all, True)
-        self.assertDictEqual(
-            {
-                (e5.type, e5.state_key): e5.event_id,
-            },
-            state_dict,
-        )
+        self.assertDictEqual({(e5.type, e5.state_key): e5.event_id}, state_dict)
diff --git a/tests/test_mau.py b/tests/test_mau.py
index 0732615447ea..bdbacb844848 100644
--- a/tests/test_mau.py
+++ b/tests/test_mau.py
@@ -185,20 +185,20 @@ def test_trial_users_cant_come_back(self):
         self.assertEqual(e.errcode, Codes.RESOURCE_LIMIT_EXCEEDED)
 
     def create_user(self, localpart):
-        request_data = json.dumps({
-            "username": localpart,
-            "password": "monkey",
-            "auth": {"type": LoginType.DUMMY},
-        })
+        request_data = json.dumps(
+            {
+                "username": localpart,
+                "password": "monkey",
+                "auth": {"type": LoginType.DUMMY},
+            }
+        )
 
-        request, channel = make_request(b"POST", b"/register", request_data)
+        request, channel = make_request("POST", "/register", request_data)
         render(request, self.resource, self.reactor)
 
-        if channel.result["code"] != b"200":
+        if channel.code != 200:
             raise HttpResponseException(
-                int(channel.result["code"]),
-                channel.result["reason"],
-                channel.result["body"],
+                channel.code, channel.result["reason"], channel.result["body"]
             ).to_synapse_error()
 
         access_token = channel.json_body["access_token"]
@@ -206,12 +206,12 @@ def create_user(self, localpart):
         return access_token
 
     def do_sync_for_user(self, token):
-        request, channel = make_request(b"GET", b"/sync", access_token=token)
+        request, channel = make_request(
+            "GET", "/sync", access_token=token.encode('ascii')
+        )
         render(request, self.resource, self.reactor)
 
-        if channel.result["code"] != b"200":
+        if channel.code != 200:
             raise HttpResponseException(
-                int(channel.result["code"]),
-                channel.result["reason"],
-                channel.result["body"],
+                channel.code, channel.result["reason"], channel.result["body"]
             ).to_synapse_error()
diff --git a/tests/test_state.py b/tests/test_state.py
index 452a123c3a29..e20c33322a47 100644
--- a/tests/test_state.py
+++ b/tests/test_state.py
@@ -180,7 +180,7 @@ def test_branch_no_conflict(self):
         graph = Graph(
             nodes={
                 "START": DictObj(
-                    type=EventTypes.Create, state_key="", content={}, depth=1,
+                    type=EventTypes.Create, state_key="", content={}, depth=1
                 ),
                 "A": DictObj(type=EventTypes.Message, depth=2),
                 "B": DictObj(type=EventTypes.Message, depth=3),
diff --git a/tests/utils.py b/tests/utils.py
index 91173fa45d51..215226debfec 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -100,8 +100,13 @@ class TestHomeServer(HomeServer):
 
 @defer.inlineCallbacks
 def setup_test_homeserver(
-    cleanup_func, name="test", datastore=None, config=None, reactor=None,
-    homeserverToUse=TestHomeServer, **kargs
+    cleanup_func,
+    name="test",
+    datastore=None,
+    config=None,
+    reactor=None,
+    homeserverToUse=TestHomeServer,
+    **kargs
 ):
     """
     Setup a homeserver suitable for running tests against.  Keyword arguments
@@ -323,8 +328,7 @@ def trigger_get(self, path):
     @patch('twisted.web.http.Request')
     @defer.inlineCallbacks
     def trigger(
-        self, http_method, path, content, mock_request,
-        federation_auth_origin=None,
+        self, http_method, path, content, mock_request, federation_auth_origin=None
     ):
         """ Fire an HTTP event.
 
@@ -357,7 +361,7 @@ def trigger(
         headers = {}
         if federation_auth_origin is not None:
             headers[b"Authorization"] = [
-                b"X-Matrix origin=%s,key=,sig=" % (federation_auth_origin, )
+                b"X-Matrix origin=%s,key=,sig=" % (federation_auth_origin,)
             ]
         mock_request.requestHeaders.getRawHeaders = mock_getRawHeaders(headers)
 
@@ -577,16 +581,16 @@ def create_room(hs, room_id, creator_id):
     event_builder_factory = hs.get_event_builder_factory()
     event_creation_handler = hs.get_event_creation_handler()
 
-    builder = event_builder_factory.new({
-        "type": EventTypes.Create,
-        "state_key": "",
-        "sender": creator_id,
-        "room_id": room_id,
-        "content": {},
-    })
-
-    event, context = yield event_creation_handler.create_new_client_event(
-        builder
+    builder = event_builder_factory.new(
+        {
+            "type": EventTypes.Create,
+            "state_key": "",
+            "sender": creator_id,
+            "room_id": room_id,
+            "content": {},
+        }
     )
 
+    event, context = yield event_creation_handler.create_new_client_event(builder)
+
     yield store.persist_event(event, context)

From 806964b5de3e5036a897d4b5d477e7616779d019 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 6 Sep 2018 18:51:06 +0100
Subject: [PATCH 575/830] add some logging for the keyring queue

why is it so damn slow?
---
 synapse/crypto/keyring.py | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py
index 30e2742102d6..a6b31e8b5be2 100644
--- a/synapse/crypto/keyring.py
+++ b/synapse/crypto/keyring.py
@@ -40,6 +40,7 @@
 from synapse.crypto.keyclient import fetch_server_key
 from synapse.util import logcontext, unwrapFirstError
 from synapse.util.logcontext import (
+    LoggingContext,
     PreserveLoggingContext,
     preserve_fn,
     run_in_background,
@@ -216,23 +217,34 @@ def wait_for_previous_lookups(self, server_names, server_to_deferred):
             servers have completed. Follows the synapse rules of logcontext
             preservation.
         """
+        loop_count = 1
         while True:
             wait_on = [
-                self.key_downloads[server_name]
+                (server_name, self.key_downloads[server_name])
                 for server_name in server_names
                 if server_name in self.key_downloads
             ]
-            if wait_on:
-                with PreserveLoggingContext():
-                    yield defer.DeferredList(wait_on)
-            else:
+            if not wait_on:
                 break
+            logger.info(
+                "Waiting for existing lookups for %s to complete [loop %i]",
+                [w[0] for w in wait_on], loop_count,
+            )
+            with PreserveLoggingContext():
+                yield defer.DeferredList((w[1] for w in wait_on))
+
+            loop_count += 1
+
+        ctx = LoggingContext.current_context()
 
         def rm(r, server_name_):
-            self.key_downloads.pop(server_name_, None)
+            with PreserveLoggingContext(ctx):
+                logger.debug("Releasing key lookup lock on %s", server_name_)
+                self.key_downloads.pop(server_name_, None)
             return r
 
         for server_name, deferred in server_to_deferred.items():
+            logger.debug("Got key lookup lock on %s", server_name)
             self.key_downloads[server_name] = deferred
             deferred.addBoth(rm, server_name)
 

From cd7ef4387205eaeb64b77860a82f782f91e41e93 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 6 Sep 2018 23:56:47 +0100
Subject: [PATCH 576/830] clearer logging when things fail, too

---
 synapse/federation/federation_base.py | 34 +++++++++++++++++++++------
 synapse/storage/keys.py               |  1 +
 2 files changed, 28 insertions(+), 7 deletions(-)

diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py
index 5be8e66fb81d..61782ae1c011 100644
--- a/synapse/federation/federation_base.py
+++ b/synapse/federation/federation_base.py
@@ -143,11 +143,31 @@ def _check_sigs_and_hashes(self, pdus):
         def callback(_, pdu):
             with logcontext.PreserveLoggingContext(ctx):
                 if not check_event_content_hash(pdu):
-                    logger.warn(
-                        "Event content has been tampered, redacting %s: %s",
-                        pdu.event_id, pdu.get_pdu_json()
-                    )
-                    return prune_event(pdu)
+                    # let's try to distinguish between failures because the event was
+                    # redacted (which are somewhat expected) vs actual ball-tampering
+                    # incidents.
+                    #
+                    # This is just a heuristic, so we just assume that if the keys are
+                    # about the same between the redacted and received events, then the
+                    # received event was probably a redacted copy (but we then use our
+                    # *actual* redacted copy to be on the safe side.)
+                    redacted_event = prune_event(pdu)
+                    if (
+                        set(six.iterkeys(redacted_event)) == set(six.iterkeys(pdu)) and
+                        set(six.iterkeys(redacted_event.content))
+                            == set(six.iterkeys(pdu.content))
+                    ):
+                        logger.info(
+                            "Event %s seems to have been redacted; using our redacted "
+                            "copy",
+                            pdu.event_id,
+                        )
+                    else:
+                        logger.warning(
+                            "Event %s content has been tampered, redacting",
+                            pdu.event_id, pdu.get_pdu_json(),
+                        )
+                    return redacted_event
 
                 if self.spam_checker.check_event_for_spam(pdu):
                     logger.warn(
@@ -162,8 +182,8 @@ def errback(failure, pdu):
             failure.trap(SynapseError)
             with logcontext.PreserveLoggingContext(ctx):
                 logger.warn(
-                    "Signature check failed for %s",
-                    pdu.event_id,
+                    "Signature check failed for %s: %s",
+                    pdu.event_id, failure.getErrorMessage(),
                 )
             return failure
 
diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py
index f54797760041..a1331c1a6176 100644
--- a/synapse/storage/keys.py
+++ b/synapse/storage/keys.py
@@ -134,6 +134,7 @@ def store_server_verify_key(self, server_name, from_server, time_now_ms,
         """
         key_id = "%s:%s" % (verify_key.alg, verify_key.version)
 
+        # XXX fix this to not need a lock (#3819)
         def _txn(txn):
             self._simple_upsert_txn(
                 txn,

From 6febd8e8f737d28a9fec6cd2bfef0e80bea8437c Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Fri, 7 Sep 2018 21:40:57 +1000
Subject: [PATCH 577/830] version

---
 synapse/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/__init__.py b/synapse/__init__.py
index 063c30453400..65a2b894ccc1 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -17,4 +17,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.33.4rc2"
+__version__ = "0.33.4"

From b60749a1ec22269216a503ccd35aaf4454dc507d Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Fri, 7 Sep 2018 21:41:57 +1000
Subject: [PATCH 578/830] changelog

---
 CHANGES.md            | 9 +++++++++
 changelog.d/3802.misc | 1 -
 2 files changed, 9 insertions(+), 1 deletion(-)
 delete mode 100644 changelog.d/3802.misc

diff --git a/CHANGES.md b/CHANGES.md
index fecfa7feeca4..ee864c3c639c 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,12 @@
+Synapse 0.33.4 (2018-09-07)
+===========================
+
+Internal Changes
+----------------
+
+- Unignore synctl in .dockerignore to fix docker builds ([\#3802](https://github.com/matrix-org/synapse/issues/3802))
+
+
 Synapse 0.33.4rc2 (2018-09-06)
 ==============================
 
diff --git a/changelog.d/3802.misc b/changelog.d/3802.misc
deleted file mode 100644
index a00eccb28329..000000000000
--- a/changelog.d/3802.misc
+++ /dev/null
@@ -1 +0,0 @@
-Unignore synctl in .dockerignore to fix docker builds

From b8ad756bd0d7c42c7c241fa08f5e078561fddded Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 7 Sep 2018 14:20:54 +0100
Subject: [PATCH 579/830] Fix jwt import check

This handy code attempted to check that we could import jwt, but utterly failed
to check it was the right jwt.

Fixes https://github.com/matrix-org/synapse/issues/3793
---
 synapse/config/homeserver.py             | 2 +-
 synapse/config/{jwt.py => jwt_config.py} | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename synapse/config/{jwt.py => jwt_config.py} (100%)

diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py
index 2fd9c48abff6..b8d5690f2be4 100644
--- a/synapse/config/homeserver.py
+++ b/synapse/config/homeserver.py
@@ -21,7 +21,7 @@
 from .database import DatabaseConfig
 from .emailconfig import EmailConfig
 from .groups import GroupsConfig
-from .jwt import JWTConfig
+from .jwt_config import JWTConfig
 from .key import KeyConfig
 from .logger import LoggingConfig
 from .metrics import MetricsConfig
diff --git a/synapse/config/jwt.py b/synapse/config/jwt_config.py
similarity index 100%
rename from synapse/config/jwt.py
rename to synapse/config/jwt_config.py

From edda9f5cac2b25386c42c6eabf2e5a7dae159986 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 7 Sep 2018 14:23:35 +0100
Subject: [PATCH 580/830] changelog

---
 changelog.d/3824.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3824.bugfix

diff --git a/changelog.d/3824.bugfix b/changelog.d/3824.bugfix
new file mode 100644
index 000000000000..99f199dcc658
--- /dev/null
+++ b/changelog.d/3824.bugfix
@@ -0,0 +1 @@
+Fix jwt import check
\ No newline at end of file

From 1e4c7fff5f30d195071e83190bb480f18cd94142 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 7 Sep 2018 16:37:30 +0100
Subject: [PATCH 581/830] changelog

---
 changelog.d/3826.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3826.misc

diff --git a/changelog.d/3826.misc b/changelog.d/3826.misc
new file mode 100644
index 000000000000..a4d9a012f9e6
--- /dev/null
+++ b/changelog.d/3826.misc
@@ -0,0 +1 @@
+add some logging for the keyring queue

From b7e7712f07272e1612b47d9d174f0f55d0fdd247 Mon Sep 17 00:00:00 2001
From: Mathijs van Gorcum <mvgorcum@gmail.com>
Date: Mon, 10 Sep 2018 12:21:42 +0000
Subject: [PATCH 582/830] Remove build requirements after building

---
 docker/Dockerfile | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index 777976217d4f..f6da05f4efdb 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -26,8 +26,9 @@ RUN cd /synapse \
  && rm -rf \
         setup.cfg \
         setup.py \
-        synapse
-
+        synapse \
+ && apk del .nacl_deps
+ 
 VOLUME ["/data"]
 
 EXPOSE 8008/tcp 8448/tcp

From 7bc22539ffb8375f92a1b26f6b4cdcfcfa9efd9a Mon Sep 17 00:00:00 2001
From: Krombel <krombel@krombel.de>
Date: Mon, 10 Sep 2018 14:30:08 +0200
Subject: [PATCH 583/830] fix VOIP crashes under Python 3 (#3821)

---
 synapse/rest/client/v1/voip.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/synapse/rest/client/v1/voip.py b/synapse/rest/client/v1/voip.py
index 62f4c3d93ea2..53da905eeadc 100644
--- a/synapse/rest/client/v1/voip.py
+++ b/synapse/rest/client/v1/voip.py
@@ -42,7 +42,11 @@ def on_GET(self, request):
             expiry = (self.hs.get_clock().time_msec() + userLifetime) / 1000
             username = "%d:%s" % (expiry, requester.user.to_string())
 
-            mac = hmac.new(turnSecret, msg=username, digestmod=hashlib.sha1)
+            mac = hmac.new(
+                turnSecret.encode(),
+                msg=username.encode(),
+                digestmod=hashlib.sha1
+            )
             # We need to use standard padded base64 encoding here
             # encode_base64 because we need to add the standard padding to get the
             # same result as the TURN server.

From 3572a206d30ce178c9f7a2c852f43bb86238cf0b Mon Sep 17 00:00:00 2001
From: Krombel <krombel@krombel.de>
Date: Mon, 10 Sep 2018 14:33:08 +0200
Subject: [PATCH 584/830] add changelog

---
 changelog.d/3835.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3835.bugfix

diff --git a/changelog.d/3835.bugfix b/changelog.d/3835.bugfix
new file mode 100644
index 000000000000..00dbcbc8dcf5
--- /dev/null
+++ b/changelog.d/3835.bugfix
@@ -0,0 +1 @@
+fix VOIP crashes under Python 3 (#3821)

From e586916cda1f3549bd9ebab2e75d3824840c76a0 Mon Sep 17 00:00:00 2001
From: Mathijs van Gorcum <mvgorcum@gmail.com>
Date: Mon, 10 Sep 2018 14:02:42 +0000
Subject: [PATCH 585/830] Move COPY before RUN and merge RUNs

---
 docker/Dockerfile | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index f6da05f4efdb..b9f0acccc33e 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -1,5 +1,7 @@
 FROM docker.io/python:2-alpine3.8
 
+COPY . /synapse
+
 RUN apk add --no-cache --virtual .nacl_deps \
         build-base \
         libffi-dev \
@@ -9,12 +11,10 @@ RUN apk add --no-cache --virtual .nacl_deps \
         linux-headers \
         postgresql-dev \
         su-exec \
-        zlib-dev
-
-COPY . /synapse
-
+        zlib-dev \
+   
 # A wheel cache may be provided in ./cache for faster build
-RUN cd /synapse \
+ && cd /synapse \
  && pip install --upgrade \
         lxml \
         pip \

From e957428a154ed9faf8b481947e40f74cdc65b75c Mon Sep 17 00:00:00 2001
From: Mathijs van Gorcum <mvgorcum@gmail.com>
Date: Mon, 10 Sep 2018 14:53:05 +0000
Subject: [PATCH 586/830]  Newsfile

---
 changelog.d/3834.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3834.misc

diff --git a/changelog.d/3834.misc b/changelog.d/3834.misc
new file mode 100644
index 000000000000..8902f8fba7a6
--- /dev/null
+++ b/changelog.d/3834.misc
@@ -0,0 +1 @@
+Improved Dockerfile to remove build requirements after building reducing the image size.

From af10fa6536bf25630f4a290c756cd7f78e394a41 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Christian=20Gr=C3=BCnhage?=
 <jan.christian@gruenhage.xyz>
Date: Mon, 10 Sep 2018 17:35:01 +0200
Subject: [PATCH 587/830] add runtime dependencies

---
 docker/Dockerfile | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index b9f0acccc33e..20d3fe3bd891 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -2,7 +2,7 @@ FROM docker.io/python:2-alpine3.8
 
 COPY . /synapse
 
-RUN apk add --no-cache --virtual .nacl_deps \
+RUN apk add --no-cache --virtual .build_deps \
         build-base \
         libffi-dev \
         libjpeg-turbo-dev \
@@ -10,11 +10,16 @@ RUN apk add --no-cache --virtual .nacl_deps \
         libxslt-dev \
         linux-headers \
         postgresql-dev \
-        su-exec \
         zlib-dev \
-   
-# A wheel cache may be provided in ./cache for faster build
  && cd /synapse \
+ && apk add --no-cache --virtual .runtime_deps \
+ 	libffi \
+        libjpeg-turbo \
+	libressl \
+	libxslt \
+	libpq \
+	zlib \
+	su-exec \
  && pip install --upgrade \
         lxml \
         pip \
@@ -27,7 +32,7 @@ RUN apk add --no-cache --virtual .nacl_deps \
         setup.cfg \
         setup.py \
         synapse \
- && apk del .nacl_deps
+ && apk del .build_deps
  
 VOLUME ["/data"]
 

From 9e05c8d3090a956054b5b39347359655236caaef Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 11 Sep 2018 10:42:10 +0100
Subject: [PATCH 588/830] Change the manhole SSH key to have more bits

Newer versions of openssh client refuse to connect to the old key due to
its length.
---
 synapse/util/manhole.py | 44 +++++++++++++++++++++++++++++------------
 1 file changed, 31 insertions(+), 13 deletions(-)

diff --git a/synapse/util/manhole.py b/synapse/util/manhole.py
index 14be3c7396a4..8d0f2a891811 100644
--- a/synapse/util/manhole.py
+++ b/synapse/util/manhole.py
@@ -19,22 +19,40 @@
 from twisted.cred import checkers, portal
 
 PUBLIC_KEY = (
-    "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAGEArzJx8OYOnJmzf4tfBEvLi8DVPrJ3/c9k2I/Az"
-    "64fxjHf9imyRJbixtQhlH9lfNjUIx+4LmrJH5QNRsFporcHDKOTwTTYLh5KmRpslkYHRivcJS"
-    "kbh/C+BR3utDS555mV"
+    "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDHhGATaW4KhE23+7nrH4jFx3yLq9OjaEs5"
+    "XALqeK+7385NlLja3DE/DO9mGhnd9+bAy39EKT3sTV6+WXQ4yD0TvEEyUEMtjWkSEm6U32+C"
+    "DaS3TW/vPBUMeJQwq+Ydcif1UlnpXrDDTamD0AU9VaEvHq+3HAkipqn0TGpKON6aqk4vauDx"
+    "oXSsV5TXBVrxP/y7HpMOpU4GUWsaaacBTKKNnUaQB4UflvydaPJUuwdaCUJGTMjbhWrjVfK+"
+    "jslseSPxU6XvrkZMyCr4znxvuDxjMk1RGIdO7v+rbBMLEgqtSMNqJbYeVCnj2CFgc3fcTcld"
+    "X2uOJDrJb/WRlHulthCh"
 )
 
 PRIVATE_KEY = """-----BEGIN RSA PRIVATE KEY-----
-MIIByAIBAAJhAK8ycfDmDpyZs3+LXwRLy4vA1T6yd/3PZNiPwM+uH8Yx3/YpskSW
-4sbUIZR/ZXzY1CMfuC5qyR+UDUbBaaK3Bwyjk8E02C4eSpkabJZGB0Yr3CUpG4fw
-vgUd7rQ0ueeZlQIBIwJgbh+1VZfr7WftK5lu7MHtqE1S1vPWZQYE3+VUn8yJADyb
-Z4fsZaCrzW9lkIqXkE3GIY+ojdhZhkO1gbG0118sIgphwSWKRxK0mvh6ERxKqIt1
-xJEJO74EykXZV4oNJ8sjAjEA3J9r2ZghVhGN6V8DnQrTk24Td0E8hU8AcP0FVP+8
-PQm/g/aXf2QQkQT+omdHVEJrAjEAy0pL0EBH6EVS98evDCBtQw22OZT52qXlAwZ2
-gyTriKFVoqjeEjt3SZKKqXHSApP/AjBLpF99zcJJZRq2abgYlf9lv1chkrWqDHUu
-DZttmYJeEfiFBBavVYIF1dOlZT0G8jMCMBc7sOSZodFnAiryP+Qg9otSBjJ3bQML
-pSTqy7c3a2AScC/YyOwkDaICHnnD3XyjMwIxALRzl0tQEKMXs6hH8ToUdlLROCrP
-EhQ0wahUTCk1gKA4uPD6TMTChavbh4K63OvbKg==
+MIIEpQIBAAKCAQEAx4RgE2luCoRNt/u56x+Ixcd8i6vTo2hLOVwC6nivu9/OTZS4
+2twxPwzvZhoZ3ffmwMt/RCk97E1evll0OMg9E7xBMlBDLY1pEhJulN9vgg2kt01v
+7zwVDHiUMKvmHXIn9VJZ6V6ww02pg9AFPVWhLx6vtxwJIqap9ExqSjjemqpOL2rg
+8aF0rFeU1wVa8T/8ux6TDqVOBlFrGmmnAUyijZ1GkAeFH5b8nWjyVLsHWglCRkzI
+24Vq41Xyvo7JbHkj8VOl765GTMgq+M58b7g8YzJNURiHTu7/q2wTCxIKrUjDaiW2
+HlQp49ghYHN33E3JXV9rjiQ6yW/1kZR7pbYQoQIDAQABAoIBAQC8KJ0q8Wzzwh5B
+esa1dQHZ8+4DEsL/Amae66VcVwD0X3cCN1W2IZ7X5W0Ij2kBqr8V51RYhcR+S+Ek
+BtzSiBUBvbKGrqcMGKaUgomDIMzai99hd0gvCCyZnEW1OQhFkNkaRNXCfqiZJ27M
+fqvSUiU2eOwh9fCvmxoA6Of8o3FbzcJ+1GMcobWRllDtLmj6lgVbDzuA+0jC5daB
+9Tj1pBzu3wn3ufxiS+gBnJ+7NcXH3E73lqCcPa2ufbZ1haxfiGCnRIhFXuQDgxFX
+vKdEfDgtvas6r1ahGbc+b/q8E8fZT7cABuIU4yfOORK+MhpyWbvoyyzuVGKj3PKt
+KSPJu5CZAoGBAOkoJfAVyYteqKcmGTanGqQnAY43CaYf6GdSPX/jg+JmKZg0zqMC
+jWZUtPb93i+jnOInbrnuHOiHAxI8wmhEPed28H2lC/LU8PzlqFkZXKFZ4vLOhhRB
+/HeHCFIDosPFlohWi3b+GAjD7sXgnIuGmnXWe2ea/TS3yersifDEoKKjAoGBANsQ
+gJX2cJv1c3jhdgcs8vAt5zIOKcCLTOr/QPmVf/kxjNgndswcKHwsxE/voTO9q+TF
+v/6yCSTxAdjuKz1oIYWgi/dZo82bBKWxNRpgrGviU3/zwxiHlyIXUhzQu78q3VS/
+7S1XVbc7qMV++XkYKHPVD+nVG/gGzFxumX7MLXfrAoGBAJit9cn2OnjNj9uFE1W6
+r7N254ndeLAUjPe73xH0RtTm2a4WRopwjW/JYIetTuYbWgyujc+robqTTuuOZjAp
+H/CG7o0Ym251CypQqaFO/l2aowclPp/dZhpPjp9GSjuxFBZLtiBB3DNBOwbRQzIK
+/vLTdRQvZkgzYkI4i0vjNt3JAoGBANP8HSKBLymMlShlrSx2b8TB9tc2Y2riohVJ
+2ttqs0M2kt/dGJWdrgOz4mikL+983Olt/0P9juHDoxEEMK2kpcPEv40lnmBpYU7h
+s8yJvnBLvJe2EJYdJ8AipyAhUX1FgpbvfxmASP8eaUxsegeXvBWTGWojAoS6N2o+
+0KSl+l3vAoGAFqm0gO9f/Q1Se60YQd4l2PZeMnJFv0slpgHHUwegmd6wJhOD7zJ1
+CkZcXwiv7Nog7AI9qKJEUXLjoqL+vJskBzSOqU3tcd670YQMi1aXSXJqYE202K7o
+EddTrx3TNpr1D5m/f+6mnXWrc8u9y1+GNx9yz889xMjIBTBI9KqaaOs=
 -----END RSA PRIVATE KEY-----"""
 
 

From 9a68778ac20f35ccd3aeb8b1392518a1f802955b Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 11 Sep 2018 10:44:40 +0100
Subject: [PATCH 589/830] Newsfile

---
 changelog.d/3841.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3841.bugfix

diff --git a/changelog.d/3841.bugfix b/changelog.d/3841.bugfix
new file mode 100644
index 000000000000..2a48a7dd66ee
--- /dev/null
+++ b/changelog.d/3841.bugfix
@@ -0,0 +1 @@
+Fix manhole so that it works with latest openssh clients

From b041115415191f8353177f7e0c1beec81be0c921 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@arasphere.net>
Date: Wed, 12 Sep 2018 00:50:39 +0100
Subject: [PATCH 590/830] Speed up lazy loading (#3827)

* speed up room summaries by pulling their data from room_memberships rather than room state
* disable LL for incr syncs, and log incr sync stats  (#3840)
---
 changelog.d/3827.misc         |   1 +
 changelog.d/3840.misc         |   1 +
 synapse/handlers/sync.py      | 118 +++++++++++++++++++++++++---------
 synapse/storage/events.py     |   4 ++
 synapse/storage/roommember.py |  65 +++++++++++++++++++
 5 files changed, 159 insertions(+), 30 deletions(-)
 create mode 100644 changelog.d/3827.misc
 create mode 100644 changelog.d/3840.misc

diff --git a/changelog.d/3827.misc b/changelog.d/3827.misc
new file mode 100644
index 000000000000..bc294706cf56
--- /dev/null
+++ b/changelog.d/3827.misc
@@ -0,0 +1 @@
+speed up lazy loading by 2-3x
diff --git a/changelog.d/3840.misc b/changelog.d/3840.misc
new file mode 100644
index 000000000000..b9585ae9bee4
--- /dev/null
+++ b/changelog.d/3840.misc
@@ -0,0 +1 @@
+Disable lazy loading for incremental syncs for now
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 7eed2fcc9bb4..23983a51ab18 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -24,6 +24,7 @@
 
 from synapse.api.constants import EventTypes, Membership
 from synapse.push.clientformat import format_push_rules_for_user
+from synapse.storage.roommember import MemberSummary
 from synapse.types import RoomStreamToken
 from synapse.util.async_helpers import concurrently_execute
 from synapse.util.caches.expiringcache import ExpiringCache
@@ -525,6 +526,8 @@ def compute_summary(self, room_id, sync_config, batch, state, now_token):
              A deferred dict describing the room summary
         """
 
+        # FIXME: we could/should get this from room_stats when matthew/stats lands
+
         # FIXME: this promulgates https://github.com/matrix-org/synapse/issues/3305
         last_events, _ = yield self.store.get_recent_event_ids_for_room(
             room_id, end_token=now_token.room_key, limit=1,
@@ -537,44 +540,54 @@ def compute_summary(self, room_id, sync_config, batch, state, now_token):
         last_event = last_events[-1]
         state_ids = yield self.store.get_state_ids_for_event(
             last_event.event_id, [
-                (EventTypes.Member, None),
                 (EventTypes.Name, ''),
                 (EventTypes.CanonicalAlias, ''),
             ]
         )
 
-        member_ids = {
-            state_key: event_id
-            for (t, state_key), event_id in iteritems(state_ids)
-            if t == EventTypes.Member
-        }
+        # this is heavily cached, thus: fast.
+        details = yield self.store.get_room_summary(room_id)
+
         name_id = state_ids.get((EventTypes.Name, ''))
         canonical_alias_id = state_ids.get((EventTypes.CanonicalAlias, ''))
 
         summary = {}
-
-        # FIXME: it feels very heavy to load up every single membership event
-        # just to calculate the counts.
-        member_events = yield self.store.get_events(member_ids.values())
-
-        joined_user_ids = []
-        invited_user_ids = []
-
-        for ev in member_events.values():
-            if ev.content.get("membership") == Membership.JOIN:
-                joined_user_ids.append(ev.state_key)
-            elif ev.content.get("membership") == Membership.INVITE:
-                invited_user_ids.append(ev.state_key)
+        empty_ms = MemberSummary([], 0)
 
         # TODO: only send these when they change.
-        summary["m.joined_member_count"] = len(joined_user_ids)
-        summary["m.invited_member_count"] = len(invited_user_ids)
+        summary["m.joined_member_count"] = (
+            details.get(Membership.JOIN, empty_ms).count
+        )
+        summary["m.invited_member_count"] = (
+            details.get(Membership.INVITE, empty_ms).count
+        )
 
         if name_id or canonical_alias_id:
             defer.returnValue(summary)
 
-        # FIXME: order by stream ordering, not alphabetic
+        joined_user_ids = [
+            r[0] for r in details.get(Membership.JOIN, empty_ms).members
+        ]
+        invited_user_ids = [
+            r[0] for r in details.get(Membership.INVITE, empty_ms).members
+        ]
+        gone_user_ids = (
+            [r[0] for r in details.get(Membership.LEAVE, empty_ms).members] +
+            [r[0] for r in details.get(Membership.BAN, empty_ms).members]
+        )
+
+        # FIXME: only build up a member_ids list for our heroes
+        member_ids = {}
+        for membership in (
+            Membership.JOIN,
+            Membership.INVITE,
+            Membership.LEAVE,
+            Membership.BAN
+        ):
+            for user_id, event_id in details.get(membership, empty_ms).members:
+                member_ids[user_id] = event_id
 
+        # FIXME: order by stream ordering rather than as returned by SQL
         me = sync_config.user.to_string()
         if (joined_user_ids or invited_user_ids):
             summary['m.heroes'] = sorted(
@@ -586,7 +599,11 @@ def compute_summary(self, room_id, sync_config, batch, state, now_token):
             )[0:5]
         else:
             summary['m.heroes'] = sorted(
-                [user_id for user_id in member_ids.keys() if user_id != me]
+                [
+                    user_id
+                    for user_id in gone_user_ids
+                    if user_id != me
+                ]
             )[0:5]
 
         if not sync_config.filter_collection.lazy_load_members():
@@ -719,6 +736,26 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                     lazy_load_members=lazy_load_members,
                 )
             elif batch.limited:
+                state_at_timeline_start = yield self.store.get_state_ids_for_event(
+                    batch.events[0].event_id, types=types,
+                    filtered_types=filtered_types,
+                )
+
+                # for now, we disable LL for gappy syncs - see
+                # https://github.com/vector-im/riot-web/issues/7211#issuecomment-419976346
+                # N.B. this slows down incr syncs as we are now processing way
+                # more state in the server than if we were LLing.
+                #
+                # We still have to filter timeline_start to LL entries (above) in order
+                # for _calculate_state's LL logic to work, as we have to include LL
+                # members for timeline senders in case they weren't loaded in the initial
+                # sync.  We do this by (counterintuitively) by filtering timeline_start
+                # members to just be ones which were timeline senders, which then ensures
+                # all of the rest get included in the state block (if we need to know
+                # about them).
+                types = None
+                filtered_types = None
+
                 state_at_previous_sync = yield self.get_state_at(
                     room_id, stream_position=since_token, types=types,
                     filtered_types=filtered_types,
@@ -729,24 +766,21 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                     filtered_types=filtered_types,
                 )
 
-                state_at_timeline_start = yield self.store.get_state_ids_for_event(
-                    batch.events[0].event_id, types=types,
-                    filtered_types=filtered_types,
-                )
-
                 state_ids = _calculate_state(
                     timeline_contains=timeline_state,
                     timeline_start=state_at_timeline_start,
                     previous=state_at_previous_sync,
                     current=current_state_ids,
+                    # we have to include LL members in case LL initial sync missed them
                     lazy_load_members=lazy_load_members,
                 )
             else:
                 state_ids = {}
                 if lazy_load_members:
                     if types:
-                        # We're returning an incremental sync, with no "gap" since
-                        # the previous sync, so normally there would be no state to return
+                        # We're returning an incremental sync, with no
+                        # "gap" since the previous sync, so normally there would be
+                        # no state to return.
                         # But we're lazy-loading, so the client might need some more
                         # member events to understand the events in this timeline.
                         # So we fish out all the member events corresponding to the
@@ -1616,10 +1650,24 @@ def _generate_room_entry(self, sync_result_builder, ignored_users,
         )
 
         summary = {}
+
+        # we include a summary in room responses when we're lazy loading
+        # members (as the client otherwise doesn't have enough info to form
+        # the name itself).
         if (
             sync_config.filter_collection.lazy_load_members() and
             (
+                # we recalulate the summary:
+                #   if there are membership changes in the timeline, or
+                #   if membership has changed during a gappy sync, or
+                #   if this is an initial sync.
                 any(ev.type == EventTypes.Member for ev in batch.events) or
+                (
+                    # XXX: this may include false positives in the form of LL
+                    # members which have snuck into state
+                    batch.limited and
+                    any(t == EventTypes.Member for (t, k) in state)
+                ) or
                 since_token is None
             )
         ):
@@ -1649,6 +1697,16 @@ def _generate_room_entry(self, sync_result_builder, ignored_users,
                     unread_notifications["highlight_count"] = notifs["highlight_count"]
 
                 sync_result_builder.joined.append(room_sync)
+
+            if batch.limited:
+                user_id = sync_result_builder.sync_config.user.to_string()
+                logger.info(
+                    "Incremental syncing room %s for user %s with %d state events" % (
+                        room_id,
+                        user_id,
+                        len(state),
+                    )
+                )
         elif room_builder.rtype == "archived":
             room_sync = ArchivedSyncResult(
                 room_id=room_id,
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 8bf87f38f7d6..30ff87a4c464 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -929,6 +929,10 @@ def _update_current_state_txn(self, txn, state_delta_by_room, max_stream_order):
                     txn, self.get_users_in_room, (room_id,)
                 )
 
+                self._invalidate_cache_and_stream(
+                    txn, self.get_room_summary, (room_id,)
+                )
+
                 self._invalidate_cache_and_stream(
                     txn, self.get_current_state_ids, (room_id,)
                 )
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index 9b4e6d6aa820..0707f9a86aa0 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -51,6 +51,12 @@
     "ProfileInfo", ("avatar_url", "display_name")
 )
 
+# "members" points to a truncated list of (user_id, event_id) tuples for users of
+# a given membership type, suitable for use in calculating heroes for a room.
+# "count" points to the total numberr of users of a given membership type.
+MemberSummary = namedtuple(
+    "MemberSummary", ("members", "count")
+)
 
 _MEMBERSHIP_PROFILE_UPDATE_NAME = "room_membership_profile_update"
 
@@ -82,6 +88,65 @@ def f(txn):
             return [to_ascii(r[0]) for r in txn]
         return self.runInteraction("get_users_in_room", f)
 
+    @cached(max_entries=100000)
+    def get_room_summary(self, room_id):
+        """ Get the details of a room roughly suitable for use by the room
+        summary extension to /sync. Useful when lazy loading room members.
+        Args:
+            room_id (str): The room ID to query
+        Returns:
+            Deferred[dict[str, MemberSummary]:
+                dict of membership states, pointing to a MemberSummary named tuple.
+        """
+
+        def _get_room_summary_txn(txn):
+            # first get counts.
+            # We do this all in one transaction to keep the cache small.
+            # FIXME: get rid of this when we have room_stats
+            sql = """
+                SELECT count(*), m.membership FROM room_memberships as m
+                 INNER JOIN current_state_events as c
+                 ON m.event_id = c.event_id
+                 AND m.room_id = c.room_id
+                 AND m.user_id = c.state_key
+                 WHERE c.type = 'm.room.member' AND c.room_id = ?
+                 GROUP BY m.membership
+            """
+
+            txn.execute(sql, (room_id,))
+            res = {}
+            for count, membership in txn:
+                summary = res.setdefault(to_ascii(membership), MemberSummary([], count))
+
+            # we order by membership and then fairly arbitrarily by event_id so
+            # heroes are consistent
+            sql = """
+                SELECT m.user_id, m.membership, m.event_id
+                FROM room_memberships as m
+                 INNER JOIN current_state_events as c
+                 ON m.event_id = c.event_id
+                 AND m.room_id = c.room_id
+                 AND m.user_id = c.state_key
+                 WHERE c.type = 'm.room.member' AND c.room_id = ?
+                 ORDER BY
+                    CASE m.membership WHEN ? THEN 1 WHEN ? THEN 2 ELSE 3 END ASC,
+                    m.event_id ASC
+                 LIMIT ?
+            """
+
+            # 6 is 5 (number of heroes) plus 1, in case one of them is the calling user.
+            txn.execute(sql, (room_id, Membership.JOIN, Membership.INVITE, 6))
+            for user_id, membership, event_id in txn:
+                summary = res[to_ascii(membership)]
+                # we will always have a summary for this membership type at this
+                # point given the summary currently contains the counts.
+                members = summary.members
+                members.append((to_ascii(user_id), to_ascii(event_id)))
+
+            return res
+
+        return self.runInteraction("get_room_summary", _get_room_summary_txn)
+
     @cached()
     def get_invited_rooms_for_user(self, user_id):
         """ Get all the rooms the user is invited to

From 4084a774a89f0d02406eebda8279c2b8aab89812 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 12 Sep 2018 09:59:15 +0100
Subject: [PATCH 591/830] Timeout reading body for outbound HTTP requests

---
 synapse/http/matrixfederationclient.py | 52 ++++++++++++++++++++++----
 1 file changed, 45 insertions(+), 7 deletions(-)

diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 6a1fc8ca553f..f9a1fbf95d76 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -280,7 +280,10 @@ def _request(self, destination, method, path,
                 # :'(
                 # Update transactions table?
                 with logcontext.PreserveLoggingContext():
-                    body = yield treq.content(response)
+                    body = yield self._timeout_deferred(
+                        treq.content(response),
+                        timeout,
+                    )
                 raise HttpResponseException(
                     response.code, response.phrase, body
                 )
@@ -394,7 +397,10 @@ def put_json(self, destination, path, args={}, data={},
             check_content_type_is_json(response.headers)
 
         with logcontext.PreserveLoggingContext():
-            body = yield treq.json_content(response)
+            body = yield self._timeout_deferred(
+                treq.json_content(response),
+                timeout,
+            )
         defer.returnValue(body)
 
     @defer.inlineCallbacks
@@ -444,7 +450,10 @@ def post_json(self, destination, path, data={}, long_retries=False,
             check_content_type_is_json(response.headers)
 
         with logcontext.PreserveLoggingContext():
-            body = yield treq.json_content(response)
+            body = yield self._timeout_deferred(
+                treq.json_content(response),
+                timeout,
+            )
 
         defer.returnValue(body)
 
@@ -496,7 +505,10 @@ def get_json(self, destination, path, args=None, retry_on_dns_fail=True,
             check_content_type_is_json(response.headers)
 
         with logcontext.PreserveLoggingContext():
-            body = yield treq.json_content(response)
+            body = yield self._timeout_deferred(
+                treq.json_content(response),
+                timeout,
+            )
 
         defer.returnValue(body)
 
@@ -543,7 +555,10 @@ def delete_json(self, destination, path, long_retries=False,
             check_content_type_is_json(response.headers)
 
         with logcontext.PreserveLoggingContext():
-            body = yield treq.json_content(response)
+            body = yield self._timeout_deferred(
+                treq.json_content(response),
+                timeout,
+            )
 
         defer.returnValue(body)
 
@@ -585,8 +600,10 @@ def get_file(self, destination, path, output_stream, args={},
 
         try:
             with logcontext.PreserveLoggingContext():
-                length = yield _readBodyToFile(
-                    response, output_stream, max_size
+                length = yield self._timeout_deferred(
+                    _readBodyToFile(
+                        response, output_stream, max_size
+                    ),
                 )
         except Exception:
             logger.exception("Failed to download body")
@@ -594,6 +611,27 @@ def get_file(self, destination, path, output_stream, args={},
 
         defer.returnValue((length, headers))
 
+    def _timeout_deferred(self, deferred, timeout_ms=None):
+        """Times the deferred out after `timeout_ms` ms
+
+        Args:
+            deferred (Deferred)
+            timeout_ms (int|None): Timeout in milliseconds. If None defaults
+                to 60 seconds.
+
+        Returns:
+            Deferred
+        """
+
+        add_timeout_to_deferred(
+            deferred,
+            timeout_ms / 1000. if timeout_ms else 60,
+            self.hs.get_reactor(),
+            cancelled_to_request_timed_out_error,
+        )
+
+        return deferred
+
 
 class _ReadBodyToFileProtocol(protocol.Protocol):
     def __init__(self, stream, deferred, max_size):

From 649c647955dee037e9f7d0d0d81341cac020a901 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 12 Sep 2018 10:10:32 +0100
Subject: [PATCH 592/830] Newsfile

---
 changelog.d/3845.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3845.bugfix

diff --git a/changelog.d/3845.bugfix b/changelog.d/3845.bugfix
new file mode 100644
index 000000000000..5b7e8f193498
--- /dev/null
+++ b/changelog.d/3845.bugfix
@@ -0,0 +1 @@
+Fix outbound requests occasionally wedging, which can result in federation breaking between servers.

From 8fd93b5eeaeddce16e0b510741dc5d4768cbc78d Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Wed, 12 Sep 2018 20:16:31 +1000
Subject: [PATCH 593/830] Port crypto/ to Python 3 (#3822)

---
 changelog.d/3822.misc             | 1 +
 synapse/crypto/context_factory.py | 2 +-
 synapse/crypto/keyclient.py       | 8 +++++++-
 synapse/crypto/keyring.py         | 9 +++++----
 4 files changed, 14 insertions(+), 6 deletions(-)
 create mode 100644 changelog.d/3822.misc

diff --git a/changelog.d/3822.misc b/changelog.d/3822.misc
new file mode 100644
index 000000000000..5250f318969a
--- /dev/null
+++ b/changelog.d/3822.misc
@@ -0,0 +1 @@
+crypto/ is now ported to Python 3.
diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py
index 1a391adec198..02b76dfcfb8f 100644
--- a/synapse/crypto/context_factory.py
+++ b/synapse/crypto/context_factory.py
@@ -123,6 +123,6 @@ def __init__(self, config):
 
     def get_options(self, host):
         return ClientTLSOptions(
-            host.decode('utf-8'),
+            host,
             CertificateOptions(verify=False).getContext()
         )
diff --git a/synapse/crypto/keyclient.py b/synapse/crypto/keyclient.py
index e94400b8e26f..57d4665e841c 100644
--- a/synapse/crypto/keyclient.py
+++ b/synapse/crypto/keyclient.py
@@ -50,7 +50,7 @@ def fetch_server_key(server_name, tls_client_options_factory, path=KEY_API_V1):
                 defer.returnValue((server_response, server_certificate))
         except SynapseKeyClientError as e:
             logger.warn("Error getting key for %r: %s", server_name, e)
-            if e.status.startswith("4"):
+            if e.status.startswith(b"4"):
                 # Don't retry for 4xx responses.
                 raise IOError("Cannot get key for %r" % server_name)
         except (ConnectError, DomainError) as e:
@@ -82,6 +82,12 @@ def connectionMade(self):
         self._peer = self.transport.getPeer()
         logger.debug("Connected to %s", self._peer)
 
+        if not isinstance(self.path, bytes):
+            self.path = self.path.encode('ascii')
+
+        if not isinstance(self.host, bytes):
+            self.host = self.host.encode('ascii')
+
         self.sendCommand(b"GET", self.path)
         if self.host:
             self.sendHeader(b"Host", self.host)
diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py
index 30e2742102d6..9d497abf179a 100644
--- a/synapse/crypto/keyring.py
+++ b/synapse/crypto/keyring.py
@@ -16,9 +16,10 @@
 
 import hashlib
 import logging
-import urllib
 from collections import namedtuple
 
+from six.moves import urllib
+
 from signedjson.key import (
     decode_verify_key_bytes,
     encode_verify_key_base64,
@@ -432,7 +433,7 @@ def get_server_verify_key_v2_indirect(self, server_names_and_key_ids,
         # an incoming request.
         query_response = yield self.client.post_json(
             destination=perspective_name,
-            path=b"/_matrix/key/v2/query",
+            path="/_matrix/key/v2/query",
             data={
                 u"server_keys": {
                     server_name: {
@@ -513,8 +514,8 @@ def get_server_verify_key_v2_direct(self, server_name, key_ids):
 
             (response, tls_certificate) = yield fetch_server_key(
                 server_name, self.hs.tls_client_options_factory,
-                path=(b"/_matrix/key/v2/server/%s" % (
-                    urllib.quote(requested_key_id),
+                path=("/_matrix/key/v2/server/%s" % (
+                    urllib.parse.quote(requested_key_id),
                 )).encode("ascii"),
             )
 

From 02aa41809bd48cc1dd02da3634f02f5eae71b41c Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Wed, 12 Sep 2018 20:41:31 +1000
Subject: [PATCH 594/830] Port rest/ to Python 3 (#3823)

---
 changelog.d/3823.misc                         |  1 +
 synapse/rest/client/v1/admin.py               |  9 ++--
 synapse/rest/client/v1/events.py              | 12 ++---
 synapse/rest/client/v1/initial_sync.py        |  2 +-
 synapse/rest/client/v1/login.py               | 44 +++++++++----------
 synapse/rest/client/v1/push_rule.py           | 24 +++++-----
 synapse/rest/client/v1/pusher.py              |  4 +-
 synapse/rest/client/v1/room.py                | 14 +++---
 synapse/rest/client/v2_alpha/sync.py          |  2 +-
 synapse/rest/client/v2_alpha/thirdparty.py    |  4 +-
 synapse/rest/key/v1/server_key_resource.py    |  2 +-
 synapse/rest/key/v2/__init__.py               |  4 +-
 synapse/rest/key/v2/remote_key_resource.py    |  6 ++-
 synapse/rest/media/v0/content_repository.py   |  4 +-
 synapse/rest/media/v1/_base.py                | 30 ++++++-------
 synapse/rest/media/v1/download_resource.py    | 12 ++---
 synapse/rest/media/v1/media_repository.py     | 29 +++++++-----
 synapse/rest/media/v1/preview_url_resource.py | 10 ++---
 18 files changed, 113 insertions(+), 100 deletions(-)
 create mode 100644 changelog.d/3823.misc

diff --git a/changelog.d/3823.misc b/changelog.d/3823.misc
new file mode 100644
index 000000000000..0da491ddaa0f
--- /dev/null
+++ b/changelog.d/3823.misc
@@ -0,0 +1 @@
+rest/ is now ported to Python 3.
diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py
index ad536ab570f5..41534b8c2aa4 100644
--- a/synapse/rest/client/v1/admin.py
+++ b/synapse/rest/client/v1/admin.py
@@ -101,7 +101,7 @@ def on_GET(self, request):
 
         nonce = self.hs.get_secrets().token_hex(64)
         self.nonces[nonce] = int(self.reactor.seconds())
-        return (200, {"nonce": nonce.encode('ascii')})
+        return (200, {"nonce": nonce})
 
     @defer.inlineCallbacks
     def on_POST(self, request):
@@ -164,7 +164,7 @@ def on_POST(self, request):
             key=self.hs.config.registration_shared_secret.encode(),
             digestmod=hashlib.sha1,
         )
-        want_mac.update(nonce)
+        want_mac.update(nonce.encode('utf8'))
         want_mac.update(b"\x00")
         want_mac.update(username)
         want_mac.update(b"\x00")
@@ -173,7 +173,10 @@ def on_POST(self, request):
         want_mac.update(b"admin" if admin else b"notadmin")
         want_mac = want_mac.hexdigest()
 
-        if not hmac.compare_digest(want_mac, got_mac.encode('ascii')):
+        if not hmac.compare_digest(
+                want_mac.encode('ascii'),
+                got_mac.encode('ascii')
+        ):
             raise SynapseError(403, "HMAC incorrect")
 
         # Reuse the parts of RegisterRestServlet to reduce code duplication
diff --git a/synapse/rest/client/v1/events.py b/synapse/rest/client/v1/events.py
index 0f3a2e8b5104..cd9b3bdbd123 100644
--- a/synapse/rest/client/v1/events.py
+++ b/synapse/rest/client/v1/events.py
@@ -45,20 +45,20 @@ def on_GET(self, request):
         is_guest = requester.is_guest
         room_id = None
         if is_guest:
-            if "room_id" not in request.args:
+            if b"room_id" not in request.args:
                 raise SynapseError(400, "Guest users must specify room_id param")
-        if "room_id" in request.args:
-            room_id = request.args["room_id"][0]
+        if b"room_id" in request.args:
+            room_id = request.args[b"room_id"][0].decode('ascii')
 
         pagin_config = PaginationConfig.from_request(request)
         timeout = EventStreamRestServlet.DEFAULT_LONGPOLL_TIME_MS
-        if "timeout" in request.args:
+        if b"timeout" in request.args:
             try:
-                timeout = int(request.args["timeout"][0])
+                timeout = int(request.args[b"timeout"][0])
             except ValueError:
                 raise SynapseError(400, "timeout must be in milliseconds.")
 
-        as_client_event = "raw" not in request.args
+        as_client_event = b"raw" not in request.args
 
         chunk = yield self.event_stream_handler.get_stream(
             requester.user.to_string(),
diff --git a/synapse/rest/client/v1/initial_sync.py b/synapse/rest/client/v1/initial_sync.py
index fd5f85b53ee6..3ead75cb77ce 100644
--- a/synapse/rest/client/v1/initial_sync.py
+++ b/synapse/rest/client/v1/initial_sync.py
@@ -32,7 +32,7 @@ def __init__(self, hs):
     @defer.inlineCallbacks
     def on_GET(self, request):
         requester = yield self.auth.get_user_by_req(request)
-        as_client_event = "raw" not in request.args
+        as_client_event = b"raw" not in request.args
         pagination_config = PaginationConfig.from_request(request)
         include_archived = parse_boolean(request, "archived", default=False)
         content = yield self.initial_sync_handler.snapshot_all_rooms(
diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py
index cb85fa143678..0010699d3126 100644
--- a/synapse/rest/client/v1/login.py
+++ b/synapse/rest/client/v1/login.py
@@ -14,10 +14,9 @@
 # limitations under the License.
 
 import logging
-import urllib
 import xml.etree.ElementTree as ET
 
-from six.moves.urllib import parse as urlparse
+from six.moves import urllib
 
 from canonicaljson import json
 from saml2 import BINDING_HTTP_POST, config
@@ -134,7 +133,7 @@ def on_POST(self, request):
                                        LoginRestServlet.SAML2_TYPE):
                 relay_state = ""
                 if "relay_state" in login_submission:
-                    relay_state = "&RelayState=" + urllib.quote(
+                    relay_state = "&RelayState=" + urllib.parse.quote(
                                   login_submission["relay_state"])
                 result = {
                     "uri": "%s%s" % (self.idp_redirect_url, relay_state)
@@ -366,7 +365,7 @@ def on_POST(self, request):
             (user_id, token) = yield handler.register_saml2(username)
             # Forward to the RelayState callback along with ava
             if 'RelayState' in request.args:
-                request.redirect(urllib.unquote(
+                request.redirect(urllib.parse.unquote(
                                  request.args['RelayState'][0]) +
                                  '?status=authenticated&access_token=' +
                                  token + '&user_id=' + user_id + '&ava=' +
@@ -377,7 +376,7 @@ def on_POST(self, request):
                                      "user_id": user_id, "token": token,
                                      "ava": saml2_auth.ava}))
         elif 'RelayState' in request.args:
-            request.redirect(urllib.unquote(
+            request.redirect(urllib.parse.unquote(
                              request.args['RelayState'][0]) +
                              '?status=not_authenticated')
             finish_request(request)
@@ -390,21 +389,22 @@ class CasRedirectServlet(ClientV1RestServlet):
 
     def __init__(self, hs):
         super(CasRedirectServlet, self).__init__(hs)
-        self.cas_server_url = hs.config.cas_server_url
-        self.cas_service_url = hs.config.cas_service_url
+        self.cas_server_url = hs.config.cas_server_url.encode('ascii')
+        self.cas_service_url = hs.config.cas_service_url.encode('ascii')
 
     def on_GET(self, request):
         args = request.args
-        if "redirectUrl" not in args:
+        if b"redirectUrl" not in args:
             return (400, "Redirect URL not specified for CAS auth")
-        client_redirect_url_param = urllib.urlencode({
-            "redirectUrl": args["redirectUrl"][0]
-        })
-        hs_redirect_url = self.cas_service_url + "/_matrix/client/api/v1/login/cas/ticket"
-        service_param = urllib.urlencode({
-            "service": "%s?%s" % (hs_redirect_url, client_redirect_url_param)
-        })
-        request.redirect("%s/login?%s" % (self.cas_server_url, service_param))
+        client_redirect_url_param = urllib.parse.urlencode({
+            b"redirectUrl": args[b"redirectUrl"][0]
+        }).encode('ascii')
+        hs_redirect_url = (self.cas_service_url +
+                           b"/_matrix/client/api/v1/login/cas/ticket")
+        service_param = urllib.parse.urlencode({
+            b"service": b"%s?%s" % (hs_redirect_url, client_redirect_url_param)
+        }).encode('ascii')
+        request.redirect(b"%s/login?%s" % (self.cas_server_url, service_param))
         finish_request(request)
 
 
@@ -422,11 +422,11 @@ def __init__(self, hs):
 
     @defer.inlineCallbacks
     def on_GET(self, request):
-        client_redirect_url = request.args["redirectUrl"][0]
+        client_redirect_url = request.args[b"redirectUrl"][0]
         http_client = self.hs.get_simple_http_client()
         uri = self.cas_server_url + "/proxyValidate"
         args = {
-            "ticket": request.args["ticket"],
+            "ticket": request.args[b"ticket"][0].decode('ascii'),
             "service": self.cas_service_url
         }
         try:
@@ -471,11 +471,11 @@ def handle_cas_response(self, request, cas_response_body, client_redirect_url):
         finish_request(request)
 
     def add_login_token_to_redirect_url(self, url, token):
-        url_parts = list(urlparse.urlparse(url))
-        query = dict(urlparse.parse_qsl(url_parts[4]))
+        url_parts = list(urllib.parse.urlparse(url))
+        query = dict(urllib.parse.parse_qsl(url_parts[4]))
         query.update({"loginToken": token})
-        url_parts[4] = urllib.urlencode(query)
-        return urlparse.urlunparse(url_parts)
+        url_parts[4] = urllib.parse.urlencode(query).encode('ascii')
+        return urllib.parse.urlunparse(url_parts)
 
     def parse_cas_response(self, cas_response_body):
         user = None
diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py
index 6e95d9bec2d6..9382b1f12413 100644
--- a/synapse/rest/client/v1/push_rule.py
+++ b/synapse/rest/client/v1/push_rule.py
@@ -46,7 +46,7 @@ def on_PUT(self, request):
         try:
             priority_class = _priority_class_from_spec(spec)
         except InvalidRuleException as e:
-            raise SynapseError(400, e.message)
+            raise SynapseError(400, str(e))
 
         requester = yield self.auth.get_user_by_req(request)
 
@@ -73,7 +73,7 @@ def on_PUT(self, request):
                 content,
             )
         except InvalidRuleException as e:
-            raise SynapseError(400, e.message)
+            raise SynapseError(400, str(e))
 
         before = parse_string(request, "before")
         if before:
@@ -95,9 +95,9 @@ def on_PUT(self, request):
             )
             self.notify_user(user_id)
         except InconsistentRuleException as e:
-            raise SynapseError(400, e.message)
+            raise SynapseError(400, str(e))
         except RuleNotFoundException as e:
-            raise SynapseError(400, e.message)
+            raise SynapseError(400, str(e))
 
         defer.returnValue((200, {}))
 
@@ -142,10 +142,10 @@ def on_GET(self, request):
                 PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR
             )
 
-        if path[0] == '':
+        if path[0] == b'':
             defer.returnValue((200, rules))
-        elif path[0] == 'global':
-            path = path[1:]
+        elif path[0] == b'global':
+            path = [x.decode('ascii') for x in path[1:]]
             result = _filter_ruleset_with_path(rules['global'], path)
             defer.returnValue((200, result))
         else:
@@ -192,10 +192,10 @@ def set_rule_attr(self, user_id, spec, val):
 def _rule_spec_from_path(path):
     if len(path) < 2:
         raise UnrecognizedRequestError()
-    if path[0] != 'pushrules':
+    if path[0] != b'pushrules':
         raise UnrecognizedRequestError()
 
-    scope = path[1]
+    scope = path[1].decode('ascii')
     path = path[2:]
     if scope != 'global':
         raise UnrecognizedRequestError()
@@ -203,13 +203,13 @@ def _rule_spec_from_path(path):
     if len(path) == 0:
         raise UnrecognizedRequestError()
 
-    template = path[0]
+    template = path[0].decode('ascii')
     path = path[1:]
 
     if len(path) == 0 or len(path[0]) == 0:
         raise UnrecognizedRequestError()
 
-    rule_id = path[0]
+    rule_id = path[0].decode('ascii')
 
     spec = {
         'scope': scope,
@@ -220,7 +220,7 @@ def _rule_spec_from_path(path):
     path = path[1:]
 
     if len(path) > 0 and len(path[0]) > 0:
-        spec['attr'] = path[0]
+        spec['attr'] = path[0].decode('ascii')
 
     return spec
 
diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py
index 182a68b1e268..b84f0260f2c8 100644
--- a/synapse/rest/client/v1/pusher.py
+++ b/synapse/rest/client/v1/pusher.py
@@ -59,7 +59,7 @@ def on_GET(self, request):
         ]
 
         for p in pushers:
-            for k, v in p.items():
+            for k, v in list(p.items()):
                 if k not in allowed_keys:
                     del p[k]
 
@@ -126,7 +126,7 @@ def on_POST(self, request):
                 profile_tag=content.get('profile_tag', ""),
             )
         except PusherConfigException as pce:
-            raise SynapseError(400, "Config Error: " + pce.message,
+            raise SynapseError(400, "Config Error: " + str(pce),
                                errcode=Codes.MISSING_PARAM)
 
         self.notifier.on_new_replication_data()
diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py
index 976d98387d0a..663934efd04d 100644
--- a/synapse/rest/client/v1/room.py
+++ b/synapse/rest/client/v1/room.py
@@ -207,7 +207,7 @@ def on_POST(self, request, room_id, event_type, txn_id=None):
             "sender": requester.user.to_string(),
         }
 
-        if 'ts' in request.args and requester.app_service:
+        if b'ts' in request.args and requester.app_service:
             event_dict['origin_server_ts'] = parse_integer(request, "ts", 0)
 
         event = yield self.event_creation_hander.create_and_send_nonmember_event(
@@ -255,7 +255,9 @@ def on_POST(self, request, room_identifier, txn_id=None):
         if RoomID.is_valid(room_identifier):
             room_id = room_identifier
             try:
-                remote_room_hosts = request.args["server_name"]
+                remote_room_hosts = [
+                    x.decode('ascii') for x in request.args[b"server_name"]
+                ]
             except Exception:
                 remote_room_hosts = None
         elif RoomAlias.is_valid(room_identifier):
@@ -461,10 +463,10 @@ def on_GET(self, request, room_id):
         pagination_config = PaginationConfig.from_request(
             request, default_limit=10,
         )
-        as_client_event = "raw" not in request.args
-        filter_bytes = parse_string(request, "filter")
+        as_client_event = b"raw" not in request.args
+        filter_bytes = parse_string(request, b"filter", encoding=None)
         if filter_bytes:
-            filter_json = urlparse.unquote(filter_bytes).decode("UTF-8")
+            filter_json = urlparse.unquote(filter_bytes.decode("UTF-8"))
             event_filter = Filter(json.loads(filter_json))
         else:
             event_filter = None
@@ -560,7 +562,7 @@ def on_GET(self, request, room_id, event_id):
         # picking the API shape for symmetry with /messages
         filter_bytes = parse_string(request, "filter")
         if filter_bytes:
-            filter_json = urlparse.unquote(filter_bytes).decode("UTF-8")
+            filter_json = urlparse.unquote(filter_bytes)
             event_filter = Filter(json.loads(filter_json))
         else:
             event_filter = None
diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py
index 263d8eb73eb1..025114672286 100644
--- a/synapse/rest/client/v2_alpha/sync.py
+++ b/synapse/rest/client/v2_alpha/sync.py
@@ -89,7 +89,7 @@ def __init__(self, hs):
 
     @defer.inlineCallbacks
     def on_GET(self, request):
-        if "from" in request.args:
+        if b"from" in request.args:
             # /events used to use 'from', but /sync uses 'since'.
             # Lets be helpful and whine if we see a 'from'.
             raise SynapseError(
diff --git a/synapse/rest/client/v2_alpha/thirdparty.py b/synapse/rest/client/v2_alpha/thirdparty.py
index d9d379182efa..b9b5d0767733 100644
--- a/synapse/rest/client/v2_alpha/thirdparty.py
+++ b/synapse/rest/client/v2_alpha/thirdparty.py
@@ -79,7 +79,7 @@ def on_GET(self, request, protocol):
         yield self.auth.get_user_by_req(request, allow_guest=True)
 
         fields = request.args
-        fields.pop("access_token", None)
+        fields.pop(b"access_token", None)
 
         results = yield self.appservice_handler.query_3pe(
             ThirdPartyEntityKind.USER, protocol, fields
@@ -102,7 +102,7 @@ def on_GET(self, request, protocol):
         yield self.auth.get_user_by_req(request, allow_guest=True)
 
         fields = request.args
-        fields.pop("access_token", None)
+        fields.pop(b"access_token", None)
 
         results = yield self.appservice_handler.query_3pe(
             ThirdPartyEntityKind.LOCATION, protocol, fields
diff --git a/synapse/rest/key/v1/server_key_resource.py b/synapse/rest/key/v1/server_key_resource.py
index b9ee6e1c1330..38eb2ee23f93 100644
--- a/synapse/rest/key/v1/server_key_resource.py
+++ b/synapse/rest/key/v1/server_key_resource.py
@@ -88,5 +88,5 @@ def render_GET(self, request):
         )
 
     def getChild(self, name, request):
-        if name == '':
+        if name == b'':
             return self
diff --git a/synapse/rest/key/v2/__init__.py b/synapse/rest/key/v2/__init__.py
index 3491fd2118f8..cb5abcf826be 100644
--- a/synapse/rest/key/v2/__init__.py
+++ b/synapse/rest/key/v2/__init__.py
@@ -22,5 +22,5 @@
 class KeyApiV2Resource(Resource):
     def __init__(self, hs):
         Resource.__init__(self)
-        self.putChild("server", LocalKey(hs))
-        self.putChild("query", RemoteKey(hs))
+        self.putChild(b"server", LocalKey(hs))
+        self.putChild(b"query", RemoteKey(hs))
diff --git a/synapse/rest/key/v2/remote_key_resource.py b/synapse/rest/key/v2/remote_key_resource.py
index 7d67e4b0644b..eb8782aa6e1a 100644
--- a/synapse/rest/key/v2/remote_key_resource.py
+++ b/synapse/rest/key/v2/remote_key_resource.py
@@ -103,7 +103,7 @@ def render_GET(self, request):
     def async_render_GET(self, request):
         if len(request.postpath) == 1:
             server, = request.postpath
-            query = {server: {}}
+            query = {server.decode('ascii'): {}}
         elif len(request.postpath) == 2:
             server, key_id = request.postpath
             minimum_valid_until_ts = parse_integer(
@@ -112,11 +112,12 @@ def async_render_GET(self, request):
             arguments = {}
             if minimum_valid_until_ts is not None:
                 arguments["minimum_valid_until_ts"] = minimum_valid_until_ts
-            query = {server: {key_id: arguments}}
+            query = {server.decode('ascii'): {key_id.decode('ascii'): arguments}}
         else:
             raise SynapseError(
                 404, "Not found %r" % request.postpath, Codes.NOT_FOUND
             )
+
         yield self.query_keys(request, query, query_remote_on_cache_miss=True)
 
     def render_POST(self, request):
@@ -135,6 +136,7 @@ def async_render_POST(self, request):
     @defer.inlineCallbacks
     def query_keys(self, request, query, query_remote_on_cache_miss=False):
         logger.info("Handling query for keys %r", query)
+
         store_queries = []
         for server_name, key_ids in query.items():
             if (
diff --git a/synapse/rest/media/v0/content_repository.py b/synapse/rest/media/v0/content_repository.py
index f255f2883fe8..5a426ff2f63c 100644
--- a/synapse/rest/media/v0/content_repository.py
+++ b/synapse/rest/media/v0/content_repository.py
@@ -56,7 +56,7 @@ def render_GET(self, request):
         # servers.
 
         # TODO: A little crude here, we could do this better.
-        filename = request.path.split('/')[-1]
+        filename = request.path.decode('ascii').split('/')[-1]
         # be paranoid
         filename = re.sub("[^0-9A-z.-_]", "", filename)
 
@@ -78,7 +78,7 @@ def render_GET(self, request):
             # select private. don't bother setting Expires as all our matrix
             # clients are smart enough to be happy with Cache-Control (right?)
             request.setHeader(
-                "Cache-Control", "public,max-age=86400,s-maxage=86400"
+                b"Cache-Control", b"public,max-age=86400,s-maxage=86400"
             )
 
             d = FileSender().beginFileTransfer(f, request)
diff --git a/synapse/rest/media/v1/_base.py b/synapse/rest/media/v1/_base.py
index 65f4bd291004..76e479afa3b6 100644
--- a/synapse/rest/media/v1/_base.py
+++ b/synapse/rest/media/v1/_base.py
@@ -15,9 +15,8 @@
 
 import logging
 import os
-import urllib
 
-from six.moves.urllib import parse as urlparse
+from six.moves import urllib
 
 from twisted.internet import defer
 from twisted.protocols.basic import FileSender
@@ -35,10 +34,15 @@ def parse_media_id(request):
         # This allows users to append e.g. /test.png to the URL. Useful for
         # clients that parse the URL to see content type.
         server_name, media_id = request.postpath[:2]
+
+        if isinstance(server_name, bytes):
+            server_name = server_name.decode('utf-8')
+            media_id = media_id.decode('utf8')
+
         file_name = None
         if len(request.postpath) > 2:
             try:
-                file_name = urlparse.unquote(request.postpath[-1]).decode("utf-8")
+                file_name = urllib.parse.unquote(request.postpath[-1].decode("utf-8"))
             except UnicodeDecodeError:
                 pass
         return server_name, media_id, file_name
@@ -93,22 +97,18 @@ def add_file_headers(request, media_type, file_size, upload_name):
         file_size (int): Size in bytes of the media, if known.
         upload_name (str): The name of the requested file, if any.
     """
+    def _quote(x):
+        return urllib.parse.quote(x.encode("utf-8"))
+
     request.setHeader(b"Content-Type", media_type.encode("UTF-8"))
     if upload_name:
         if is_ascii(upload_name):
-            request.setHeader(
-                b"Content-Disposition",
-                b"inline; filename=%s" % (
-                    urllib.quote(upload_name.encode("utf-8")),
-                ),
-            )
+            disposition = ("inline; filename=%s" % (_quote(upload_name),)).encode("ascii")
         else:
-            request.setHeader(
-                b"Content-Disposition",
-                b"inline; filename*=utf-8''%s" % (
-                    urllib.quote(upload_name.encode("utf-8")),
-                ),
-            )
+            disposition = (
+                "inline; filename*=utf-8''%s" % (_quote(upload_name),)).encode("ascii")
+
+        request.setHeader(b"Content-Disposition", disposition)
 
     # cache for at least a day.
     # XXX: we might want to turn this off for data we don't want to
diff --git a/synapse/rest/media/v1/download_resource.py b/synapse/rest/media/v1/download_resource.py
index fbfa85f74fd9..ca90964d1dfd 100644
--- a/synapse/rest/media/v1/download_resource.py
+++ b/synapse/rest/media/v1/download_resource.py
@@ -47,12 +47,12 @@ def render_GET(self, request):
     def _async_render_GET(self, request):
         set_cors_headers(request)
         request.setHeader(
-            "Content-Security-Policy",
-            "default-src 'none';"
-            " script-src 'none';"
-            " plugin-types application/pdf;"
-            " style-src 'unsafe-inline';"
-            " object-src 'self';"
+            b"Content-Security-Policy",
+            b"default-src 'none';"
+            b" script-src 'none';"
+            b" plugin-types application/pdf;"
+            b" style-src 'unsafe-inline';"
+            b" object-src 'self';"
         )
         server_name, media_id, name = parse_media_id(request)
         if server_name == self.server_name:
diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py
index 241c972070e0..a828ff4438c9 100644
--- a/synapse/rest/media/v1/media_repository.py
+++ b/synapse/rest/media/v1/media_repository.py
@@ -20,7 +20,7 @@
 import os
 import shutil
 
-from six import iteritems
+from six import PY3, iteritems
 from six.moves.urllib import parse as urlparse
 
 import twisted.internet.error
@@ -397,13 +397,13 @@ def _download_remote_file(self, server_name, media_id, file_id):
 
             yield finish()
 
-        media_type = headers["Content-Type"][0]
+        media_type = headers[b"Content-Type"][0].decode('ascii')
 
         time_now_ms = self.clock.time_msec()
 
-        content_disposition = headers.get("Content-Disposition", None)
+        content_disposition = headers.get(b"Content-Disposition", None)
         if content_disposition:
-            _, params = cgi.parse_header(content_disposition[0],)
+            _, params = cgi.parse_header(content_disposition[0].decode('ascii'),)
             upload_name = None
 
             # First check if there is a valid UTF-8 filename
@@ -419,9 +419,13 @@ def _download_remote_file(self, server_name, media_id, file_id):
                     upload_name = upload_name_ascii
 
             if upload_name:
-                upload_name = urlparse.unquote(upload_name)
+                if PY3:
+                    upload_name = urlparse.unquote(upload_name)
+                else:
+                    upload_name = urlparse.unquote(upload_name.encode('ascii'))
                 try:
-                    upload_name = upload_name.decode("utf-8")
+                    if isinstance(upload_name, bytes):
+                        upload_name = upload_name.decode("utf-8")
                 except UnicodeDecodeError:
                     upload_name = None
         else:
@@ -755,14 +759,15 @@ def __init__(self, hs):
         Resource.__init__(self)
 
         media_repo = hs.get_media_repository()
-        self.putChild("upload", UploadResource(hs, media_repo))
-        self.putChild("download", DownloadResource(hs, media_repo))
-        self.putChild("thumbnail", ThumbnailResource(
+
+        self.putChild(b"upload", UploadResource(hs, media_repo))
+        self.putChild(b"download", DownloadResource(hs, media_repo))
+        self.putChild(b"thumbnail", ThumbnailResource(
             hs, media_repo, media_repo.media_storage,
         ))
-        self.putChild("identicon", IdenticonResource())
+        self.putChild(b"identicon", IdenticonResource())
         if hs.config.url_preview_enabled:
-            self.putChild("preview_url", PreviewUrlResource(
+            self.putChild(b"preview_url", PreviewUrlResource(
                 hs, media_repo, media_repo.media_storage,
             ))
-        self.putChild("config", MediaConfigResource(hs))
+        self.putChild(b"config", MediaConfigResource(hs))
diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py
index 778ef9733741..cad2dec33a2b 100644
--- a/synapse/rest/media/v1/preview_url_resource.py
+++ b/synapse/rest/media/v1/preview_url_resource.py
@@ -261,7 +261,7 @@ def _do_preview(self, url, user, ts):
 
         logger.debug("Calculated OG for %s as %s" % (url, og))
 
-        jsonog = json.dumps(og)
+        jsonog = json.dumps(og).encode('utf8')
 
         # store OG in history-aware DB cache
         yield self.store.store_url_cache(
@@ -301,20 +301,20 @@ def _download_url(self, url, user):
                 logger.warn("Error downloading %s: %r", url, e)
                 raise SynapseError(
                     500, "Failed to download content: %s" % (
-                        traceback.format_exception_only(sys.exc_type, e),
+                        traceback.format_exception_only(sys.exc_info()[0], e),
                     ),
                     Codes.UNKNOWN,
                 )
             yield finish()
 
         try:
-            if "Content-Type" in headers:
-                media_type = headers["Content-Type"][0]
+            if b"Content-Type" in headers:
+                media_type = headers[b"Content-Type"][0].decode('ascii')
             else:
                 media_type = "application/octet-stream"
             time_now_ms = self.clock.time_msec()
 
-            content_disposition = headers.get("Content-Disposition", None)
+            content_disposition = headers.get(b"Content-Disposition", None)
             if content_disposition:
                 _, params = cgi.parse_header(content_disposition[0],)
                 download_name = None

From 0ddf48672421aa64920342b959dc77432c869889 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 12 Sep 2018 11:58:52 +0100
Subject: [PATCH 595/830] expose number of real reserved users

---
 synapse/app/homeserver.py                  | 10 ++++---
 synapse/storage/monthly_active_users.py    | 17 ++++++++++++
 tests/storage/test_monthly_active_users.py | 31 ++++++++++++++++++++++
 3 files changed, 55 insertions(+), 3 deletions(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 3eb5b663de5b..e6372bcec2aa 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -307,6 +307,7 @@ def run_startup_checks(self, db_conn, database_engine):
 # Gauges to expose monthly active user control metrics
 current_mau_gauge = Gauge("synapse_admin_mau:current", "Current MAU")
 max_mau_gauge = Gauge("synapse_admin_mau:max", "MAU Limit")
+reserved_mau_gauge = Gauge("synapse_admin_mau:reserved", "Reserved real MAU users")
 
 
 def setup(config_options):
@@ -531,10 +532,13 @@ def generate_user_daily_visit_stats():
 
     @defer.inlineCallbacks
     def generate_monthly_active_users():
-        count = 0
+        current_mau_count = 0
+        reserved_mau_count = 0
         if hs.config.limit_usage_by_mau:
-            count = yield hs.get_datastore().get_monthly_active_count()
-        current_mau_gauge.set(float(count))
+            current_mau_count = yield hs.get_datastore().get_monthly_active_count()
+            reserved_mau_count = yield hs.get_datastore().get_reserved_real_user_account()
+        current_mau_gauge.set(float(current_mau_count))
+        reserved_mau_gauge.set(float(reserved_mau_count))
         max_mau_gauge.set(float(hs.config.max_mau_value))
 
     hs.get_datastore().initialise_reserved_users(
diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py
index b890c152dbec..53d125d30511 100644
--- a/synapse/storage/monthly_active_users.py
+++ b/synapse/storage/monthly_active_users.py
@@ -146,6 +146,23 @@ def _count_users(txn):
             return count
         return self.runInteraction("count_users", _count_users)
 
+    @defer.inlineCallbacks
+    def get_reserved_real_user_account(self):
+        """Of the reserved threepids defined in config, how many are associated
+        with registered users?
+
+        Returns:
+            Defered[int]: Number of real reserved users
+        """
+        count = 0
+        for tp in self.hs.config.mau_limits_reserved_threepids:
+            user_id = yield self.hs.get_datastore().get_user_id_by_threepid(
+                tp["medium"], tp["address"]
+            )
+            if user_id:
+                count = count + 1
+        defer.returnValue(count)
+
     @defer.inlineCallbacks
     def upsert_monthly_active_user(self, user_id):
         """
diff --git a/tests/storage/test_monthly_active_users.py b/tests/storage/test_monthly_active_users.py
index ccfc21b7fcdf..662f2ed8455d 100644
--- a/tests/storage/test_monthly_active_users.py
+++ b/tests/storage/test_monthly_active_users.py
@@ -183,3 +183,34 @@ def test_populate_monthly_users_should_not_update(self):
         self.store.populate_monthly_active_users('user_id')
         self.pump()
         self.store.upsert_monthly_active_user.assert_not_called()
+
+    def test_get_reserved_real_user_account(self):
+        # Test no reserved users, or reserved threepids
+        count = self.store.get_reserved_real_user_account()
+        self.assertEquals(self.get_success(count), 0)
+        # Test reserved users but no registered users
+
+        user1 = '@user1:example.com'
+        user2 = '@user2:example.com'
+        user1_email = 'user1@example.com'
+        user2_email = 'user2@example.com'
+        threepids = [
+            {'medium': 'email', 'address': user1_email},
+            {'medium': 'email', 'address': user2_email},
+        ]
+        self.hs.config.mau_limits_reserved_threepids = threepids
+        self.store.initialise_reserved_users(threepids)
+        self.pump()
+        count = self.store.get_reserved_real_user_account()
+        self.assertEquals(self.get_success(count), 0)
+
+        # Test reserved registed users
+        self.store.register(user_id=user1, token="123", password_hash=None)
+        self.store.register(user_id=user2, token="456", password_hash=None)
+        self.pump()
+
+        now = int(self.hs.get_clock().time_msec())
+        self.store.user_add_threepid(user1, "email", user1_email, now, now)
+        self.store.user_add_threepid(user2, "email", user2_email, now, now)
+        count = self.store.get_reserved_real_user_account()
+        self.assertEquals(self.get_success(count), len(threepids))

From 5cea4e16c7c6e7b38ff294cb77b4ef8f51d8b76e Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 12 Sep 2018 12:03:31 +0100
Subject: [PATCH 596/830] towncrier

---
 changelog.d/3846.feature | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3846.feature

diff --git a/changelog.d/3846.feature b/changelog.d/3846.feature
new file mode 100644
index 000000000000..dc9e9d86cc0d
--- /dev/null
+++ b/changelog.d/3846.feature
@@ -0,0 +1 @@
+create synapse_admin_mau:reserved metric to expose number of real reaserved users 

From 7ca097f77eba19403b4169e23404678aa2c7cd91 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Wed, 12 Sep 2018 23:23:32 +1000
Subject: [PATCH 597/830] Port federation/ to py3 (#3847)

---
 changelog.d/3847.misc                   |  1 +
 synapse/federation/federation_client.py |  8 ++++----
 synapse/federation/transport/client.py  |  5 +++--
 synapse/federation/transport/server.py  | 24 +++++++++++-------------
 4 files changed, 19 insertions(+), 19 deletions(-)
 create mode 100644 changelog.d/3847.misc

diff --git a/changelog.d/3847.misc b/changelog.d/3847.misc
new file mode 100644
index 000000000000..bf8b5afea4e2
--- /dev/null
+++ b/changelog.d/3847.misc
@@ -0,0 +1 @@
+federation/ is now ported to Python 3.
\ No newline at end of file
diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index c9f3c2d35262..fe67b2ff42a9 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -271,10 +271,10 @@ def get_pdu(self, destinations, event_id, outlier=False, timeout=None):
                     event_id, destination, e,
                 )
             except NotRetryingDestination as e:
-                logger.info(e.message)
+                logger.info(str(e))
                 continue
             except FederationDeniedError as e:
-                logger.info(e.message)
+                logger.info(str(e))
                 continue
             except Exception as e:
                 pdu_attempts[destination] = now
@@ -510,7 +510,7 @@ def _try_destination_list(self, description, destinations, callback):
                 else:
                     logger.warn(
                         "Failed to %s via %s: %i %s",
-                        description, destination, e.code, e.message,
+                        description, destination, e.code, e.args[0],
                     )
             except Exception:
                 logger.warn(
@@ -875,7 +875,7 @@ def forward_third_party_invite(self, destinations, room_id, event_dict):
             except Exception as e:
                 logger.exception(
                     "Failed to send_third_party_invite via %s: %s",
-                    destination, e.message
+                    destination, str(e)
                 )
 
         raise RuntimeError("Failed to send to any server.")
diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index 1054441ca5eb..2ab973d6c804 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -15,7 +15,8 @@
 # limitations under the License.
 
 import logging
-import urllib
+
+from six.moves import urllib
 
 from twisted.internet import defer
 
@@ -951,4 +952,4 @@ def _create_path(prefix, path, *args):
     Returns:
         str
     """
-    return prefix + path % tuple(urllib.quote(arg, "") for arg in args)
+    return prefix + path % tuple(urllib.parse.quote(arg, "") for arg in args)
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 3972922ff998..2f874b483815 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -90,8 +90,8 @@ def __init__(self, hs):
     @defer.inlineCallbacks
     def authenticate_request(self, request, content):
         json_request = {
-            "method": request.method,
-            "uri": request.uri,
+            "method": request.method.decode('ascii'),
+            "uri": request.uri.decode('ascii'),
             "destination": self.server_name,
             "signatures": {},
         }
@@ -252,7 +252,7 @@ def new_func(request, *args, **kwargs):
                     by the callback method. None if the request has already been handled.
             """
             content = None
-            if request.method in ["PUT", "POST"]:
+            if request.method in [b"PUT", b"POST"]:
                 # TODO: Handle other method types? other content types?
                 content = parse_json_object_from_request(request)
 
@@ -386,7 +386,7 @@ def on_GET(self, origin, content, query, context):
         return self.handler.on_context_state_request(
             origin,
             context,
-            query.get("event_id", [None])[0],
+            parse_string_from_args(query, "event_id", None),
         )
 
 
@@ -397,7 +397,7 @@ def on_GET(self, origin, content, query, room_id):
         return self.handler.on_state_ids_request(
             origin,
             room_id,
-            query.get("event_id", [None])[0],
+            parse_string_from_args(query, "event_id", None),
         )
 
 
@@ -405,14 +405,12 @@ class FederationBackfillServlet(BaseFederationServlet):
     PATH = "/backfill/(?P<context>[^/]*)/"
 
     def on_GET(self, origin, content, query, context):
-        versions = query["v"]
-        limits = query["limit"]
+        versions = [x.decode('ascii') for x in query[b"v"]]
+        limit = parse_integer_from_args(query, "limit", None)
 
-        if not limits:
+        if not limit:
             return defer.succeed((400, {"error": "Did not include limit param"}))
 
-        limit = int(limits[-1])
-
         return self.handler.on_backfill_request(origin, context, versions, limit)
 
 
@@ -423,7 +421,7 @@ class FederationQueryServlet(BaseFederationServlet):
     def on_GET(self, origin, content, query, query_type):
         return self.handler.on_query_request(
             query_type,
-            {k: v[0].decode("utf-8") for k, v in query.items()}
+            {k.decode('utf8'): v[0].decode("utf-8") for k, v in query.items()}
         )
 
 
@@ -630,14 +628,14 @@ class OpenIdUserInfo(BaseFederationServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, origin, content, query):
-        token = query.get("access_token", [None])[0]
+        token = query.get(b"access_token", [None])[0]
         if token is None:
             defer.returnValue((401, {
                 "errcode": "M_MISSING_TOKEN", "error": "Access Token required"
             }))
             return
 
-        user_id = yield self.handler.on_openid_userinfo(token)
+        user_id = yield self.handler.on_openid_userinfo(token.decode('ascii'))
 
         if user_id is None:
             defer.returnValue((401, {

From 65cd8ccc795657c3ff96b41712a95ff890a0d184 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Wed, 12 Sep 2018 23:29:21 +1000
Subject: [PATCH 598/830] Add JUnit summaries to CircleCI as well as merged
 runs (#3704)

---
 .circleci/config.yml           | 66 ++++++++++++++++++++++++++++++++++
 .circleci/merge_base_branch.sh | 27 ++++++++++++++
 changelog.d/3704.misc          |  1 +
 3 files changed, 94 insertions(+)
 create mode 100755 .circleci/merge_base_branch.sh
 create mode 100644 changelog.d/3704.misc

diff --git a/.circleci/config.yml b/.circleci/config.yml
index e03f01b83703..5266544f3c37 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -9,6 +9,8 @@ jobs:
       - store_artifacts:
           path: ~/project/logs
           destination: logs
+      - store_test_results:
+          path: logs
   sytestpy2postgres:
     machine: true
     steps:
@@ -18,6 +20,34 @@ jobs:
       - store_artifacts:
           path: ~/project/logs
           destination: logs
+      - store_test_results:
+          path: logs
+  sytestpy2merged:
+    machine: true
+    steps:
+      - checkout
+      - run: bash .circleci/merge_base_branch.sh
+      - run: docker pull matrixdotorg/sytest-synapsepy2
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy2
+      - store_artifacts:
+          path: ~/project/logs
+          destination: logs
+      - store_test_results:
+          path: logs
+
+  sytestpy2postgresmerged:
+    machine: true
+    steps:
+      - checkout
+      - run: bash .circleci/merge_base_branch.sh
+      - run: docker pull matrixdotorg/sytest-synapsepy2
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy2
+      - store_artifacts:
+          path: ~/project/logs
+          destination: logs
+      - store_test_results:
+          path: logs
+
   sytestpy3:
     machine: true
     steps:
@@ -27,6 +57,8 @@ jobs:
       - store_artifacts:
           path: ~/project/logs
           destination: logs
+      - store_test_results:
+          path: logs
   sytestpy3postgres:
     machine: true
     steps:
@@ -36,6 +68,32 @@ jobs:
       - store_artifacts:
           path: ~/project/logs
           destination: logs
+      - store_test_results:
+          path: logs
+  sytestpy3merged:
+    machine: true
+    steps:
+      - checkout
+      - run: bash .circleci/merge_base_branch.sh
+      - run: docker pull matrixdotorg/sytest-synapsepy3
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs hawkowl/sytestpy3
+      - store_artifacts:
+          path: ~/project/logs
+          destination: logs
+      - store_test_results:
+          path: logs
+  sytestpy3postgresmerged:
+    machine: true
+    steps:
+      - checkout
+      - run: bash .circleci/merge_base_branch.sh
+      - run: docker pull matrixdotorg/sytest-synapsepy3
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy3
+      - store_artifacts:
+          path: ~/project/logs
+          destination: logs
+      - store_test_results:
+          path: logs
 
 workflows:
   version: 2
@@ -43,6 +101,14 @@ workflows:
     jobs:
       - sytestpy2
       - sytestpy2postgres
+      - sytestpy2merged:
+          filters:
+            branches:
+              ignore: /develop|master/
+      - sytestpy2postgresmerged:
+          filters:
+            branches:
+              ignore: /develop|master/
 # Currently broken while the Python 3 port is incomplete
 #      - sytestpy3
 #      - sytestpy3postgres
diff --git a/.circleci/merge_base_branch.sh b/.circleci/merge_base_branch.sh
new file mode 100755
index 000000000000..2d700dbf1189
--- /dev/null
+++ b/.circleci/merge_base_branch.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+set -e
+
+# CircleCI doesn't give CIRCLE_PR_NUMBER in the environment for non-forked PRs. Wonderful.
+# In this case, we just need to do some ~shell magic~ to strip it out of the PULL_REQUEST URL.
+echo 'export CIRCLE_PR_NUMBER="${CIRCLE_PR_NUMBER:-${CIRCLE_PULL_REQUEST##*/}}"' >> "$BASH_ENV"
+source $BASH_ENV
+
+if [[ -z "${CIRCLE_PR_NUMBER}" ]]
+then
+    echo "Can't figure out what the PR number is!"
+    exit 1
+fi
+
+# Get the reference, using the GitHub API
+GITBASE=`curl -q https://api.github.com/repos/matrix-org/synapse/pulls/${CIRCLE_PR_NUMBER} | jq -r '.base.ref'`
+
+# Show what we are before
+git show -s
+
+# Fetch and merge. If it doesn't work, it will raise due to set -e.
+git fetch -u origin $GITBASE
+git merge --no-edit origin/$GITBASE
+
+# Show what we are after.
+git show -s
\ No newline at end of file
diff --git a/changelog.d/3704.misc b/changelog.d/3704.misc
new file mode 100644
index 000000000000..aaae0fbd6304
--- /dev/null
+++ b/changelog.d/3704.misc
@@ -0,0 +1 @@
+CircleCI tests now run on the potential merge of a PR.

From 8c5b84441bdcc0f321ca9a58e8ec20cb35cdb456 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 12 Sep 2018 16:22:14 +0100
Subject: [PATCH 599/830] Log outbound requests when we retry

---
 synapse/http/matrixfederationclient.py | 149 ++++++++++++-------------
 1 file changed, 74 insertions(+), 75 deletions(-)

diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index f9a1fbf95d76..cf920bc04144 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -177,11 +177,6 @@ def _request(self, destination, method, path,
             txn_id = "%s-O-%s" % (method, self._next_id)
             self._next_id = (self._next_id + 1) % (MAXINT - 1)
 
-            outbound_logger.info(
-                "{%s} [%s] Sending request: %s %s",
-                txn_id, destination, method, url
-            )
-
             # XXX: Would be much nicer to retry only at the transaction-layer
             # (once we have reliable transactions in place)
             if long_retries:
@@ -194,85 +189,89 @@ def _request(self, destination, method, path,
             ).decode('ascii')
 
             log_result = None
-            try:
-                while True:
-                    try:
-                        if json_callback:
-                            json = json_callback()
-
-                        if json:
-                            data = encode_canonical_json(json)
-                            headers_dict["Content-Type"] = ["application/json"]
-                            self.sign_request(
-                                destination, method, http_url, headers_dict, json
-                            )
-                        else:
-                            data = None
-                            self.sign_request(destination, method, http_url, headers_dict)
-
-                        request_deferred = treq.request(
-                            method,
-                            url,
-                            headers=Headers(headers_dict),
-                            data=data,
-                            agent=self.agent,
-                        )
-                        add_timeout_to_deferred(
-                            request_deferred,
-                            timeout / 1000. if timeout else 60,
-                            self.hs.get_reactor(),
-                            cancelled_to_request_timed_out_error,
-                        )
-                        response = yield make_deferred_yieldable(
-                            request_deferred,
+            while True:
+                try:
+                    if json_callback:
+                        json = json_callback()
+
+                    if json:
+                        data = encode_canonical_json(json)
+                        headers_dict["Content-Type"] = ["application/json"]
+                        self.sign_request(
+                            destination, method, http_url, headers_dict, json
                         )
+                    else:
+                        data = None
+                        self.sign_request(destination, method, http_url, headers_dict)
 
-                        log_result = "%d %s" % (response.code, response.phrase,)
-                        break
-                    except Exception as e:
-                        if not retry_on_dns_fail and isinstance(e, DNSLookupError):
-                            logger.warn(
-                                "DNS Lookup failed to %s with %s",
-                                destination,
-                                e
-                            )
-                            log_result = "DNS Lookup failed to %s with %s" % (
-                                destination, e
-                            )
-                            raise
+                    outbound_logger.info(
+                        "{%s} [%s] Sending request: %s %s",
+                        txn_id, destination, method, url
+                    )
 
+                    request_deferred = treq.request(
+                        method,
+                        url,
+                        headers=Headers(headers_dict),
+                        data=data,
+                        agent=self.agent,
+                    )
+                    add_timeout_to_deferred(
+                        request_deferred,
+                        timeout / 1000. if timeout else 60,
+                        self.hs.get_reactor(),
+                        cancelled_to_request_timed_out_error,
+                    )
+                    response = yield make_deferred_yieldable(
+                        request_deferred,
+                    )
+
+                    log_result = "%d %s" % (response.code, response.phrase,)
+                    break
+                except Exception as e:
+                    if not retry_on_dns_fail and isinstance(e, DNSLookupError):
                         logger.warn(
-                            "{%s} Sending request failed to %s: %s %s: %s",
-                            txn_id,
+                            "DNS Lookup failed to %s with %s",
                             destination,
-                            method,
-                            url,
-                            _flatten_response_never_received(e),
+                            e
                         )
+                        log_result = "DNS Lookup failed to %s with %s" % (
+                            destination, e
+                        )
+                        raise
+
+                    logger.warn(
+                        "{%s} Sending request failed to %s: %s %s: %s",
+                        txn_id,
+                        destination,
+                        method,
+                        url,
+                        _flatten_response_never_received(e),
+                    )
 
-                        log_result = _flatten_response_never_received(e)
-
-                        if retries_left and not timeout:
-                            if long_retries:
-                                delay = 4 ** (MAX_LONG_RETRIES + 1 - retries_left)
-                                delay = min(delay, 60)
-                                delay *= random.uniform(0.8, 1.4)
-                            else:
-                                delay = 0.5 * 2 ** (MAX_SHORT_RETRIES - retries_left)
-                                delay = min(delay, 2)
-                                delay *= random.uniform(0.8, 1.4)
+                    log_result = _flatten_response_never_received(e)
 
-                            yield self.clock.sleep(delay)
-                            retries_left -= 1
+                    if retries_left and not timeout:
+                        if long_retries:
+                            delay = 4 ** (MAX_LONG_RETRIES + 1 - retries_left)
+                            delay = min(delay, 60)
+                            delay *= random.uniform(0.8, 1.4)
                         else:
-                            raise
-            finally:
-                outbound_logger.info(
-                    "{%s} [%s] Result: %s",
-                    txn_id,
-                    destination,
-                    log_result,
-                )
+                            delay = 0.5 * 2 ** (MAX_SHORT_RETRIES - retries_left)
+                            delay = min(delay, 2)
+                            delay *= random.uniform(0.8, 1.4)
+
+                        yield self.clock.sleep(delay)
+                        retries_left -= 1
+                    else:
+                        raise
+                finally:
+                    outbound_logger.info(
+                        "{%s} [%s] Result: %s",
+                        txn_id,
+                        destination,
+                        log_result,
+                    )
 
             if 200 <= response.code < 300:
                 pass

From 8decd6233dabd87160794949ffd95282e60ab01e Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 12 Sep 2018 16:22:15 +0100
Subject: [PATCH 600/830] improve naming

---
 changelog.d/3846.feature                   |  2 +-
 synapse/app/homeserver.py                  | 14 +++++++++-----
 synapse/storage/monthly_active_users.py    |  2 +-
 tests/storage/test_monthly_active_users.py |  6 +++---
 4 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/changelog.d/3846.feature b/changelog.d/3846.feature
index dc9e9d86cc0d..453c11d3f8a7 100644
--- a/changelog.d/3846.feature
+++ b/changelog.d/3846.feature
@@ -1 +1 @@
-create synapse_admin_mau:reserved metric to expose number of real reaserved users 
+Add synapse_admin_mau:registered_reserved_users metric to expose number of real reaserved users 
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index e6372bcec2aa..ac97e19649b5 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -307,7 +307,10 @@ def run_startup_checks(self, db_conn, database_engine):
 # Gauges to expose monthly active user control metrics
 current_mau_gauge = Gauge("synapse_admin_mau:current", "Current MAU")
 max_mau_gauge = Gauge("synapse_admin_mau:max", "MAU Limit")
-reserved_mau_gauge = Gauge("synapse_admin_mau:reserved", "Reserved real MAU users")
+registered_reserved_users_mau_gauge = Gauge(
+    "synapse_admin_mau:registered_reserved_users",
+    "Registered users with reserved threepids"
+)
 
 
 def setup(config_options):
@@ -533,12 +536,13 @@ def generate_user_daily_visit_stats():
     @defer.inlineCallbacks
     def generate_monthly_active_users():
         current_mau_count = 0
-        reserved_mau_count = 0
+        reserved_count = 0
+        store = hs.get_datastore()
         if hs.config.limit_usage_by_mau:
-            current_mau_count = yield hs.get_datastore().get_monthly_active_count()
-            reserved_mau_count = yield hs.get_datastore().get_reserved_real_user_account()
+            current_mau_count = yield store.get_monthly_active_count()
+            reserved_count = yield store.get_registered_reserved_users_count()
         current_mau_gauge.set(float(current_mau_count))
-        reserved_mau_gauge.set(float(reserved_mau_count))
+        registered_reserved_users_mau_gauge.set(float(reserved_count))
         max_mau_gauge.set(float(hs.config.max_mau_value))
 
     hs.get_datastore().initialise_reserved_users(
diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py
index 53d125d30511..59580949f190 100644
--- a/synapse/storage/monthly_active_users.py
+++ b/synapse/storage/monthly_active_users.py
@@ -147,7 +147,7 @@ def _count_users(txn):
         return self.runInteraction("count_users", _count_users)
 
     @defer.inlineCallbacks
-    def get_reserved_real_user_account(self):
+    def get_registered_reserved_users_count(self):
         """Of the reserved threepids defined in config, how many are associated
         with registered users?
 
diff --git a/tests/storage/test_monthly_active_users.py b/tests/storage/test_monthly_active_users.py
index 662f2ed8455d..686f12a0dcdc 100644
--- a/tests/storage/test_monthly_active_users.py
+++ b/tests/storage/test_monthly_active_users.py
@@ -186,7 +186,7 @@ def test_populate_monthly_users_should_not_update(self):
 
     def test_get_reserved_real_user_account(self):
         # Test no reserved users, or reserved threepids
-        count = self.store.get_reserved_real_user_account()
+        count = self.store.get_registered_reserved_users_count()
         self.assertEquals(self.get_success(count), 0)
         # Test reserved users but no registered users
 
@@ -201,7 +201,7 @@ def test_get_reserved_real_user_account(self):
         self.hs.config.mau_limits_reserved_threepids = threepids
         self.store.initialise_reserved_users(threepids)
         self.pump()
-        count = self.store.get_reserved_real_user_account()
+        count = self.store.get_registered_reserved_users_count()
         self.assertEquals(self.get_success(count), 0)
 
         # Test reserved registed users
@@ -212,5 +212,5 @@ def test_get_reserved_real_user_account(self):
         now = int(self.hs.get_clock().time_msec())
         self.store.user_add_threepid(user1, "email", user1_email, now, now)
         self.store.user_add_threepid(user2, "email", user2_email, now, now)
-        count = self.store.get_reserved_real_user_account()
+        count = self.store.get_registered_reserved_users_count()
         self.assertEquals(self.get_success(count), len(threepids))

From 3db016b641bf9141c56d10376c7dcbc8108a6a9e Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 12 Sep 2018 16:25:18 +0100
Subject: [PATCH 601/830] Newsfile

---
 changelog.d/3853.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3853.misc

diff --git a/changelog.d/3853.misc b/changelog.d/3853.misc
new file mode 100644
index 000000000000..db45d4983db9
--- /dev/null
+++ b/changelog.d/3853.misc
@@ -0,0 +1 @@
+Log when we retry outbound requests

From 11bfc2af1cfdff08775160ea0c91c51326f1f665 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Wed, 12 Sep 2018 16:45:42 +0100
Subject: [PATCH 602/830] fix logline

---
 synapse/handlers/sync.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 23983a51ab18..34729e7f8ced 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -1701,7 +1701,7 @@ def _generate_room_entry(self, sync_result_builder, ignored_users,
             if batch.limited:
                 user_id = sync_result_builder.sync_config.user.to_string()
                 logger.info(
-                    "Incremental syncing room %s for user %s with %d state events" % (
+                    "Incremental gappy sync of room %s for user %s with %d state events" % (
                         room_id,
                         user_id,
                         len(state),

From 0e200e366d568586e8f7f9dd0beb9b53bef86047 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Wed, 12 Sep 2018 16:47:20 +0100
Subject: [PATCH 603/830] correctly log gappy sync metrics

---
 synapse/handlers/sync.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 34729e7f8ced..88ef894d6ed8 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -1698,7 +1698,7 @@ def _generate_room_entry(self, sync_result_builder, ignored_users,
 
                 sync_result_builder.joined.append(room_sync)
 
-            if batch.limited:
+            if batch.limited and since_token:
                 user_id = sync_result_builder.sync_config.user.to_string()
                 logger.info(
                     "Incremental gappy sync of room %s for user %s with %d state events" % (

From 0403cf07839abc0129ed75e785b10330338caf76 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Wed, 12 Sep 2018 16:54:28 +0100
Subject: [PATCH 604/830] argh pep8

---
 synapse/handlers/sync.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 88ef894d6ed8..45a1dd5952c3 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -1701,7 +1701,7 @@ def _generate_room_entry(self, sync_result_builder, ignored_users,
             if batch.limited and since_token:
                 user_id = sync_result_builder.sync_config.user.to_string()
                 logger.info(
-                    "Incremental gappy sync of room %s for user %s with %d state events" % (
+                    "Incremental gappy sync of %s for user %s with %d state events" % (
                         room_id,
                         user_id,
                         len(state),

From 2ac1abbc7e8647548ae2e32119cb2bd3b173d9bd Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@arasphere.net>
Date: Wed, 12 Sep 2018 17:11:05 +0100
Subject: [PATCH 605/830] show heroes if a room has a 'deleted'
 name/canonical_alias (#3851)

---
 changelog.d/3851.bugfix  |  1 +
 synapse/handlers/sync.py | 17 +++++++++++++++--
 2 files changed, 16 insertions(+), 2 deletions(-)
 create mode 100644 changelog.d/3851.bugfix

diff --git a/changelog.d/3851.bugfix b/changelog.d/3851.bugfix
new file mode 100644
index 000000000000..b53a9efe7b24
--- /dev/null
+++ b/changelog.d/3851.bugfix
@@ -0,0 +1 @@
+Show heroes if room name/canonical alias has been deleted
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 45a1dd5952c3..9bca4e70670d 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -562,8 +562,21 @@ def compute_summary(self, room_id, sync_config, batch, state, now_token):
             details.get(Membership.INVITE, empty_ms).count
         )
 
-        if name_id or canonical_alias_id:
-            defer.returnValue(summary)
+        # if the room has a name or canonical_alias set, we can skip
+        # calculating heroes.  we assume that if the event has contents, it'll
+        # be a valid name or canonical_alias - i.e. we're checking that they
+        # haven't been "deleted" by blatting {} over the top.
+        if name_id:
+            name = yield self.store.get_event(name_id, allow_none=False)
+            if name and name.content:
+                defer.returnValue(summary)
+
+        if canonical_alias_id:
+            canonical_alias = yield self.store.get_event(
+                canonical_alias_id, allow_none=False,
+            )
+            if canonical_alias and canonical_alias.content:
+                defer.returnValue(summary)
 
         joined_user_ids = [
             r[0] for r in details.get(Membership.JOIN, empty_ms).members

From b7d2fb5eb9c55f95048444bb6703abbf2fe14c5c Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Thu, 13 Sep 2018 19:59:32 +1000
Subject: [PATCH 606/830] Remove some superfluous logging (#3855)

---
 .circleci/merge_base_branch.sh |  2 +-
 changelog.d/3855.misc          |  1 +
 synapse/__init__.py            | 10 ++++++++++
 synapse/config/logger.py       | 17 ++++++++++++++++-
 4 files changed, 28 insertions(+), 2 deletions(-)
 create mode 100644 changelog.d/3855.misc

diff --git a/.circleci/merge_base_branch.sh b/.circleci/merge_base_branch.sh
index 2d700dbf1189..4e297d77da07 100755
--- a/.circleci/merge_base_branch.sh
+++ b/.circleci/merge_base_branch.sh
@@ -4,7 +4,7 @@ set -e
 
 # CircleCI doesn't give CIRCLE_PR_NUMBER in the environment for non-forked PRs. Wonderful.
 # In this case, we just need to do some ~shell magic~ to strip it out of the PULL_REQUEST URL.
-echo 'export CIRCLE_PR_NUMBER="${CIRCLE_PR_NUMBER:-${CIRCLE_PULL_REQUEST##*/}}"' >> "$BASH_ENV"
+echo 'export CIRCLE_PR_NUMBER="${CIRCLE_PR_NUMBER:-${CIRCLE_PULL_REQUEST##*/}}"' >> $BASH_ENV
 source $BASH_ENV
 
 if [[ -z "${CIRCLE_PR_NUMBER}" ]]
diff --git a/changelog.d/3855.misc b/changelog.d/3855.misc
new file mode 100644
index 000000000000..a25bb020baaa
--- /dev/null
+++ b/changelog.d/3855.misc
@@ -0,0 +1 @@
+Removed some excess logging messages.
\ No newline at end of file
diff --git a/synapse/__init__.py b/synapse/__init__.py
index 65a2b894ccc1..9dbe0b9f10e6 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -17,4 +17,14 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
+try:
+    from twisted.internet import protocol
+    from twisted.internet.protocol import Factory
+    from twisted.names.dns import DNSDatagramProtocol
+    protocol.Factory.noisy = False
+    Factory.noisy = False
+    DNSDatagramProtocol.noisy = False
+except ImportError:
+    pass
+
 __version__ = "0.33.4"
diff --git a/synapse/config/logger.py b/synapse/config/logger.py
index 3f187adfc8cd..e9a936118d85 100644
--- a/synapse/config/logger.py
+++ b/synapse/config/logger.py
@@ -227,7 +227,22 @@ def sighup(signum, stack):
     #
     # However this may not be too much of a problem if we are just writing to a file.
     observer = STDLibLogObserver()
+
+    def _log(event):
+
+        if "log_text" in event:
+            if event["log_text"].startswith("DNSDatagramProtocol starting on "):
+                return
+
+            if event["log_text"].startswith("(UDP Port "):
+                return
+
+            if event["log_text"].startswith("Timing out client"):
+                return
+
+        return observer(event)
+
     globalLogBeginner.beginLoggingTo(
-        [observer],
+        [_log],
         redirectStandardIO=not config.no_redirect_stdio,
     )

From c857f5ef9b77b07179555a63973f11b8aa8c1752 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 13 Sep 2018 12:48:10 +0100
Subject: [PATCH 607/830] Make purge history slightly faster

Don't pull out events that are outliers and won't be deleted, as nothing
should happen to them.
---
 synapse/storage/events.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 30ff87a4c464..1c8e01a47ea3 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -1930,15 +1930,22 @@ def _purge_history_txn(
         should_delete_params = ()
         if not delete_local_events:
             should_delete_expr += " AND event_id NOT LIKE ?"
-            should_delete_params += ("%:" + self.hs.hostname, )
+            should_delete_params += (
+                "%:" + self.hs.hostname,
+                "%:" + self.hs.hostname,
+            )
 
         should_delete_params += (room_id, token.topological)
 
+        # Note that we insert events that are outliers and aren't going to be
+        # deleted, as nothing will happen to them.
         txn.execute(
             "INSERT INTO events_to_purge"
             " SELECT event_id, %s"
             " FROM events AS e LEFT JOIN state_events USING (event_id)"
-            " WHERE e.room_id = ? AND topological_ordering < ?" % (
+            " WHERE (NOT outlier OR (%s)) AND e.room_id = ? AND topological_ordering < ?"
+            % (
+                should_delete_expr,
                 should_delete_expr,
             ),
             should_delete_params,

From e7cd7cb0f02f1f918387e4b3061b4a251c5d68fc Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 13 Sep 2018 12:55:40 +0100
Subject: [PATCH 608/830] Newsfile

---
 changelog.d/3856.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3856.misc

diff --git a/changelog.d/3856.misc b/changelog.d/3856.misc
new file mode 100644
index 000000000000..36c311eb3db9
--- /dev/null
+++ b/changelog.d/3856.misc
@@ -0,0 +1 @@
+Speed up purge history for rooms that have been previously purged

From 9dbe38ea7de66f6e5dfab1570dd3a5217a4883eb Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 13 Sep 2018 15:05:52 +0100
Subject: [PATCH 609/830] Create indices after insertion

---
 synapse/storage/events.py | 32 ++++++++++++++++++--------------
 1 file changed, 18 insertions(+), 14 deletions(-)

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 1c8e01a47ea3..622f2ababf1e 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -1890,20 +1890,6 @@ def _purge_history_txn(
             ")"
         )
 
-        # create an index on should_delete because later we'll be looking for
-        # the should_delete / shouldn't_delete subsets
-        txn.execute(
-            "CREATE INDEX events_to_purge_should_delete"
-            " ON events_to_purge(should_delete)",
-        )
-
-        # We do joins against events_to_purge for e.g. calculating state
-        # groups to purge, etc., so lets make an index.
-        txn.execute(
-            "CREATE INDEX events_to_purge_id"
-            " ON events_to_purge(event_id)",
-        )
-
         # First ensure that we're not about to delete all the forward extremeties
         txn.execute(
             "SELECT e.event_id, e.depth FROM events as e "
@@ -1950,6 +1936,24 @@ def _purge_history_txn(
             ),
             should_delete_params,
         )
+
+        # We create the indices *after* insertion as that's a lot faster.
+
+        # create an index on should_delete because later we'll be looking for
+        # the should_delete / shouldn't_delete subsets
+        txn.execute(
+            "CREATE INDEX events_to_purge_should_delete"
+            " ON events_to_purge(should_delete)",
+        )
+
+        # We do joins against events_to_purge for e.g. calculating state
+        # groups to purge, etc., so lets make an index.
+        txn.execute(
+            "CREATE INDEX events_to_purge_id"
+            " ON events_to_purge(event_id)",
+        )
+
+
         txn.execute(
             "SELECT event_id, should_delete FROM events_to_purge"
         )

From 9cbd0094f0dcf03676b3234896e73af80cb75021 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 13 Sep 2018 15:15:35 +0100
Subject: [PATCH 610/830] pep8

---
 synapse/storage/events.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 622f2ababf1e..bc1c18023b13 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -1953,7 +1953,6 @@ def _purge_history_txn(
             " ON events_to_purge(event_id)",
         )
 
-
         txn.execute(
             "SELECT event_id, should_delete FROM events_to_purge"
         )

From bfa0b759e02c74931606ea77f34bec99dfb10589 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Fri, 14 Sep 2018 00:15:51 +1000
Subject: [PATCH 611/830] Attempt to figure out what's going on with timeouts
 (#3857)

---
 changelog.d/3857.misc                  |   1 +
 synapse/http/matrixfederationclient.py |  98 +++++++--------
 tests/http/test_fedclient.py           | 157 +++++++++++++++++++++++++
 tests/server.py                        |  42 ++++++-
 4 files changed, 241 insertions(+), 57 deletions(-)
 create mode 100644 changelog.d/3857.misc
 create mode 100644 tests/http/test_fedclient.py

diff --git a/changelog.d/3857.misc b/changelog.d/3857.misc
new file mode 100644
index 000000000000..e128d193d9a3
--- /dev/null
+++ b/changelog.d/3857.misc
@@ -0,0 +1 @@
+Refactor some HTTP timeout code.
\ No newline at end of file
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index cf920bc04144..c49dbacd9371 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -26,7 +26,7 @@
 from prometheus_client import Counter
 from signedjson.sign import sign_json
 
-from twisted.internet import defer, protocol, reactor
+from twisted.internet import defer, protocol
 from twisted.internet.error import DNSLookupError
 from twisted.web._newclient import ResponseDone
 from twisted.web.client import Agent, HTTPConnectionPool
@@ -40,10 +40,8 @@
     HttpResponseException,
     SynapseError,
 )
-from synapse.http import cancelled_to_request_timed_out_error
 from synapse.http.endpoint import matrix_federation_endpoint
 from synapse.util import logcontext
-from synapse.util.async_helpers import add_timeout_to_deferred
 from synapse.util.logcontext import make_deferred_yieldable
 
 logger = logging.getLogger(__name__)
@@ -66,13 +64,14 @@
 
 class MatrixFederationEndpointFactory(object):
     def __init__(self, hs):
+        self.reactor = hs.get_reactor()
         self.tls_client_options_factory = hs.tls_client_options_factory
 
     def endpointForURI(self, uri):
         destination = uri.netloc.decode('ascii')
 
         return matrix_federation_endpoint(
-            reactor, destination, timeout=10,
+            self.reactor, destination, timeout=10,
             tls_client_options_factory=self.tls_client_options_factory
         )
 
@@ -90,6 +89,7 @@ def __init__(self, hs):
         self.hs = hs
         self.signing_key = hs.config.signing_key[0]
         self.server_name = hs.hostname
+        reactor = hs.get_reactor()
         pool = HTTPConnectionPool(reactor)
         pool.maxPersistentPerHost = 5
         pool.cachedConnectionTimeout = 2 * 60
@@ -100,6 +100,7 @@ def __init__(self, hs):
         self._store = hs.get_datastore()
         self.version_string = hs.version_string.encode('ascii')
         self._next_id = 1
+        self.default_timeout = 60
 
     def _create_url(self, destination, path_bytes, param_bytes, query_bytes):
         return urllib.parse.urlunparse(
@@ -143,6 +144,11 @@ def _request(self, destination, method, path,
             (May also fail with plenty of other Exceptions for things like DNS
                 failures, connection failures, SSL failures.)
         """
+        if timeout:
+            _sec_timeout = timeout / 1000
+        else:
+            _sec_timeout = self.default_timeout
+
         if (
             self.hs.config.federation_domain_whitelist is not None and
             destination not in self.hs.config.federation_domain_whitelist
@@ -215,13 +221,9 @@ def _request(self, destination, method, path,
                         headers=Headers(headers_dict),
                         data=data,
                         agent=self.agent,
+                        reactor=self.hs.get_reactor()
                     )
-                    add_timeout_to_deferred(
-                        request_deferred,
-                        timeout / 1000. if timeout else 60,
-                        self.hs.get_reactor(),
-                        cancelled_to_request_timed_out_error,
-                    )
+                    request_deferred.addTimeout(_sec_timeout, self.hs.get_reactor())
                     response = yield make_deferred_yieldable(
                         request_deferred,
                     )
@@ -261,6 +263,13 @@ def _request(self, destination, method, path,
                             delay = min(delay, 2)
                             delay *= random.uniform(0.8, 1.4)
 
+                        logger.debug(
+                            "{%s} Waiting %s before sending to %s...",
+                            txn_id,
+                            delay,
+                            destination
+                        )
+
                         yield self.clock.sleep(delay)
                         retries_left -= 1
                     else:
@@ -279,10 +288,9 @@ def _request(self, destination, method, path,
                 # :'(
                 # Update transactions table?
                 with logcontext.PreserveLoggingContext():
-                    body = yield self._timeout_deferred(
-                        treq.content(response),
-                        timeout,
-                    )
+                    d = treq.content(response)
+                    d.addTimeout(_sec_timeout, self.hs.get_reactor())
+                    body = yield make_deferred_yieldable(d)
                 raise HttpResponseException(
                     response.code, response.phrase, body
                 )
@@ -396,10 +404,9 @@ def put_json(self, destination, path, args={}, data={},
             check_content_type_is_json(response.headers)
 
         with logcontext.PreserveLoggingContext():
-            body = yield self._timeout_deferred(
-                treq.json_content(response),
-                timeout,
-            )
+            d = treq.json_content(response)
+            d.addTimeout(self.default_timeout, self.hs.get_reactor())
+            body = yield make_deferred_yieldable(d)
         defer.returnValue(body)
 
     @defer.inlineCallbacks
@@ -449,10 +456,14 @@ def post_json(self, destination, path, data={}, long_retries=False,
             check_content_type_is_json(response.headers)
 
         with logcontext.PreserveLoggingContext():
-            body = yield self._timeout_deferred(
-                treq.json_content(response),
-                timeout,
-            )
+            d = treq.json_content(response)
+            if timeout:
+                _sec_timeout = timeout / 1000
+            else:
+                _sec_timeout = self.default_timeout
+
+            d.addTimeout(_sec_timeout, self.hs.get_reactor())
+            body = yield make_deferred_yieldable(d)
 
         defer.returnValue(body)
 
@@ -504,10 +515,9 @@ def get_json(self, destination, path, args=None, retry_on_dns_fail=True,
             check_content_type_is_json(response.headers)
 
         with logcontext.PreserveLoggingContext():
-            body = yield self._timeout_deferred(
-                treq.json_content(response),
-                timeout,
-            )
+            d = treq.json_content(response)
+            d.addTimeout(self.default_timeout, self.hs.get_reactor())
+            body = yield make_deferred_yieldable(d)
 
         defer.returnValue(body)
 
@@ -554,10 +564,9 @@ def delete_json(self, destination, path, long_retries=False,
             check_content_type_is_json(response.headers)
 
         with logcontext.PreserveLoggingContext():
-            body = yield self._timeout_deferred(
-                treq.json_content(response),
-                timeout,
-            )
+            d = treq.json_content(response)
+            d.addTimeout(self.default_timeout, self.hs.get_reactor())
+            body = yield make_deferred_yieldable(d)
 
         defer.returnValue(body)
 
@@ -599,38 +608,15 @@ def get_file(self, destination, path, output_stream, args={},
 
         try:
             with logcontext.PreserveLoggingContext():
-                length = yield self._timeout_deferred(
-                    _readBodyToFile(
-                        response, output_stream, max_size
-                    ),
-                )
+                d = _readBodyToFile(response, output_stream, max_size)
+                d.addTimeout(self.default_timeout, self.hs.get_reactor())
+                length = yield make_deferred_yieldable(d)
         except Exception:
             logger.exception("Failed to download body")
             raise
 
         defer.returnValue((length, headers))
 
-    def _timeout_deferred(self, deferred, timeout_ms=None):
-        """Times the deferred out after `timeout_ms` ms
-
-        Args:
-            deferred (Deferred)
-            timeout_ms (int|None): Timeout in milliseconds. If None defaults
-                to 60 seconds.
-
-        Returns:
-            Deferred
-        """
-
-        add_timeout_to_deferred(
-            deferred,
-            timeout_ms / 1000. if timeout_ms else 60,
-            self.hs.get_reactor(),
-            cancelled_to_request_timed_out_error,
-        )
-
-        return deferred
-
 
 class _ReadBodyToFileProtocol(protocol.Protocol):
     def __init__(self, stream, deferred, max_size):
diff --git a/tests/http/test_fedclient.py b/tests/http/test_fedclient.py
new file mode 100644
index 000000000000..1c46c9cfeb9e
--- /dev/null
+++ b/tests/http/test_fedclient.py
@@ -0,0 +1,157 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from mock import Mock
+
+from twisted.internet.defer import TimeoutError
+from twisted.internet.error import ConnectingCancelledError, DNSLookupError
+from twisted.web.client import ResponseNeverReceived
+
+from synapse.http.matrixfederationclient import MatrixFederationHttpClient
+
+from tests.unittest import HomeserverTestCase
+
+
+class FederationClientTests(HomeserverTestCase):
+    def make_homeserver(self, reactor, clock):
+
+        hs = self.setup_test_homeserver(reactor=reactor, clock=clock)
+        hs.tls_client_options_factory = None
+        return hs
+
+    def prepare(self, reactor, clock, homeserver):
+
+        self.cl = MatrixFederationHttpClient(self.hs)
+        self.reactor.lookups["testserv"] = "1.2.3.4"
+
+    def test_dns_error(self):
+        """
+        If the DNS raising returns an error, it will bubble up.
+        """
+        d = self.cl._request("testserv2:8008", "GET", "foo/bar", timeout=10000)
+        self.pump()
+
+        f = self.failureResultOf(d)
+        self.assertIsInstance(f.value, DNSLookupError)
+
+    def test_client_never_connect(self):
+        """
+        If the HTTP request is not connected and is timed out, it'll give a
+        ConnectingCancelledError.
+        """
+        d = self.cl._request("testserv:8008", "GET", "foo/bar", timeout=10000)
+
+        self.pump()
+
+        # Nothing happened yet
+        self.assertFalse(d.called)
+
+        # Make sure treq is trying to connect
+        clients = self.reactor.tcpClients
+        self.assertEqual(len(clients), 1)
+        self.assertEqual(clients[0][0], '1.2.3.4')
+        self.assertEqual(clients[0][1], 8008)
+
+        # Deferred is still without a result
+        self.assertFalse(d.called)
+
+        # Push by enough to time it out
+        self.reactor.advance(10.5)
+        f = self.failureResultOf(d)
+
+        self.assertIsInstance(f.value, ConnectingCancelledError)
+
+    def test_client_connect_no_response(self):
+        """
+        If the HTTP request is connected, but gets no response before being
+        timed out, it'll give a ResponseNeverReceived.
+        """
+        d = self.cl._request("testserv:8008", "GET", "foo/bar", timeout=10000)
+
+        self.pump()
+
+        # Nothing happened yet
+        self.assertFalse(d.called)
+
+        # Make sure treq is trying to connect
+        clients = self.reactor.tcpClients
+        self.assertEqual(len(clients), 1)
+        self.assertEqual(clients[0][0], '1.2.3.4')
+        self.assertEqual(clients[0][1], 8008)
+
+        conn = Mock()
+        client = clients[0][2].buildProtocol(None)
+        client.makeConnection(conn)
+
+        # Deferred is still without a result
+        self.assertFalse(d.called)
+
+        # Push by enough to time it out
+        self.reactor.advance(10.5)
+        f = self.failureResultOf(d)
+
+        self.assertIsInstance(f.value, ResponseNeverReceived)
+
+    def test_client_gets_headers(self):
+        """
+        Once the client gets the headers, _request returns successfully.
+        """
+        d = self.cl._request("testserv:8008", "GET", "foo/bar", timeout=10000)
+
+        self.pump()
+
+        conn = Mock()
+        clients = self.reactor.tcpClients
+        client = clients[0][2].buildProtocol(None)
+        client.makeConnection(conn)
+
+        # Deferred does not have a result
+        self.assertFalse(d.called)
+
+        # Send it the HTTP response
+        client.dataReceived(b"HTTP/1.1 200 OK\r\nServer: Fake\r\n\r\n")
+
+        # We should get a successful response
+        r = self.successResultOf(d)
+        self.assertEqual(r.code, 200)
+
+    def test_client_headers_no_body(self):
+        """
+        If the HTTP request is connected, but gets no response before being
+        timed out, it'll give a ResponseNeverReceived.
+        """
+        d = self.cl.post_json("testserv:8008", "foo/bar", timeout=10000)
+
+        self.pump()
+
+        conn = Mock()
+        clients = self.reactor.tcpClients
+        client = clients[0][2].buildProtocol(None)
+        client.makeConnection(conn)
+
+        # Deferred does not have a result
+        self.assertFalse(d.called)
+
+        # Send it the HTTP response
+        client.dataReceived(
+            (b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n"
+             b"Server: Fake\r\n\r\n")
+        )
+
+        # Push by enough to time it out
+        self.reactor.advance(10.5)
+        f = self.failureResultOf(d)
+
+        self.assertIsInstance(f.value, TimeoutError)
diff --git a/tests/server.py b/tests/server.py
index a2c3ca61f66d..420ec4e08859 100644
--- a/tests/server.py
+++ b/tests/server.py
@@ -4,9 +4,14 @@
 from six import text_type
 
 import attr
+from zope.interface import implementer
 
-from twisted.internet import address, threads
+from twisted.internet import address, threads, udp
+from twisted.internet._resolver import HostResolution
+from twisted.internet.address import IPv4Address
 from twisted.internet.defer import Deferred
+from twisted.internet.error import DNSLookupError
+from twisted.internet.interfaces import IReactorPluggableNameResolver
 from twisted.python.failure import Failure
 from twisted.test.proto_helpers import MemoryReactorClock
 
@@ -154,11 +159,46 @@ def render(request, resource, clock):
     wait_until_result(clock, request)
 
 
+@implementer(IReactorPluggableNameResolver)
 class ThreadedMemoryReactorClock(MemoryReactorClock):
     """
     A MemoryReactorClock that supports callFromThread.
     """
 
+    def __init__(self):
+        self._udp = []
+        self.lookups = {}
+
+        class Resolver(object):
+            def resolveHostName(
+                _self,
+                resolutionReceiver,
+                hostName,
+                portNumber=0,
+                addressTypes=None,
+                transportSemantics='TCP',
+            ):
+
+                resolution = HostResolution(hostName)
+                resolutionReceiver.resolutionBegan(resolution)
+                if hostName not in self.lookups:
+                    raise DNSLookupError("OH NO")
+
+                resolutionReceiver.addressResolved(
+                    IPv4Address('TCP', self.lookups[hostName], portNumber)
+                )
+                resolutionReceiver.resolutionComplete()
+                return resolution
+
+        self.nameResolver = Resolver()
+        super(ThreadedMemoryReactorClock, self).__init__()
+
+    def listenUDP(self, port, protocol, interface='', maxPacketSize=8196):
+        p = udp.Port(port, protocol, interface, maxPacketSize, self)
+        p.startListening()
+        self._udp.append(p)
+        return p
+
     def callFromThread(self, callback, *args, **kwargs):
         """
         Make the callback fire in the next reactor iteration.

From 89a76d18898a0172010c25e5a7c66d57c16b9196 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 13 Sep 2018 15:33:16 +0100
Subject: [PATCH 612/830] Fix handling of redacted events from federation

If we receive an event that doesn't pass their content hash check (e.g.
due to already being redacted) then we hit a bug which causes an
exception to be raised, which then promplty stops the event (and
request) from being processed.

This effects all sorts of federation APIs, including joining rooms with
a redacted state event.
---
 synapse/events/__init__.py            | 5 +++++
 synapse/federation/federation_base.py | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py
index 51f9084b9026..b782af630807 100644
--- a/synapse/events/__init__.py
+++ b/synapse/events/__init__.py
@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import six
+
 from synapse.util.caches import intern_dict
 from synapse.util.frozenutils import freeze
 
@@ -147,6 +149,9 @@ def __contains__(self, field):
     def items(self):
         return list(self._event_dict.items())
 
+    def keys(self):
+        return six.iterkeys(self._event_dict)
+
 
 class FrozenEvent(EventBase):
     def __init__(self, event_dict, internal_metadata_dict={}, rejected_reason=None):
diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py
index 61782ae1c011..b7ad729c634e 100644
--- a/synapse/federation/federation_base.py
+++ b/synapse/federation/federation_base.py
@@ -153,7 +153,7 @@ def callback(_, pdu):
                     # *actual* redacted copy to be on the safe side.)
                     redacted_event = prune_event(pdu)
                     if (
-                        set(six.iterkeys(redacted_event)) == set(six.iterkeys(pdu)) and
+                        set(redacted_event.keys()) == set(pdu.keys()) and
                         set(six.iterkeys(redacted_event.content))
                             == set(six.iterkeys(pdu.content))
                     ):

From 3126b88d35994b09096239462abbbbcd17d9cbf0 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Fri, 14 Sep 2018 00:44:31 +1000
Subject: [PATCH 613/830] fix circleci merged builds (#3858)

* fix

* changelog
---
 .circleci/merge_base_branch.sh | 4 ++++
 changelog.d/3858.misc          | 1 +
 2 files changed, 5 insertions(+)
 create mode 100644 changelog.d/3858.misc

diff --git a/.circleci/merge_base_branch.sh b/.circleci/merge_base_branch.sh
index 4e297d77da07..9614eb91b6fb 100755
--- a/.circleci/merge_base_branch.sh
+++ b/.circleci/merge_base_branch.sh
@@ -19,6 +19,10 @@ GITBASE=`curl -q https://api.github.com/repos/matrix-org/synapse/pulls/${CIRCLE_
 # Show what we are before
 git show -s
 
+# Set up username so it can do a merge
+git config --global user.email bot@matrix.org
+git config --global user.name "A robot"
+
 # Fetch and merge. If it doesn't work, it will raise due to set -e.
 git fetch -u origin $GITBASE
 git merge --no-edit origin/$GITBASE
diff --git a/changelog.d/3858.misc b/changelog.d/3858.misc
new file mode 100644
index 000000000000..4644db5330fc
--- /dev/null
+++ b/changelog.d/3858.misc
@@ -0,0 +1 @@
+Fix running merged builds on CircleCI
\ No newline at end of file

From 13193a6e2bd3557cae89e34438e8598f75276c34 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 13 Sep 2018 15:46:45 +0100
Subject: [PATCH 614/830] Newsfile

---
 changelog.d/3859.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3859.bugfix

diff --git a/changelog.d/3859.bugfix b/changelog.d/3859.bugfix
new file mode 100644
index 000000000000..ec5b172464e4
--- /dev/null
+++ b/changelog.d/3859.bugfix
@@ -0,0 +1 @@
+Fix handling of redacted events from federation

From ed5331a6272280efd4fc98ae425711acec8c6be5 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 13 Sep 2018 16:10:56 +0100
Subject: [PATCH 615/830] comment

---
 synapse/storage/events.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index bc1c18023b13..e7487311ce9b 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -1916,6 +1916,8 @@ def _purge_history_txn(
         should_delete_params = ()
         if not delete_local_events:
             should_delete_expr += " AND event_id NOT LIKE ?"
+
+            # We include the parameter twice since we use the expression twice
             should_delete_params += (
                 "%:" + self.hs.hostname,
                 "%:" + self.hs.hostname,

From bc74925c5b94f0c02603297d4ccb7e05008d5124 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Thu, 13 Sep 2018 17:02:59 +0100
Subject: [PATCH 616/830] WIP e2e key backups

Continues from uhoreg's branch

This just fixed the errcode on /room_keys/version if no backup and
updates the schema delta to be on the latest so it gets run
---
 synapse/rest/client/v2_alpha/room_keys.py          | 14 ++++++++++----
 .../schema/delta/{46 => 51}/e2e_room_keys.sql      |  0
 2 files changed, 10 insertions(+), 4 deletions(-)
 rename synapse/storage/schema/delta/{46 => 51}/e2e_room_keys.sql (100%)

diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py
index 1ed18e986fa4..ea114bc8b48e 100644
--- a/synapse/rest/client/v2_alpha/room_keys.py
+++ b/synapse/rest/client/v2_alpha/room_keys.py
@@ -17,7 +17,7 @@
 
 from twisted.internet import defer
 
-from synapse.api.errors import SynapseError
+from synapse.api.errors import SynapseError, Codes
 from synapse.http.servlet import (
     RestServlet,
     parse_json_object_from_request,
@@ -324,9 +324,15 @@ def on_GET(self, request, version):
         requester = yield self.auth.get_user_by_req(request, allow_guest=False)
         user_id = requester.user.to_string()
 
-        info = yield self.e2e_room_keys_handler.get_version_info(
-            user_id, version
-        )
+        try:
+            info = yield self.e2e_room_keys_handler.get_version_info(
+                user_id, version
+            )
+        except SynapseError as e:
+            if e.code == 404:
+                e.errcode = Codes.NOT_FOUND
+                e.msg = "No backup found"
+                raise e
         defer.returnValue((200, info))
 
     @defer.inlineCallbacks
diff --git a/synapse/storage/schema/delta/46/e2e_room_keys.sql b/synapse/storage/schema/delta/51/e2e_room_keys.sql
similarity index 100%
rename from synapse/storage/schema/delta/46/e2e_room_keys.sql
rename to synapse/storage/schema/delta/51/e2e_room_keys.sql

From 1c3f4d9ca53b3e3a5ef1849a8da68c6d8cf82938 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Fri, 14 Sep 2018 03:09:13 +1000
Subject: [PATCH 617/830] buffer?

---
 synapse/http/matrixfederationclient.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index c49dbacd9371..cdbe27eb37d0 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -221,7 +221,8 @@ def _request(self, destination, method, path,
                         headers=Headers(headers_dict),
                         data=data,
                         agent=self.agent,
-                        reactor=self.hs.get_reactor()
+                        reactor=self.hs.get_reactor(),
+                        unbuffered=True
                     )
                     request_deferred.addTimeout(_sec_timeout, self.hs.get_reactor())
                     response = yield make_deferred_yieldable(

From 7c27c4d51cfd2a3d6504889b1ccdddabf38ae05c Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Fri, 14 Sep 2018 03:11:11 +1000
Subject: [PATCH 618/830] merge (#3576)

---
 .circleci/config.yml                         | 17 +++++---
 .travis.yml                                  |  3 ++
 changelog.d/3576.feature                     |  1 +
 synapse/http/client.py                       | 14 +++++--
 synapse/push/httppusher.py                   |  7 +++-
 synapse/push/mailer.py                       |  7 ++--
 synapse/replication/slave/storage/devices.py | 23 ++++++----
 tox.ini                                      | 44 ++------------------
 8 files changed, 55 insertions(+), 61 deletions(-)
 create mode 100644 changelog.d/3576.feature

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 5266544f3c37..605430fb3f09 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -53,7 +53,7 @@ jobs:
     steps:
       - checkout
       - run: docker pull matrixdotorg/sytest-synapsepy3
-      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs hawkowl/sytestpy3
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy3
       - store_artifacts:
           path: ~/project/logs
           destination: logs
@@ -76,7 +76,7 @@ jobs:
       - checkout
       - run: bash .circleci/merge_base_branch.sh
       - run: docker pull matrixdotorg/sytest-synapsepy3
-      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs hawkowl/sytestpy3
+      - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy3
       - store_artifacts:
           path: ~/project/logs
           destination: logs
@@ -101,6 +101,8 @@ workflows:
     jobs:
       - sytestpy2
       - sytestpy2postgres
+      - sytestpy3
+      - sytestpy3postgres
       - sytestpy2merged:
           filters:
             branches:
@@ -109,6 +111,11 @@ workflows:
           filters:
             branches:
               ignore: /develop|master/
-# Currently broken while the Python 3 port is incomplete
-#      - sytestpy3
-#      - sytestpy3postgres
+      - sytestpy3merged:
+          filters:
+            branches:
+              ignore: /develop|master/
+      - sytestpy3postgresmerged:
+          filters:
+            branches:
+              ignore: /develop|master/
diff --git a/.travis.yml b/.travis.yml
index ebc972ed2492..b3ee66da8ff1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -25,6 +25,9 @@ matrix:
     services:
       - postgresql
 
+  - python: 3.5
+    env: TOX_ENV=py35
+
   - python: 3.6
     env: TOX_ENV=py36
 
diff --git a/changelog.d/3576.feature b/changelog.d/3576.feature
new file mode 100644
index 000000000000..02a10e370da2
--- /dev/null
+++ b/changelog.d/3576.feature
@@ -0,0 +1 @@
+Python 3.5+ is now supported.
diff --git a/synapse/http/client.py b/synapse/http/client.py
index 4ba54fed05ef..d60f87b04ea0 100644
--- a/synapse/http/client.py
+++ b/synapse/http/client.py
@@ -348,7 +348,8 @@ def get_file(self, url, output_stream, max_size=None, headers=None):
 
         resp_headers = dict(response.headers.getAllRawHeaders())
 
-        if 'Content-Length' in resp_headers and resp_headers['Content-Length'] > max_size:
+        if (b'Content-Length' in resp_headers and
+                int(resp_headers[b'Content-Length']) > max_size):
             logger.warn("Requested URL is too large > %r bytes" % (self.max_size,))
             raise SynapseError(
                 502,
@@ -381,7 +382,12 @@ def get_file(self, url, output_stream, max_size=None, headers=None):
             )
 
         defer.returnValue(
-            (length, resp_headers, response.request.absoluteURI, response.code),
+            (
+                length,
+                resp_headers,
+                response.request.absoluteURI.decode('ascii'),
+                response.code,
+            ),
         )
 
 
@@ -466,9 +472,9 @@ def __init__(self, hs):
     def endpointForURI(self, uri):
         logger.info("Getting endpoint for %s", uri.toBytes())
 
-        if uri.scheme == "http":
+        if uri.scheme == b"http":
             endpoint_factory = HostnameEndpoint
-        elif uri.scheme == "https":
+        elif uri.scheme == b"https":
             tlsCreator = self.policyForHTTPS.creatorForNetloc(uri.host, uri.port)
 
             def endpoint_factory(reactor, host, port, **kw):
diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py
index 81e18bcf7d98..48abd5e4d689 100644
--- a/synapse/push/httppusher.py
+++ b/synapse/push/httppusher.py
@@ -15,6 +15,8 @@
 # limitations under the License.
 import logging
 
+import six
+
 from prometheus_client import Counter
 
 from twisted.internet import defer
@@ -26,6 +28,9 @@
 
 from . import push_rule_evaluator, push_tools
 
+if six.PY3:
+    long = int
+
 logger = logging.getLogger(__name__)
 
 http_push_processed_counter = Counter("synapse_http_httppusher_http_pushes_processed", "")
@@ -96,7 +101,7 @@ def on_started(self):
 
     @defer.inlineCallbacks
     def on_new_notifications(self, min_stream_ordering, max_stream_ordering):
-        self.max_stream_ordering = max(max_stream_ordering, self.max_stream_ordering)
+        self.max_stream_ordering = max(max_stream_ordering, self.max_stream_ordering or 0)
         yield self._process()
 
     @defer.inlineCallbacks
diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py
index bfa6df7b68b8..b78ce10396b7 100644
--- a/synapse/push/mailer.py
+++ b/synapse/push/mailer.py
@@ -17,10 +17,11 @@
 import email.utils
 import logging
 import time
-import urllib
 from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
 
+from six.moves import urllib
+
 import bleach
 import jinja2
 
@@ -474,7 +475,7 @@ def make_unsubscribe_link(self, user_id, app_id, email_address):
         # XXX: make r0 once API is stable
         return "%s_matrix/client/unstable/pushers/remove?%s" % (
             self.hs.config.public_baseurl,
-            urllib.urlencode(params),
+            urllib.parse.urlencode(params),
         )
 
 
@@ -561,7 +562,7 @@ def mxc_to_http_filter(value, width, height, resize_method="crop"):
         return "%s_matrix/media/v1/thumbnail/%s?%s%s" % (
             config.public_baseurl,
             serverAndMediaId,
-            urllib.urlencode(params),
+            urllib.parse.urlencode(params),
             fragment or "",
         )
 
diff --git a/synapse/replication/slave/storage/devices.py b/synapse/replication/slave/storage/devices.py
index 8206a988f793..21b8c468facd 100644
--- a/synapse/replication/slave/storage/devices.py
+++ b/synapse/replication/slave/storage/devices.py
@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import six
+
 from synapse.storage import DataStore
 from synapse.storage.end_to_end_keys import EndToEndKeyStore
 from synapse.util.caches.stream_change_cache import StreamChangeCache
@@ -21,6 +23,13 @@
 from ._slaved_id_tracker import SlavedIdTracker
 
 
+def __func__(inp):
+    if six.PY3:
+        return inp
+    else:
+        return inp.__func__
+
+
 class SlavedDeviceStore(BaseSlavedStore):
     def __init__(self, db_conn, hs):
         super(SlavedDeviceStore, self).__init__(db_conn, hs)
@@ -38,14 +47,14 @@ def __init__(self, db_conn, hs):
             "DeviceListFederationStreamChangeCache", device_list_max,
         )
 
-    get_device_stream_token = DataStore.get_device_stream_token.__func__
-    get_user_whose_devices_changed = DataStore.get_user_whose_devices_changed.__func__
-    get_devices_by_remote = DataStore.get_devices_by_remote.__func__
-    _get_devices_by_remote_txn = DataStore._get_devices_by_remote_txn.__func__
-    _get_e2e_device_keys_txn = DataStore._get_e2e_device_keys_txn.__func__
-    mark_as_sent_devices_by_remote = DataStore.mark_as_sent_devices_by_remote.__func__
+    get_device_stream_token = __func__(DataStore.get_device_stream_token)
+    get_user_whose_devices_changed = __func__(DataStore.get_user_whose_devices_changed)
+    get_devices_by_remote = __func__(DataStore.get_devices_by_remote)
+    _get_devices_by_remote_txn = __func__(DataStore._get_devices_by_remote_txn)
+    _get_e2e_device_keys_txn = __func__(DataStore._get_e2e_device_keys_txn)
+    mark_as_sent_devices_by_remote = __func__(DataStore.mark_as_sent_devices_by_remote)
     _mark_as_sent_devices_by_remote_txn = (
-        DataStore._mark_as_sent_devices_by_remote_txn.__func__
+        __func__(DataStore._mark_as_sent_devices_by_remote_txn)
     )
     count_e2e_one_time_keys = EndToEndKeyStore.__dict__["count_e2e_one_time_keys"]
 
diff --git a/tox.ini b/tox.ini
index 085f43898908..80ac9324df76 100644
--- a/tox.ini
+++ b/tox.ini
@@ -64,49 +64,11 @@ setenv =
     {[base]setenv}
     SYNAPSE_POSTGRES = 1
 
+[testenv:py35]
+usedevelop=true
+
 [testenv:py36]
 usedevelop=true
-commands =
-    /usr/bin/find "{toxinidir}" -name '*.pyc' -delete
-    coverage run {env:COVERAGE_OPTS:} --source="{toxinidir}/synapse" \
-        "{envbindir}/trial" {env:TRIAL_FLAGS:} {posargs:tests/config \
-		tests/api/test_filtering.py \
-		tests/api/test_ratelimiting.py \
-		tests/appservice \
-		tests/crypto \
-		tests/events \
-		tests/handlers/test_appservice.py \
-		tests/handlers/test_auth.py \
-		tests/handlers/test_device.py \
-		tests/handlers/test_directory.py \
-		tests/handlers/test_e2e_keys.py \
-		tests/handlers/test_presence.py \
-		tests/handlers/test_profile.py \
-		tests/handlers/test_register.py \
-		tests/replication/slave/storage/test_account_data.py \
-		tests/replication/slave/storage/test_receipts.py \
-		tests/storage/test_appservice.py \
-		tests/storage/test_background_update.py \
-		tests/storage/test_base.py \
-		tests/storage/test__base.py \
-		tests/storage/test_client_ips.py \
-		tests/storage/test_devices.py \
-		tests/storage/test_end_to_end_keys.py \
-		tests/storage/test_event_push_actions.py \
-		tests/storage/test_keys.py \
-		tests/storage/test_presence.py \
-		tests/storage/test_profile.py \
-		tests/storage/test_registration.py \
-		tests/storage/test_room.py \
-		tests/storage/test_user_directory.py \
-		tests/test_distributor.py \
-		tests/test_dns.py \
-		tests/test_preview.py \
-		tests/test_test_utils.py \
-		tests/test_types.py \
-		tests/util} \
-        {env:TOXSUFFIX:}
-    {env:DUMP_COVERAGE_COMMAND:coverage report -m}
 
 [testenv:packaging]
 deps =

From 63755fa4c25f120941aba24d6192815313d12ea7 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Fri, 14 Sep 2018 03:21:47 +1000
Subject: [PATCH 619/830] we do that higher up

---
 synapse/http/matrixfederationclient.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index cdbe27eb37d0..d1a6e877063d 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -91,6 +91,7 @@ def __init__(self, hs):
         self.server_name = hs.hostname
         reactor = hs.get_reactor()
         pool = HTTPConnectionPool(reactor)
+        pool.retryAutomatically = False
         pool.maxPersistentPerHost = 5
         pool.cachedConnectionTimeout = 2 * 60
         self.agent = Agent.usingEndpointFactory(

From 7c33ab76da91a747fd442f65c9bfe187a81864f7 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Fri, 14 Sep 2018 03:45:34 +1000
Subject: [PATCH 620/830] redact better

---
 synapse/http/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/synapse/http/__init__.py b/synapse/http/__init__.py
index 58ef8d3ce461..a3f9e4f67ce4 100644
--- a/synapse/http/__init__.py
+++ b/synapse/http/__init__.py
@@ -38,12 +38,12 @@ def cancelled_to_request_timed_out_error(value, timeout):
     return value
 
 
-ACCESS_TOKEN_RE = re.compile(br'(\?.*access(_|%5[Ff])token=)[^&]*(.*)$')
+ACCESS_TOKEN_RE = re.compile(r'(\?.*access(_|%5[Ff])token=)[^&]*(.*)$')
 
 
 def redact_uri(uri):
     """Strips access tokens from the uri replaces with <redacted>"""
     return ACCESS_TOKEN_RE.sub(
-        br'\1<redacted>\3',
+        r'\1<redacted>\3',
         uri
     )

From f1a72646636c08e5e0f6dc6f583d0bffebf94c85 Mon Sep 17 00:00:00 2001
From: Travis Ralston <travpc@gmail.com>
Date: Thu, 13 Sep 2018 11:51:12 -0600
Subject: [PATCH 621/830] Fix minor typo in exception

---
 synapse/replication/tcp/streams.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/replication/tcp/streams.py b/synapse/replication/tcp/streams.py
index 55fe701c5c8d..c1e626be3f5f 100644
--- a/synapse/replication/tcp/streams.py
+++ b/synapse/replication/tcp/streams.py
@@ -196,7 +196,7 @@ def get_updates_since(self, from_token):
             )
 
             if len(rows) >= MAX_EVENTS_BEHIND:
-                raise Exception("stream %s has fallen behined" % (self.NAME))
+                raise Exception("stream %s has fallen behind" % (self.NAME))
         else:
             rows = yield self.update_function(
                 from_token, current_token,

From 8f08d848f53bc79cdd47f8d4025313268a16b227 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Fri, 14 Sep 2018 03:53:56 +1000
Subject: [PATCH 622/830] fix

---
 synapse/http/site.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/http/site.py b/synapse/http/site.py
index f0828c654243..18f2d37e00a7 100644
--- a/synapse/http/site.py
+++ b/synapse/http/site.py
@@ -85,7 +85,7 @@ def get_request_id(self):
         return "%s-%i" % (self.method, self.request_seq)
 
     def get_redacted_uri(self):
-        return redact_uri(self.uri)
+        return redact_uri(self.uri.decode('ascii'))
 
     def get_user_agent(self):
         return self.requestHeaders.getRawHeaders(b"User-Agent", [None])[-1]

From c971aa7b9d5b4d3900404a285f34dd5004a76f37 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Fri, 14 Sep 2018 03:57:02 +1000
Subject: [PATCH 623/830] fix

---
 synapse/http/site.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/synapse/http/site.py b/synapse/http/site.py
index 18f2d37e00a7..e1e53e8ae5f7 100644
--- a/synapse/http/site.py
+++ b/synapse/http/site.py
@@ -85,7 +85,10 @@ def get_request_id(self):
         return "%s-%i" % (self.method, self.request_seq)
 
     def get_redacted_uri(self):
-        return redact_uri(self.uri.decode('ascii'))
+        uri = self.uri
+        if isinstance(uri, bytes):
+            uri = self.uri.decode('ascii')
+        return redact_uri(uri)
 
     def get_user_agent(self):
         return self.requestHeaders.getRawHeaders(b"User-Agent", [None])[-1]

From 984db8bb0858757884c0f742bdaa27ae044d2c85 Mon Sep 17 00:00:00 2001
From: Travis Ralston <travpc@gmail.com>
Date: Thu, 13 Sep 2018 12:06:04 -0600
Subject: [PATCH 624/830] Create 3860.misc

---
 changelog.d/3860.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3860.misc

diff --git a/changelog.d/3860.misc b/changelog.d/3860.misc
new file mode 100644
index 000000000000..364239d3e33e
--- /dev/null
+++ b/changelog.d/3860.misc
@@ -0,0 +1 @@
+Fix typo in replication stream exception.

From 7de1989ea2809c13b67be6c4990b269d817fae44 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Thu, 13 Sep 2018 22:43:50 +0100
Subject: [PATCH 625/830] fix link for case that config.email_riot_base_url is
 set

---
 synapse/push/mailer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py
index b78ce10396b7..1a5a10d97476 100644
--- a/synapse/push/mailer.py
+++ b/synapse/push/mailer.py
@@ -441,7 +441,7 @@ def make_summary_text(self, notifs_by_room, room_state_ids,
 
     def make_room_link(self, room_id):
         if self.hs.config.email_riot_base_url:
-            base_url = self.hs.config.email_riot_base_url
+            base_url = "%s/#/room" % (self.hs.config.email_riot_base_url)
         elif self.app_name == "Vector":
             # need /beta for Universal Links to work on iOS
             base_url = "https://vector.im/beta/#/room"

From bef1d6f3bfdda4b07b7c72d284a6c79aed9bc93c Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Thu, 13 Sep 2018 22:53:35 +0100
Subject: [PATCH 626/830] towncrier

---
 changelog.d/3868.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3868.bugfix

diff --git a/changelog.d/3868.bugfix b/changelog.d/3868.bugfix
new file mode 100644
index 000000000000..99da8389a5e7
--- /dev/null
+++ b/changelog.d/3868.bugfix
@@ -0,0 +1 @@
+Fix broken invite email links for self hosted riots 

From 0a81038ea0e308d2944b7deae43060e4982b0abe Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 14 Sep 2018 14:39:59 +0100
Subject: [PATCH 627/830] Add in flight real time metrics for Measure blocks

---
 synapse/metrics/__init__.py | 110 +++++++++++++++++++++++++++++++++++-
 synapse/util/metrics.py     |  22 ++++++++
 2 files changed, 131 insertions(+), 1 deletion(-)

diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index 550f8443f78a..98333b20489f 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -18,8 +18,11 @@
 import logging
 import os
 import platform
+import threading
 import time
 
+import six
+
 import attr
 from prometheus_client import Counter, Gauge, Histogram
 from prometheus_client.core import REGISTRY, GaugeMetricFamily
@@ -68,7 +71,7 @@ def collect(self):
             return
 
         if isinstance(calls, dict):
-            for k, v in calls.items():
+            for k, v in six.iteritems(calls):
                 g.add_metric(k, v)
         else:
             g.add_metric([], calls)
@@ -87,6 +90,111 @@ def _register(self):
         all_gauges[self.name] = self
 
 
+class InFlightGauge(object):
+    """Tracks number of things (e.g. requests, Measure blocks, etc) in flight
+    at any given time.
+
+    Each InFlightGauge will create a metric called `<name>_total` that counts
+    the number of in flight blocks, as well as a metrics for each item in the
+    given `sub_metrics` as `<name>_<sub_metric>` which will get updated by the
+    callbacks.
+
+    Args:
+        name (str)
+        desc (str)
+        labels (list[str])
+        sub_metrics (list[str]): A list of sub metrics that the callbacks
+            will update.
+    """
+
+    # TODO: Expand this to
+
+    def __init__(self, name, desc, labels, sub_metrics):
+        self.name = name
+        self.desc = desc
+        self.labels = labels
+        self.sub_metrics = sub_metrics
+
+        # Create a class which have the sub_metrics values as attributes, which
+        # default to 0 on initialization. Used to pass to registered callbacks.
+        self._metrics_class = attr.make_class(
+            "_MetricsEntry",
+            attrs={x: attr.ib(0) for x in sub_metrics},
+            slots=True,
+        )
+
+        # Counts number of in flight blocks for a given set of label values
+        self._registrations = {}
+
+        # Protects access to _registrations
+        self._lock = threading.Lock()
+
+        self._register_with_collector()
+
+    def register(self, key, callback):
+        """Registers that we've entered a new block with labels `key`.
+
+        `callback` gets called each time the metrics are collected. The same
+        value must also be given to `unregister`.
+
+        `callback` gets called with an object that has an attribute per
+        sub_metric, which should be updated with the necessary values. Note that
+        the metrics object is shared between all callbacks registered with the
+        same key.
+
+        Note that `callback` may be called on a separate thread.
+        """
+        with self._lock:
+            self._registrations.setdefault(key, set()).add(callback)
+
+    def unregister(self, key, callback):
+        """Registers that we've exited a block with labels `key`.
+        """
+
+        with self._lock:
+            self._registrations.setdefault(key, set()).discard(callback)
+
+    def collect(self):
+        """Called by prometheus client when it reads metrics.
+
+        Note: may be called by a separate thread.
+        """
+        in_flight = GaugeMetricFamily(self.name + "_total", self.desc, labels=self.labels)
+
+        metrics_by_key = {}
+
+        # We copy so that we don't mutate the list while iterating
+        with self._lock:
+            keys = list(self._registrations)
+
+        for key in keys:
+            with self._lock:
+                callbacks = set(self._registrations[key])
+
+            in_flight.add_metric(key, len(callbacks))
+
+            metrics = self._metrics_class()
+            metrics_by_key[key] = metrics
+            for callback in callbacks:
+                callback(metrics)
+
+        yield in_flight
+
+        for name in self.sub_metrics:
+            gauge = GaugeMetricFamily("_".join([self.name, name]), "", labels=self.labels)
+            for key, metrics in six.iteritems(metrics_by_key):
+                gauge.add_metric(key, getattr(metrics, name))
+            yield gauge
+
+    def _register_with_collector(self):
+        if self.name in all_gauges.keys():
+            logger.warning("%s already registered, reregistering" % (self.name,))
+            REGISTRY.unregister(all_gauges.pop(self.name))
+
+        REGISTRY.register(self)
+        all_gauges[self.name] = self
+
+
 #
 # Detailed CPU metrics
 #
diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py
index 97f1267380bf..4b4ac5f6c7d8 100644
--- a/synapse/util/metrics.py
+++ b/synapse/util/metrics.py
@@ -20,6 +20,7 @@
 
 from twisted.internet import defer
 
+from synapse.metrics import InFlightGauge
 from synapse.util.logcontext import LoggingContext
 
 logger = logging.getLogger(__name__)
@@ -45,6 +46,13 @@
 block_db_sched_duration = Counter(
     "synapse_util_metrics_block_db_sched_duration_seconds", "", ["block_name"])
 
+# Tracks the number of blocks currently active
+in_flight = InFlightGauge(
+    "synapse_util_metrics_block_in_flight", "",
+    labels=["block_name"],
+    sub_metrics=["real_time_max", "real_time_sum"],
+)
+
 
 def measure_func(name):
     def wrapper(func):
@@ -82,10 +90,14 @@ def __enter__(self):
 
         self.start_usage = self.start_context.get_resource_usage()
 
+        in_flight.register((self.name,), self._update_in_flight)
+
     def __exit__(self, exc_type, exc_val, exc_tb):
         if isinstance(exc_type, Exception) or not self.start_context:
             return
 
+        in_flight.unregister((self.name,), self._update_in_flight)
+
         duration = self.clock.time() - self.start
 
         block_counter.labels(self.name).inc()
@@ -120,3 +132,13 @@ def __exit__(self, exc_type, exc_val, exc_tb):
 
         if self.created_context:
             self.start_context.__exit__(exc_type, exc_val, exc_tb)
+
+    def _update_in_flight(self, metrics):
+        """Gets called when processing in flight metrics
+        """
+        duration = self.clock.time() - self.start
+
+        metrics.real_time_max = max(metrics.real_time_max, duration)
+        metrics.real_time_sum += duration
+
+        # TODO: Add other in flight metrics.

From f6e82dcddb49a6e6118d85e54655f5adaf1eda4e Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 14 Sep 2018 15:08:22 +0100
Subject: [PATCH 628/830] Tests

---
 tests/test_metrics.py | 81 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 81 insertions(+)
 create mode 100644 tests/test_metrics.py

diff --git a/tests/test_metrics.py b/tests/test_metrics.py
new file mode 100644
index 000000000000..17897711a14b
--- /dev/null
+++ b/tests/test_metrics.py
@@ -0,0 +1,81 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from synapse.metrics import InFlightGauge
+
+from tests import unittest
+
+
+class TestMauLimit(unittest.TestCase):
+    def test_basic(self):
+        gauge = InFlightGauge(
+            "test1", "",
+            labels=["test_label"],
+            sub_metrics=["foo", "bar"],
+        )
+
+        def handle1(metrics):
+            metrics.foo += 2
+            metrics.bar = max(metrics.bar, 5)
+
+        def handle2(metrics):
+            metrics.foo += 3
+            metrics.bar = max(metrics.bar, 7)
+
+        gauge.register(("key1",), handle1)
+
+        self.assert_dict({
+            "test1_total": {("key1",): 1},
+            "test1_foo": {("key1",): 2},
+            "test1_bar": {("key1",): 5},
+        }, self.get_metrics_from_gauge(gauge))
+
+        gauge.unregister(("key1",), handle1)
+
+        self.assert_dict({
+            "test1_total": {("key1",): 0},
+            "test1_foo": {("key1",): 0},
+            "test1_bar": {("key1",): 0},
+        }, self.get_metrics_from_gauge(gauge))
+
+        gauge.register(("key1",), handle1)
+        gauge.register(("key2",), handle2)
+
+        self.assert_dict({
+            "test1_total": {("key1",): 1, ("key2",): 1},
+            "test1_foo": {("key1",): 2, ("key2",): 3},
+            "test1_bar": {("key1",): 5, ("key2",): 7},
+        }, self.get_metrics_from_gauge(gauge))
+
+        gauge.unregister(("key2",), handle2)
+        gauge.register(("key1",), handle2)
+
+        self.assert_dict({
+            "test1_total": {("key1",): 2, ("key2",): 0},
+            "test1_foo": {("key1",): 5, ("key2",): 0},
+            "test1_bar": {("key1",): 7, ("key2",): 0},
+        }, self.get_metrics_from_gauge(gauge))
+
+    def get_metrics_from_gauge(self, gauge):
+        results = {}
+
+        for r in gauge.collect():
+            results[r.name] = {
+                tuple(labels[x] for x in gauge.labels): value
+                for _, labels, value in r.samples
+            }
+
+        return results

From 941ac0f0856729087bd28d9243964c35147fcb7b Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 14 Sep 2018 14:41:16 +0100
Subject: [PATCH 629/830] Newsfile

---
 changelog.d/3871.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3871.misc

diff --git a/changelog.d/3871.misc b/changelog.d/3871.misc
new file mode 100644
index 000000000000..dd9510ceb6ac
--- /dev/null
+++ b/changelog.d/3871.misc
@@ -0,0 +1 @@
+Add in flight real time metrics for Measure blocks

From 9e2f9a7b57a8c4d2238684a9ccd6145948229e2c Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 14 Sep 2018 15:11:26 +0100
Subject: [PATCH 630/830] Measure outbound requests

---
 synapse/http/matrixfederationclient.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index c49dbacd9371..c3542b935320 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -43,6 +43,7 @@
 from synapse.http.endpoint import matrix_federation_endpoint
 from synapse.util import logcontext
 from synapse.util.logcontext import make_deferred_yieldable
+from synapse.util.metrics import Measure
 
 logger = logging.getLogger(__name__)
 outbound_logger = logging.getLogger("synapse.http.outbound")
@@ -224,9 +225,11 @@ def _request(self, destination, method, path,
                         reactor=self.hs.get_reactor()
                     )
                     request_deferred.addTimeout(_sec_timeout, self.hs.get_reactor())
-                    response = yield make_deferred_yieldable(
-                        request_deferred,
-                    )
+
+                    with Measure(self.clock, "outbound_request"):
+                        response = yield make_deferred_yieldable(
+                            request_deferred,
+                        )
 
                     log_result = "%d %s" % (response.code, response.phrase,)
                     break

From d0f6c1ce2189693db40a385f9ff19bdbbc5c87cf Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 14 Sep 2018 15:12:36 +0100
Subject: [PATCH 631/830] Remove spurious comment

---
 synapse/metrics/__init__.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index 98333b20489f..59900aa5d1b5 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -107,8 +107,6 @@ class InFlightGauge(object):
             will update.
     """
 
-    # TODO: Expand this to
-
     def __init__(self, name, desc, labels, sub_metrics):
         self.name = name
         self.desc = desc

From 90f8e606e2f6869dac20bc4481b9e8c62d32c534 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Sat, 15 Sep 2018 00:23:55 +1000
Subject: [PATCH 632/830] changelog

---
 changelog.d/3872.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3872.misc

diff --git a/changelog.d/3872.misc b/changelog.d/3872.misc
new file mode 100644
index 000000000000..b450c506d873
--- /dev/null
+++ b/changelog.d/3872.misc
@@ -0,0 +1 @@
+Disable buffering and automatic retrying in treq requests to prevent timeouts.
\ No newline at end of file

From bc9af88a2d971cd75aad419c60e13b0778640b28 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Sat, 15 Sep 2018 00:26:00 +1000
Subject: [PATCH 633/830] fix

---
 synapse/http/client.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/synapse/http/client.py b/synapse/http/client.py
index d60f87b04ea0..ec339a92ade5 100644
--- a/synapse/http/client.py
+++ b/synapse/http/client.py
@@ -93,7 +93,7 @@ def request(self, method, uri, data=b'', headers=None):
         outgoing_requests_counter.labels(method).inc()
 
         # log request but strip `access_token` (AS requests for example include this)
-        logger.info("Sending request %s %s", method, redact_uri(uri.encode('ascii')))
+        logger.info("Sending request %s %s", method, redact_uri(uri))
 
         try:
             request_deferred = treq.request(
@@ -108,14 +108,14 @@ def request(self, method, uri, data=b'', headers=None):
             incoming_responses_counter.labels(method, response.code).inc()
             logger.info(
                 "Received response to  %s %s: %s",
-                method, redact_uri(uri.encode('ascii')), response.code
+                method, redact_uri(uri), response.code
             )
             defer.returnValue(response)
         except Exception as e:
             incoming_responses_counter.labels(method, "ERR").inc()
             logger.info(
                 "Error sending request to  %s %s: %s %s",
-                method, redact_uri(uri.encode('ascii')), type(e).__name__, e.args[0]
+                method, redact_uri(uri), type(e).__name__, e.args[0]
             )
             raise
 

From 024be6cf1891fc20baac2356246595dc7f8ba5c5 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@arasphere.net>
Date: Fri, 14 Sep 2018 18:12:52 +0100
Subject: [PATCH 634/830] don't filter membership events based on history
 visibility (#3874)

don't filter membership events based on history visibility
as we will already have filtered the messages in the timeline, and state events
are always visible.

and because @erikjohnston said so.
---
 changelog.d/3874.bugfix        | 0
 synapse/handlers/pagination.py | 9 +--------
 2 files changed, 1 insertion(+), 8 deletions(-)
 create mode 100644 changelog.d/3874.bugfix

diff --git a/changelog.d/3874.bugfix b/changelog.d/3874.bugfix
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py
index 5170d093e365..a155b6e938ff 100644
--- a/synapse/handlers/pagination.py
+++ b/synapse/handlers/pagination.py
@@ -269,14 +269,7 @@ def get_messages(self, requester, room_id=None, pagin_config=None,
 
             if state_ids:
                 state = yield self.store.get_events(list(state_ids.values()))
-
-            if state:
-                state = yield filter_events_for_client(
-                    self.store,
-                    user_id,
-                    state.values(),
-                    is_peeking=(member_event_id is None),
-                )
+                state = state.values()
 
         time_now = self.clock.time_msec()
 

From fcfe7a850dada2e65982f2bc805d8bc409f07512 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 14 Sep 2018 19:23:07 +0100
Subject: [PATCH 635/830] Add an awful secondary timeout to fix wedged requests

This is an attempt to mitigate #3842 by adding yet-another-timeout
---
 synapse/http/matrixfederationclient.py | 11 ++++++
 synapse/util/async_helpers.py          | 51 ++++++++++++++++++++++++++
 2 files changed, 62 insertions(+)

diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index da16b5dd8cd7..13b19f7626f7 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -42,6 +42,7 @@
 )
 from synapse.http.endpoint import matrix_federation_endpoint
 from synapse.util import logcontext
+from synapse.util.async_helpers import timeout_no_seriously
 from synapse.util.logcontext import make_deferred_yieldable
 from synapse.util.metrics import Measure
 
@@ -228,6 +229,16 @@ def _request(self, destination, method, path,
                     )
                     request_deferred.addTimeout(_sec_timeout, self.hs.get_reactor())
 
+                    # Sometimes the timeout above doesn't work, so lets hack yet
+                    # another layer of timeouts in in the vain hope that at some
+                    # point the world made sense and this really really really
+                    # should work.
+                    request_deferred = timeout_no_seriously(
+                        request_deferred,
+                        timeout=_sec_timeout * 2,
+                        reactor=self.hs.get_reactor(),
+                    )
+
                     with Measure(self.clock, "outbound_request"):
                         response = yield make_deferred_yieldable(
                             request_deferred,
diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py
index 9b3f2f4b96df..083e4f412861 100644
--- a/synapse/util/async_helpers.py
+++ b/synapse/util/async_helpers.py
@@ -438,3 +438,54 @@ def _cancelled_to_timed_out_error(value, timeout):
         value.trap(CancelledError)
         raise DeferredTimeoutError(timeout, "Deferred")
     return value
+
+
+def timeout_no_seriously(deferred, timeout, reactor):
+    """The in build twisted deferred addTimeout (and the method above)
+    completely fail to time things out under some unknown circumstances.
+
+    Lets try a different way of timing things out and maybe that will make
+    things work?!
+
+    TODO: Kill this with fire.
+    """
+
+    new_d = defer.Deferred()
+
+    timed_out = [False]
+
+    def time_it_out():
+        timed_out[0] = True
+        deferred.cancel()
+
+        if not new_d.called:
+            new_d.errback(DeferredTimeoutError(timeout, "Deferred"))
+
+    delayed_call = reactor.callLater(timeout, time_it_out)
+
+    def convert_cancelled(value):
+        if timed_out[0]:
+            return _cancelled_to_timed_out_error(value, timeout)
+        return value
+
+    deferred.addBoth(convert_cancelled)
+
+    def cancel_timeout(result):
+        # stop the pending call to cancel the deferred if it's been fired
+        if delayed_call.active():
+            delayed_call.cancel()
+        return result
+
+    deferred.addBoth(cancel_timeout)
+
+    def success_cb(val):
+        if not new_d.called:
+            new_d.callback(val)
+
+    def failure_cb(val):
+        if not new_d.called:
+            new_d.errback(val)
+
+    deferred.addCallbacks(success_cb, failure_cb)
+
+    return new_d

From 335b23a078b416b3ea97d1bef73da6ca5abd8d4b Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 14 Sep 2018 19:25:58 +0100
Subject: [PATCH 636/830] Newsfile

---
 changelog.d/3875.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3875.bugfix

diff --git a/changelog.d/3875.bugfix b/changelog.d/3875.bugfix
new file mode 100644
index 000000000000..2d2147dd4bc7
--- /dev/null
+++ b/changelog.d/3875.bugfix
@@ -0,0 +1 @@
+Mitigate outbound federation randomly becoming wedged

From 24efb2a70dfd8aa100507612836601991e11affc Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Sat, 15 Sep 2018 11:38:39 +0100
Subject: [PATCH 637/830] Fix timeout function

Turns out deferred.cancel sometimes throws, so we do that last to ensure
that we always do resolve the new deferred.
---
 synapse/util/async_helpers.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py
index 083e4f412861..40c294612990 100644
--- a/synapse/util/async_helpers.py
+++ b/synapse/util/async_helpers.py
@@ -456,11 +456,12 @@ def timeout_no_seriously(deferred, timeout, reactor):
 
     def time_it_out():
         timed_out[0] = True
-        deferred.cancel()
 
         if not new_d.called:
             new_d.errback(DeferredTimeoutError(timeout, "Deferred"))
 
+        deferred.cancel()
+
     delayed_call = reactor.callLater(timeout, time_it_out)
 
     def convert_cancelled(value):

From c8642720c9baf16ac434e2cb1b88e0805f2eb00f Mon Sep 17 00:00:00 2001
From: Vincent Breitmoser <look@my.amazin.horse>
Date: Sat, 15 Sep 2018 22:03:27 +0200
Subject: [PATCH 638/830] mention libjemalloc in readme (#3877)

---
 README.rst | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/README.rst b/README.rst
index 2471619706e4..cfcf8b521986 100644
--- a/README.rst
+++ b/README.rst
@@ -963,5 +963,13 @@ variable.  The default is 0.5, which can be decreased to reduce RAM usage
 in memory constrained enviroments, or increased if performance starts to
 degrade.
 
+Using `libjemalloc <http://jemalloc.net/>`_ can also yield a significant
+improvement in overall amount, and especially in terms of giving back RAM
+to the OS. To use it, the library must simply be put in the LD_PRELOAD
+environment variable when launching Synapse. On Debian, this can be done
+by installing the ``libjemalloc1`` package and adding this line to
+``/etc/default/matrix-synaspse``::
+
+    LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1
 
 .. _`key_management`: https://matrix.org/docs/spec/server_server/unstable.html#retrieving-server-keys

From 9d13ff4da8483e4fdd66dd16e8ad4ac299f10e43 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Sat, 15 Sep 2018 21:04:10 +0100
Subject: [PATCH 639/830] missing changelog

---
 changelog.d/3877.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3877.misc

diff --git a/changelog.d/3877.misc b/changelog.d/3877.misc
new file mode 100644
index 000000000000..a80fec4bd8a1
--- /dev/null
+++ b/changelog.d/3877.misc
@@ -0,0 +1 @@
+mention jemalloc in the README

From d42d79e3c3cc229fa6bb268a97edde4ac81e2df5 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Sat, 15 Sep 2018 22:27:41 +0100
Subject: [PATCH 640/830] don't ratelimit autojoins

---
 synapse/handlers/register.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index 1e53f2c635c3..da914c46ffe4 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -534,4 +534,5 @@ def _join_user_to_room(self, requester, room_identifier):
             room_id=room_id,
             remote_room_hosts=remote_room_hosts,
             action="join",
+            ratelimit=False,
         )

From c71b93f2a4bec7d12ce905388336e897a5227b35 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Sat, 15 Sep 2018 22:28:28 +0100
Subject: [PATCH 641/830] changelog

---
 changelog.d/3879.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3879.bugfix

diff --git a/changelog.d/3879.bugfix b/changelog.d/3879.bugfix
new file mode 100644
index 000000000000..82cb2a8ebc49
--- /dev/null
+++ b/changelog.d/3879.bugfix
@@ -0,0 +1 @@
+Don't ratelimit autojoins

From 9c749a6b612d538a0d8374152ff358508f733734 Mon Sep 17 00:00:00 2001
From: Simon Dwyer <simon@thedwyers.co>
Date: Sun, 16 Sep 2018 10:22:27 +1000
Subject: [PATCH 642/830] Added 'MAX_UPLOAD_SIZE' variable and set default to
 "10M"

---
 docker/conf/homeserver.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/conf/homeserver.yaml b/docker/conf/homeserver.yaml
index 6bc25bb45f2a..cfe88788f2cc 100644
--- a/docker/conf/homeserver.yaml
+++ b/docker/conf/homeserver.yaml
@@ -85,7 +85,7 @@ federation_rc_concurrent: 3
 
 media_store_path: "/data/media"
 uploads_path: "/data/uploads"
-max_upload_size: "10M"
+max_upload_size: "{{ SYNAPSE_MAX_UPLOAD_SIZE or "10M" }}"
 max_image_pixels: "32M"
 dynamic_thumbnails: false
 

From f472abd792c5f3d969bb16c9ad7ee278b87ee761 Mon Sep 17 00:00:00 2001
From: Simon Dwyer <simon@thedwyers.co>
Date: Sun, 16 Sep 2018 10:28:29 +1000
Subject: [PATCH 643/830] Added description for "SYNAPSE_MAX_UPLOAD_SIZE"
 variable.

---
 docker/README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker/README.md b/docker/README.md
index 038c78f7c052..ecdf8a8b17c9 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -88,6 +88,7 @@ variables are available for configuration:
 * ``SYNAPSE_TURN_URIS``, set this variable to the coma-separated list of TURN
   uris to enable TURN for this homeserver.
 * ``SYNAPSE_TURN_SECRET``, set this to the TURN shared secret if required.
+* ``SYNAPSE_MAX_UPLOAD_SIZE``, the max upload size [default `10M`].
 
 Shared secrets, that will be initialized to random values if not set:
 

From da864a92c9eb8335fdd5602bcd4afe21c98cca39 Mon Sep 17 00:00:00 2001
From: Simon Dwyer <simon@thedwyers.co>
Date: Sun, 16 Sep 2018 12:59:23 +1000
Subject: [PATCH 644/830] Added description for "SYNAPSE_MAX_UPLOAD_SIZE"
 variable.

---
 docker/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/README.md b/docker/README.md
index ecdf8a8b17c9..3c00d1e9480d 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -88,7 +88,7 @@ variables are available for configuration:
 * ``SYNAPSE_TURN_URIS``, set this variable to the coma-separated list of TURN
   uris to enable TURN for this homeserver.
 * ``SYNAPSE_TURN_SECRET``, set this to the TURN shared secret if required.
-* ``SYNAPSE_MAX_UPLOAD_SIZE``, the max upload size [default `10M`].
+* ``SYNAPSE_MAX_UPLOAD_SIZE``, set this variable to change the max upload size [default `10M`].
 
 Shared secrets, that will be initialized to random values if not set:
 

From 0e46ff690473538477037d5dbd2200f6863fc68e Mon Sep 17 00:00:00 2001
From: Simon Dwyer <simon@thedwyers.co>
Date: Sun, 16 Sep 2018 13:33:33 +1000
Subject: [PATCH 645/830] Adding the ability to change MAX_UPLOAD_SIZE for the
 docker container variables.

Signed-off-by: Simon Dwyer <simon@thedwyers.co>
---
 changelog.d/3883.feature | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3883.feature

diff --git a/changelog.d/3883.feature b/changelog.d/3883.feature
new file mode 100644
index 000000000000..c11e5c53090e
--- /dev/null
+++ b/changelog.d/3883.feature
@@ -0,0 +1 @@
+Adding the ability to change MAX_UPLOAD_SIZE for the docker container variables.
\ No newline at end of file

From e7b3b4d8c22952d4f090da6623fefb6c1caf0070 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 17 Sep 2018 11:42:42 +0100
Subject: [PATCH 646/830] Remove nuke-room-from-db.sh script

The problem with this script is that it is largely untested, entirely
unmaintained, and running it is likely to make your synapse blow up in
exciting ways.

For example, it leaves a bunch of tables with dead values in it, like
event_to_state_groups.

Having it here sends a message that it is a supported part of
synapse, which is absolutely not the case.
---
 scripts-dev/nuke-room-from-db.sh | 57 --------------------------------
 1 file changed, 57 deletions(-)
 delete mode 100755 scripts-dev/nuke-room-from-db.sh

diff --git a/scripts-dev/nuke-room-from-db.sh b/scripts-dev/nuke-room-from-db.sh
deleted file mode 100755
index c62928afdb69..000000000000
--- a/scripts-dev/nuke-room-from-db.sh
+++ /dev/null
@@ -1,57 +0,0 @@
-#!/bin/bash
-
-## CAUTION:
-## This script will remove (hopefully) all trace of the given room ID from
-## your homeserver.db
-
-## Do not run it lightly.
-
-set -e
-
-if [ "$1" == "-h" ] || [ "$1" == "" ]; then
-  echo "Call with ROOM_ID as first option and then pipe it into the database. So for instance you might run"
-  echo " nuke-room-from-db.sh <room_id> | sqlite3 homeserver.db"
-  echo "or"
-  echo " nuke-room-from-db.sh <room_id> | psql --dbname=synapse"
-  exit
-fi
-
-ROOMID="$1"
-
-cat <<EOF
-DELETE FROM event_forward_extremities WHERE room_id = '$ROOMID';
-DELETE FROM event_backward_extremities WHERE room_id = '$ROOMID';
-DELETE FROM event_edges WHERE room_id = '$ROOMID';
-DELETE FROM room_depth WHERE room_id = '$ROOMID';
-DELETE FROM state_forward_extremities WHERE room_id = '$ROOMID';
-DELETE FROM events WHERE room_id = '$ROOMID';
-DELETE FROM event_json WHERE room_id = '$ROOMID';
-DELETE FROM state_events WHERE room_id = '$ROOMID';
-DELETE FROM current_state_events WHERE room_id = '$ROOMID';
-DELETE FROM room_memberships WHERE room_id = '$ROOMID';
-DELETE FROM feedback WHERE room_id = '$ROOMID';
-DELETE FROM topics WHERE room_id = '$ROOMID';
-DELETE FROM room_names WHERE room_id = '$ROOMID';
-DELETE FROM rooms WHERE room_id = '$ROOMID';
-DELETE FROM room_hosts WHERE room_id = '$ROOMID';
-DELETE FROM room_aliases WHERE room_id = '$ROOMID';
-DELETE FROM state_groups WHERE room_id = '$ROOMID';
-DELETE FROM state_groups_state WHERE room_id = '$ROOMID';
-DELETE FROM receipts_graph WHERE room_id = '$ROOMID';
-DELETE FROM receipts_linearized WHERE room_id = '$ROOMID';
-DELETE FROM event_search WHERE room_id = '$ROOMID'; 
-DELETE FROM guest_access WHERE room_id = '$ROOMID';
-DELETE FROM history_visibility WHERE room_id = '$ROOMID';
-DELETE FROM room_tags WHERE room_id = '$ROOMID';
-DELETE FROM room_tags_revisions WHERE room_id = '$ROOMID';
-DELETE FROM room_account_data WHERE room_id = '$ROOMID';
-DELETE FROM event_push_actions WHERE room_id = '$ROOMID';
-DELETE FROM local_invites WHERE room_id = '$ROOMID';
-DELETE FROM pusher_throttle WHERE room_id = '$ROOMID';
-DELETE FROM event_reports WHERE room_id = '$ROOMID';
-DELETE FROM public_room_list_stream WHERE room_id = '$ROOMID';
-DELETE FROM stream_ordering_to_exterm WHERE room_id = '$ROOMID';
-DELETE FROM event_auth WHERE room_id = '$ROOMID';
-DELETE FROM appservice_room_list WHERE room_id = '$ROOMID';
-VACUUM;
-EOF

From 2a8996b67d7ec387d2c4e4843e2cf5a8322c15f0 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 17 Sep 2018 11:51:38 +0100
Subject: [PATCH 647/830] changelog

---
 changelog.d/3888.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3888.misc

diff --git a/changelog.d/3888.misc b/changelog.d/3888.misc
new file mode 100644
index 000000000000..a10ede547e7b
--- /dev/null
+++ b/changelog.d/3888.misc
@@ -0,0 +1 @@
+Remove unmaintained "nuke-room-from-db.sh" script

From 85a43f4167b5e775f7c3afe96d71137818561272 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 17 Sep 2018 13:19:00 +0100
Subject: [PATCH 648/830] Return a 404 when deleting unknown room alias

As per https://github.com/matrix-org/matrix-doc/issues/1675

Fixes https://github.com/matrix-org/synapse/issues/2782
---
 synapse/handlers/directory.py | 19 ++++++++++++++++---
 synapse/storage/directory.py  |  1 -
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py
index ef866da1b676..c745e6740b3f 100644
--- a/synapse/handlers/directory.py
+++ b/synapse/handlers/directory.py
@@ -20,7 +20,14 @@
 from twisted.internet import defer
 
 from synapse.api.constants import EventTypes
-from synapse.api.errors import AuthError, CodeMessageException, Codes, SynapseError
+from synapse.api.errors import (
+    AuthError,
+    CodeMessageException,
+    Codes,
+    NotFoundError,
+    StoreError,
+    SynapseError,
+)
 from synapse.types import RoomAlias, UserID, get_domain_from_id
 
 from ._base import BaseHandler
@@ -109,7 +116,13 @@ def create_appservice_association(self, service, room_alias, room_id,
     def delete_association(self, requester, user_id, room_alias):
         # association deletion for human users
 
-        can_delete = yield self._user_can_delete_alias(room_alias, user_id)
+        try:
+            can_delete = yield self._user_can_delete_alias(room_alias, user_id)
+        except StoreError as e:
+            if e.code == 404:
+                raise NotFoundError("Unknown room alias")
+            raise
+
         if not can_delete:
             raise AuthError(
                 403, "You don't have permission to delete the alias.",
@@ -320,7 +333,7 @@ def can_modify_alias(self, alias, user_id=None):
     def _user_can_delete_alias(self, alias, user_id):
         creator = yield self.store.get_room_alias_creator(alias.to_string())
 
-        if creator and creator == user_id:
+        if creator == user_id:
             defer.returnValue(True)
 
         is_admin = yield self.auth.is_server_admin(UserID.from_string(user_id))
diff --git a/synapse/storage/directory.py b/synapse/storage/directory.py
index 808194236a85..cfb687cb53db 100644
--- a/synapse/storage/directory.py
+++ b/synapse/storage/directory.py
@@ -75,7 +75,6 @@ def get_room_alias_creator(self, room_alias):
             },
             retcol="creator",
             desc="get_room_alias_creator",
-            allow_none=True
         )
 
     @cached(max_entries=5000)

From c1ae6b1bce61fecda91aa6ab8a51d152418cf79e Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 17 Sep 2018 13:21:08 +0100
Subject: [PATCH 649/830] changelog

---
 changelog.d/3889.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3889.bugfix

diff --git a/changelog.d/3889.bugfix b/changelog.d/3889.bugfix
new file mode 100644
index 000000000000..e97642598757
--- /dev/null
+++ b/changelog.d/3889.bugfix
@@ -0,0 +1 @@
+Fix 500 error when deleting unknown room alias

From fe88907d044c79844612b3950421b48c20fd22ad Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Mon, 17 Sep 2018 22:33:22 +1000
Subject: [PATCH 650/830] version

---
 synapse/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/__init__.py b/synapse/__init__.py
index 9dbe0b9f10e6..f6a194dfc2e0 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -27,4 +27,4 @@
 except ImportError:
     pass
 
-__version__ = "0.33.4"
+__version__ = "0.33.5rc1"

From b56ef14629fe4ee07f3a166073a55509fc69282d Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Mon, 17 Sep 2018 22:36:41 +1000
Subject: [PATCH 651/830] update python 3 changelog

---
 changelog.d/3576.feature | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/changelog.d/3576.feature b/changelog.d/3576.feature
index 02a10e370da2..a624780babfb 100644
--- a/changelog.d/3576.feature
+++ b/changelog.d/3576.feature
@@ -1 +1 @@
-Python 3.5+ is now supported.
+Python 3.5 and 3.6 support is now in beta.

From 1e70f1dbab352fc559e3ae815f7855503cca6d41 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Mon, 17 Sep 2018 22:36:56 +1000
Subject: [PATCH 652/830] changelog

---
 CHANGES.md               | 58 ++++++++++++++++++++++++++++++++++++++++
 changelog.d/3576.feature |  1 -
 changelog.d/3704.misc    |  1 -
 changelog.d/3771.misc    |  1 -
 changelog.d/3788.bugfix  |  1 -
 changelog.d/3789.misc    |  1 -
 changelog.d/3790.feature |  1 -
 changelog.d/3795.misc    |  1 -
 changelog.d/3800.bugfix  |  1 -
 changelog.d/3803.misc    |  1 -
 changelog.d/3804.bugfix  |  1 -
 changelog.d/3805.misc    |  1 -
 changelog.d/3806.misc    |  1 -
 changelog.d/3808.misc    |  1 -
 changelog.d/3810.bugfix  |  1 -
 changelog.d/3822.misc    |  1 -
 changelog.d/3823.misc    |  1 -
 changelog.d/3824.bugfix  |  1 -
 changelog.d/3826.misc    |  1 -
 changelog.d/3827.misc    |  1 -
 changelog.d/3834.misc    |  1 -
 changelog.d/3835.bugfix  |  1 -
 changelog.d/3840.misc    |  1 -
 changelog.d/3841.bugfix  |  1 -
 changelog.d/3845.bugfix  |  1 -
 changelog.d/3846.feature |  1 -
 changelog.d/3847.misc    |  1 -
 changelog.d/3851.bugfix  |  1 -
 changelog.d/3853.misc    |  1 -
 changelog.d/3855.misc    |  1 -
 changelog.d/3856.misc    |  1 -
 changelog.d/3857.misc    |  1 -
 changelog.d/3858.misc    |  1 -
 changelog.d/3859.bugfix  |  1 -
 changelog.d/3860.misc    |  1 -
 changelog.d/3871.misc    |  1 -
 changelog.d/3872.misc    |  1 -
 changelog.d/3874.bugfix  |  0
 changelog.d/3875.bugfix  |  1 -
 changelog.d/3877.misc    |  1 -
 changelog.d/3888.misc    |  1 -
 41 files changed, 58 insertions(+), 39 deletions(-)
 delete mode 100644 changelog.d/3576.feature
 delete mode 100644 changelog.d/3704.misc
 delete mode 100644 changelog.d/3771.misc
 delete mode 100644 changelog.d/3788.bugfix
 delete mode 100644 changelog.d/3789.misc
 delete mode 100644 changelog.d/3790.feature
 delete mode 100644 changelog.d/3795.misc
 delete mode 100644 changelog.d/3800.bugfix
 delete mode 100644 changelog.d/3803.misc
 delete mode 100644 changelog.d/3804.bugfix
 delete mode 100644 changelog.d/3805.misc
 delete mode 100644 changelog.d/3806.misc
 delete mode 100644 changelog.d/3808.misc
 delete mode 100644 changelog.d/3810.bugfix
 delete mode 100644 changelog.d/3822.misc
 delete mode 100644 changelog.d/3823.misc
 delete mode 100644 changelog.d/3824.bugfix
 delete mode 100644 changelog.d/3826.misc
 delete mode 100644 changelog.d/3827.misc
 delete mode 100644 changelog.d/3834.misc
 delete mode 100644 changelog.d/3835.bugfix
 delete mode 100644 changelog.d/3840.misc
 delete mode 100644 changelog.d/3841.bugfix
 delete mode 100644 changelog.d/3845.bugfix
 delete mode 100644 changelog.d/3846.feature
 delete mode 100644 changelog.d/3847.misc
 delete mode 100644 changelog.d/3851.bugfix
 delete mode 100644 changelog.d/3853.misc
 delete mode 100644 changelog.d/3855.misc
 delete mode 100644 changelog.d/3856.misc
 delete mode 100644 changelog.d/3857.misc
 delete mode 100644 changelog.d/3858.misc
 delete mode 100644 changelog.d/3859.bugfix
 delete mode 100644 changelog.d/3860.misc
 delete mode 100644 changelog.d/3871.misc
 delete mode 100644 changelog.d/3872.misc
 delete mode 100644 changelog.d/3874.bugfix
 delete mode 100644 changelog.d/3875.bugfix
 delete mode 100644 changelog.d/3877.misc
 delete mode 100644 changelog.d/3888.misc

diff --git a/CHANGES.md b/CHANGES.md
index ee864c3c639c..00dbed702164 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,61 @@
+Synapse 0.33.5rc1 (2018-09-17)
+==============================
+
+Features
+--------
+
+- Python 3.5 and 3.6 support is now in beta. ([\#3576](https://github.com/matrix-org/synapse/issues/3576))
+- Implement `event_format` filter param in `/sync` ([\#3790](https://github.com/matrix-org/synapse/issues/3790))
+- Add synapse_admin_mau:registered_reserved_users metric to expose number of real reaserved users ([\#3846](https://github.com/matrix-org/synapse/issues/3846))
+
+
+Bugfixes
+--------
+
+- Remove connection ID for replication prometheus metrics, as it creates a large number of new series. ([\#3788](https://github.com/matrix-org/synapse/issues/3788))
+- guest users should not be part of mau total ([\#3800](https://github.com/matrix-org/synapse/issues/3800))
+- Bump dependency on pyopenssl 16.x, to avoid incompatibility with recent Twisted. ([\#3804](https://github.com/matrix-org/synapse/issues/3804))
+- Fix existing room tags not coming down sync when joining a room ([\#3810](https://github.com/matrix-org/synapse/issues/3810))
+- Fix jwt import check ([\#3824](https://github.com/matrix-org/synapse/issues/3824))
+- fix VOIP crashes under Python 3 (#3821) ([\#3835](https://github.com/matrix-org/synapse/issues/3835))
+- Fix manhole so that it works with latest openssh clients ([\#3841](https://github.com/matrix-org/synapse/issues/3841))
+- Fix outbound requests occasionally wedging, which can result in federation breaking between servers. ([\#3845](https://github.com/matrix-org/synapse/issues/3845))
+- Show heroes if room name/canonical alias has been deleted ([\#3851](https://github.com/matrix-org/synapse/issues/3851))
+- Fix handling of redacted events from federation ([\#3859](https://github.com/matrix-org/synapse/issues/3859))
+-  ([\#3874](https://github.com/matrix-org/synapse/issues/3874))
+- Mitigate outbound federation randomly becoming wedged ([\#3875](https://github.com/matrix-org/synapse/issues/3875))
+
+
+Internal Changes
+----------------
+
+- CircleCI tests now run on the potential merge of a PR. ([\#3704](https://github.com/matrix-org/synapse/issues/3704))
+- http/ is now ported to Python 3. ([\#3771](https://github.com/matrix-org/synapse/issues/3771))
+- Improve human readable error messages for threepid registration/account update ([\#3789](https://github.com/matrix-org/synapse/issues/3789))
+- Make /sync slightly faster by avoiding needless copies ([\#3795](https://github.com/matrix-org/synapse/issues/3795))
+- handlers/ is now ported to Python 3. ([\#3803](https://github.com/matrix-org/synapse/issues/3803))
+- Limit the number of PDUs/EDUs per federation transaction ([\#3805](https://github.com/matrix-org/synapse/issues/3805))
+- Only start postgres instance for postgres tests on Travis CI ([\#3806](https://github.com/matrix-org/synapse/issues/3806))
+- tests/ is now ported to Python 3. ([\#3808](https://github.com/matrix-org/synapse/issues/3808))
+- crypto/ is now ported to Python 3. ([\#3822](https://github.com/matrix-org/synapse/issues/3822))
+- rest/ is now ported to Python 3. ([\#3823](https://github.com/matrix-org/synapse/issues/3823))
+- add some logging for the keyring queue ([\#3826](https://github.com/matrix-org/synapse/issues/3826))
+- speed up lazy loading by 2-3x ([\#3827](https://github.com/matrix-org/synapse/issues/3827))
+- Improved Dockerfile to remove build requirements after building reducing the image size. ([\#3834](https://github.com/matrix-org/synapse/issues/3834))
+- Disable lazy loading for incremental syncs for now ([\#3840](https://github.com/matrix-org/synapse/issues/3840))
+- federation/ is now ported to Python 3. ([\#3847](https://github.com/matrix-org/synapse/issues/3847))
+- Log when we retry outbound requests ([\#3853](https://github.com/matrix-org/synapse/issues/3853))
+- Removed some excess logging messages. ([\#3855](https://github.com/matrix-org/synapse/issues/3855))
+- Speed up purge history for rooms that have been previously purged ([\#3856](https://github.com/matrix-org/synapse/issues/3856))
+- Refactor some HTTP timeout code. ([\#3857](https://github.com/matrix-org/synapse/issues/3857))
+- Fix running merged builds on CircleCI ([\#3858](https://github.com/matrix-org/synapse/issues/3858))
+- Fix typo in replication stream exception. ([\#3860](https://github.com/matrix-org/synapse/issues/3860))
+- Add in flight real time metrics for Measure blocks ([\#3871](https://github.com/matrix-org/synapse/issues/3871))
+- Disable buffering and automatic retrying in treq requests to prevent timeouts. ([\#3872](https://github.com/matrix-org/synapse/issues/3872))
+- mention jemalloc in the README ([\#3877](https://github.com/matrix-org/synapse/issues/3877))
+- Remove unmaintained "nuke-room-from-db.sh" script ([\#3888](https://github.com/matrix-org/synapse/issues/3888))
+
+
 Synapse 0.33.4 (2018-09-07)
 ===========================
 
diff --git a/changelog.d/3576.feature b/changelog.d/3576.feature
deleted file mode 100644
index a624780babfb..000000000000
--- a/changelog.d/3576.feature
+++ /dev/null
@@ -1 +0,0 @@
-Python 3.5 and 3.6 support is now in beta.
diff --git a/changelog.d/3704.misc b/changelog.d/3704.misc
deleted file mode 100644
index aaae0fbd6304..000000000000
--- a/changelog.d/3704.misc
+++ /dev/null
@@ -1 +0,0 @@
-CircleCI tests now run on the potential merge of a PR.
diff --git a/changelog.d/3771.misc b/changelog.d/3771.misc
deleted file mode 100644
index 47aa34bc04b6..000000000000
--- a/changelog.d/3771.misc
+++ /dev/null
@@ -1 +0,0 @@
-http/ is now ported to Python 3.
diff --git a/changelog.d/3788.bugfix b/changelog.d/3788.bugfix
deleted file mode 100644
index 72316fb881fa..000000000000
--- a/changelog.d/3788.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Remove connection ID for replication prometheus metrics, as it creates a large number of new series.
diff --git a/changelog.d/3789.misc b/changelog.d/3789.misc
deleted file mode 100644
index d2d5d91091a4..000000000000
--- a/changelog.d/3789.misc
+++ /dev/null
@@ -1 +0,0 @@
-Improve human readable error messages for threepid registration/account update
diff --git a/changelog.d/3790.feature b/changelog.d/3790.feature
deleted file mode 100644
index 2c4ac62fb570..000000000000
--- a/changelog.d/3790.feature
+++ /dev/null
@@ -1 +0,0 @@
-Implement `event_format` filter param in `/sync`
diff --git a/changelog.d/3795.misc b/changelog.d/3795.misc
deleted file mode 100644
index 9f64ee5e2bb2..000000000000
--- a/changelog.d/3795.misc
+++ /dev/null
@@ -1 +0,0 @@
-Make /sync slightly faster by avoiding needless copies
diff --git a/changelog.d/3800.bugfix b/changelog.d/3800.bugfix
deleted file mode 100644
index 6b2e18b4a6e4..000000000000
--- a/changelog.d/3800.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-guest users should not be part of mau total
diff --git a/changelog.d/3803.misc b/changelog.d/3803.misc
deleted file mode 100644
index 2b60653c298d..000000000000
--- a/changelog.d/3803.misc
+++ /dev/null
@@ -1 +0,0 @@
-handlers/ is now ported to Python 3.
diff --git a/changelog.d/3804.bugfix b/changelog.d/3804.bugfix
deleted file mode 100644
index a0cef20e3fdf..000000000000
--- a/changelog.d/3804.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Bump dependency on pyopenssl 16.x, to avoid incompatibility with recent Twisted.
diff --git a/changelog.d/3805.misc b/changelog.d/3805.misc
deleted file mode 100644
index 257feeb0718f..000000000000
--- a/changelog.d/3805.misc
+++ /dev/null
@@ -1 +0,0 @@
-Limit the number of PDUs/EDUs per federation transaction
diff --git a/changelog.d/3806.misc b/changelog.d/3806.misc
deleted file mode 100644
index 3c722eef2dc7..000000000000
--- a/changelog.d/3806.misc
+++ /dev/null
@@ -1 +0,0 @@
-Only start postgres instance for postgres tests on Travis CI
diff --git a/changelog.d/3808.misc b/changelog.d/3808.misc
deleted file mode 100644
index e5e1cd9e0e1a..000000000000
--- a/changelog.d/3808.misc
+++ /dev/null
@@ -1 +0,0 @@
-tests/ is now ported to Python 3.
diff --git a/changelog.d/3810.bugfix b/changelog.d/3810.bugfix
deleted file mode 100644
index 2b938a81aef0..000000000000
--- a/changelog.d/3810.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix existing room tags not coming down sync when joining a room
diff --git a/changelog.d/3822.misc b/changelog.d/3822.misc
deleted file mode 100644
index 5250f318969a..000000000000
--- a/changelog.d/3822.misc
+++ /dev/null
@@ -1 +0,0 @@
-crypto/ is now ported to Python 3.
diff --git a/changelog.d/3823.misc b/changelog.d/3823.misc
deleted file mode 100644
index 0da491ddaa0f..000000000000
--- a/changelog.d/3823.misc
+++ /dev/null
@@ -1 +0,0 @@
-rest/ is now ported to Python 3.
diff --git a/changelog.d/3824.bugfix b/changelog.d/3824.bugfix
deleted file mode 100644
index 99f199dcc658..000000000000
--- a/changelog.d/3824.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix jwt import check
\ No newline at end of file
diff --git a/changelog.d/3826.misc b/changelog.d/3826.misc
deleted file mode 100644
index a4d9a012f9e6..000000000000
--- a/changelog.d/3826.misc
+++ /dev/null
@@ -1 +0,0 @@
-add some logging for the keyring queue
diff --git a/changelog.d/3827.misc b/changelog.d/3827.misc
deleted file mode 100644
index bc294706cf56..000000000000
--- a/changelog.d/3827.misc
+++ /dev/null
@@ -1 +0,0 @@
-speed up lazy loading by 2-3x
diff --git a/changelog.d/3834.misc b/changelog.d/3834.misc
deleted file mode 100644
index 8902f8fba7a6..000000000000
--- a/changelog.d/3834.misc
+++ /dev/null
@@ -1 +0,0 @@
-Improved Dockerfile to remove build requirements after building reducing the image size.
diff --git a/changelog.d/3835.bugfix b/changelog.d/3835.bugfix
deleted file mode 100644
index 00dbcbc8dcf5..000000000000
--- a/changelog.d/3835.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-fix VOIP crashes under Python 3 (#3821)
diff --git a/changelog.d/3840.misc b/changelog.d/3840.misc
deleted file mode 100644
index b9585ae9bee4..000000000000
--- a/changelog.d/3840.misc
+++ /dev/null
@@ -1 +0,0 @@
-Disable lazy loading for incremental syncs for now
diff --git a/changelog.d/3841.bugfix b/changelog.d/3841.bugfix
deleted file mode 100644
index 2a48a7dd66ee..000000000000
--- a/changelog.d/3841.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix manhole so that it works with latest openssh clients
diff --git a/changelog.d/3845.bugfix b/changelog.d/3845.bugfix
deleted file mode 100644
index 5b7e8f193498..000000000000
--- a/changelog.d/3845.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix outbound requests occasionally wedging, which can result in federation breaking between servers.
diff --git a/changelog.d/3846.feature b/changelog.d/3846.feature
deleted file mode 100644
index 453c11d3f8a7..000000000000
--- a/changelog.d/3846.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add synapse_admin_mau:registered_reserved_users metric to expose number of real reaserved users 
diff --git a/changelog.d/3847.misc b/changelog.d/3847.misc
deleted file mode 100644
index bf8b5afea4e2..000000000000
--- a/changelog.d/3847.misc
+++ /dev/null
@@ -1 +0,0 @@
-federation/ is now ported to Python 3.
\ No newline at end of file
diff --git a/changelog.d/3851.bugfix b/changelog.d/3851.bugfix
deleted file mode 100644
index b53a9efe7b24..000000000000
--- a/changelog.d/3851.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Show heroes if room name/canonical alias has been deleted
diff --git a/changelog.d/3853.misc b/changelog.d/3853.misc
deleted file mode 100644
index db45d4983db9..000000000000
--- a/changelog.d/3853.misc
+++ /dev/null
@@ -1 +0,0 @@
-Log when we retry outbound requests
diff --git a/changelog.d/3855.misc b/changelog.d/3855.misc
deleted file mode 100644
index a25bb020baaa..000000000000
--- a/changelog.d/3855.misc
+++ /dev/null
@@ -1 +0,0 @@
-Removed some excess logging messages.
\ No newline at end of file
diff --git a/changelog.d/3856.misc b/changelog.d/3856.misc
deleted file mode 100644
index 36c311eb3db9..000000000000
--- a/changelog.d/3856.misc
+++ /dev/null
@@ -1 +0,0 @@
-Speed up purge history for rooms that have been previously purged
diff --git a/changelog.d/3857.misc b/changelog.d/3857.misc
deleted file mode 100644
index e128d193d9a3..000000000000
--- a/changelog.d/3857.misc
+++ /dev/null
@@ -1 +0,0 @@
-Refactor some HTTP timeout code.
\ No newline at end of file
diff --git a/changelog.d/3858.misc b/changelog.d/3858.misc
deleted file mode 100644
index 4644db5330fc..000000000000
--- a/changelog.d/3858.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix running merged builds on CircleCI
\ No newline at end of file
diff --git a/changelog.d/3859.bugfix b/changelog.d/3859.bugfix
deleted file mode 100644
index ec5b172464e4..000000000000
--- a/changelog.d/3859.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix handling of redacted events from federation
diff --git a/changelog.d/3860.misc b/changelog.d/3860.misc
deleted file mode 100644
index 364239d3e33e..000000000000
--- a/changelog.d/3860.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix typo in replication stream exception.
diff --git a/changelog.d/3871.misc b/changelog.d/3871.misc
deleted file mode 100644
index dd9510ceb6ac..000000000000
--- a/changelog.d/3871.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add in flight real time metrics for Measure blocks
diff --git a/changelog.d/3872.misc b/changelog.d/3872.misc
deleted file mode 100644
index b450c506d873..000000000000
--- a/changelog.d/3872.misc
+++ /dev/null
@@ -1 +0,0 @@
-Disable buffering and automatic retrying in treq requests to prevent timeouts.
\ No newline at end of file
diff --git a/changelog.d/3874.bugfix b/changelog.d/3874.bugfix
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/changelog.d/3875.bugfix b/changelog.d/3875.bugfix
deleted file mode 100644
index 2d2147dd4bc7..000000000000
--- a/changelog.d/3875.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Mitigate outbound federation randomly becoming wedged
diff --git a/changelog.d/3877.misc b/changelog.d/3877.misc
deleted file mode 100644
index a80fec4bd8a1..000000000000
--- a/changelog.d/3877.misc
+++ /dev/null
@@ -1 +0,0 @@
-mention jemalloc in the README
diff --git a/changelog.d/3888.misc b/changelog.d/3888.misc
deleted file mode 100644
index a10ede547e7b..000000000000
--- a/changelog.d/3888.misc
+++ /dev/null
@@ -1 +0,0 @@
-Remove unmaintained "nuke-room-from-db.sh" script

From c9c50284d71669ad2496c2b0af30c392428a67e5 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Mon, 17 Sep 2018 15:58:47 +0100
Subject: [PATCH 653/830] README: run python_dependencies with -m

... to stop things which try to import `types` getting `synapse.types` instead
---
 README.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index cfcf8b521986..0147429aed26 100644
--- a/README.rst
+++ b/README.rst
@@ -908,7 +908,7 @@ to install using pip and a virtualenv::
 
     virtualenv -p python2.7 env
     source env/bin/activate
-    python synapse/python_dependencies.py | xargs pip install
+    python -m synapse.python_dependencies | xargs pip install
     pip install lxml mock
 
 This will run a process of downloading and installing all the needed

From f00a9d2636211e6762d8c8c627ebee3f6cea05c9 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 17 Sep 2018 16:15:42 +0100
Subject: [PATCH 654/830] Fix some b'abcd' noise in logs and metrics

Python 3 compatibility: make sure that we decode some byte sequences before we
use them to create log lines and metrics labels.
---
 synapse/http/matrixfederationclient.py | 5 ++++-
 synapse/http/site.py                   | 8 ++++----
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 13b19f7626f7..083484a687ce 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -244,7 +244,10 @@ def _request(self, destination, method, path,
                             request_deferred,
                         )
 
-                    log_result = "%d %s" % (response.code, response.phrase,)
+                    log_result = "%d %s" % (
+                        response.code,
+                        response.phrase.decode('ascii', errors='replace'),
+                    )
                     break
                 except Exception as e:
                     if not retry_on_dns_fail and isinstance(e, DNSLookupError):
diff --git a/synapse/http/site.py b/synapse/http/site.py
index e1e53e8ae5f7..2191de9836bd 100644
--- a/synapse/http/site.py
+++ b/synapse/http/site.py
@@ -82,7 +82,7 @@ def __repr__(self):
         )
 
     def get_request_id(self):
-        return "%s-%i" % (self.method, self.request_seq)
+        return "%s-%i" % (self.method.decode('ascii'), self.request_seq)
 
     def get_redacted_uri(self):
         uri = self.uri
@@ -119,7 +119,7 @@ def render(self, resrc):
             # dispatching to the handler, so that the handler
             # can update the servlet name in the request
             # metrics
-            requests_counter.labels(self.method,
+            requests_counter.labels(self.method.decode('ascii'),
                                     self.request_metrics.name).inc()
 
     @contextlib.contextmanager
@@ -280,9 +280,9 @@ def _finished_processing(self):
             int(usage.db_txn_count),
             self.sentLength,
             code,
-            self.method,
+            self.method.decode('ascii'),
             self.get_redacted_uri(),
-            self.clientproto,
+            self.clientproto.decode('ascii', errors='replace'),
             user_agent,
             usage.evt_db_fetch_count,
         )

From 0cb7afff35dacb2b43d695536336a403b14d4b62 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 17 Sep 2018 16:17:25 +0100
Subject: [PATCH 655/830] changelog

---
 changelog.d/3892.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3892.bugfix

diff --git a/changelog.d/3892.bugfix b/changelog.d/3892.bugfix
new file mode 100644
index 000000000000..8b30afab040a
--- /dev/null
+++ b/changelog.d/3892.bugfix
@@ -0,0 +1 @@
+Fix some b'abcd' noise in logs and metrics

From 2b39494cd5a0a93811024c31d8c41e7b5e9e21a3 Mon Sep 17 00:00:00 2001
From: Will Hunt <half-shot@molrams.com>
Date: Mon, 17 Sep 2018 16:35:18 +0100
Subject: [PATCH 656/830] Add python_version phone home stat

---
 synapse/app/homeserver.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index ac97e19649b5..3d9d303839d1 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -457,6 +457,8 @@ def phone_stats_home():
         stats["homeserver"] = hs.config.server_name
         stats["timestamp"] = now
         stats["uptime_seconds"] = uptime
+        version = sys.version_info
+        stats["python_version"] = (version.major) + (version.minor / 10) + (version.micro / 100)
         stats["total_users"] = yield hs.get_datastore().count_all_users()
 
         total_nonbridged_users = yield hs.get_datastore().count_nonbridged_users()

From f75b9961c6c782d2ca4586782aa5cd2c0ae9a5b2 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 17 Sep 2018 16:52:02 +0100
Subject: [PATCH 657/830] Reinstate missing null check

---
 synapse/handlers/directory.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py
index c745e6740b3f..18741c5face2 100644
--- a/synapse/handlers/directory.py
+++ b/synapse/handlers/directory.py
@@ -333,7 +333,7 @@ def can_modify_alias(self, alias, user_id=None):
     def _user_can_delete_alias(self, alias, user_id):
         creator = yield self.store.get_room_alias_creator(alias.to_string())
 
-        if creator == user_id:
+        if creator is not None and creator == user_id:
             defer.returnValue(True)
 
         is_admin = yield self.auth.is_server_admin(UserID.from_string(user_id))

From 9a1cceeca90e6ae52433d50dfd866d68836f686e Mon Sep 17 00:00:00 2001
From: Will Hunt <half-shot@molrams.com>
Date: Mon, 17 Sep 2018 17:09:06 +0100
Subject: [PATCH 658/830] Use a string for versions

---
 synapse/app/homeserver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 3d9d303839d1..be9be423f0bc 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -458,7 +458,7 @@ def phone_stats_home():
         stats["timestamp"] = now
         stats["uptime_seconds"] = uptime
         version = sys.version_info
-        stats["python_version"] = (version.major) + (version.minor / 10) + (version.micro / 100)
+        stats["python_version"] = "{}.{}.{}".format(version.major, version.minor, version.micro)
         stats["total_users"] = yield hs.get_datastore().count_all_users()
 
         total_nonbridged_users = yield hs.get_datastore().count_nonbridged_users()

From ac80cb08fe247812b46c70843b2d5c9764cca619 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 17 Sep 2018 17:16:50 +0100
Subject: [PATCH 659/830] Fix more b'abcd' noise in metrics

---
 synapse/http/request_metrics.py | 22 +++++++++++-----------
 synapse/http/site.py            |  2 +-
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py
index 72c26546788c..fedb4e6b1808 100644
--- a/synapse/http/request_metrics.py
+++ b/synapse/http/request_metrics.py
@@ -162,7 +162,7 @@ def start(self, time_sec, name, method):
         with _in_flight_requests_lock:
             _in_flight_requests.add(self)
 
-    def stop(self, time_sec, request):
+    def stop(self, time_sec, response_code, sent_bytes):
         with _in_flight_requests_lock:
             _in_flight_requests.discard(self)
 
@@ -179,35 +179,35 @@ def stop(self, time_sec, request):
                 )
                 return
 
-        response_code = str(request.code)
+        response_code = str(response_code)
 
-        outgoing_responses_counter.labels(request.method, response_code).inc()
+        outgoing_responses_counter.labels(self.method, response_code).inc()
 
-        response_count.labels(request.method, self.name, tag).inc()
+        response_count.labels(self.method, self.name, tag).inc()
 
-        response_timer.labels(request.method, self.name, tag, response_code).observe(
+        response_timer.labels(self.method, self.name, tag, response_code).observe(
             time_sec - self.start
         )
 
         resource_usage = context.get_resource_usage()
 
-        response_ru_utime.labels(request.method, self.name, tag).inc(
+        response_ru_utime.labels(self.method, self.name, tag).inc(
             resource_usage.ru_utime,
         )
-        response_ru_stime.labels(request.method, self.name, tag).inc(
+        response_ru_stime.labels(self.method, self.name, tag).inc(
             resource_usage.ru_stime,
         )
-        response_db_txn_count.labels(request.method, self.name, tag).inc(
+        response_db_txn_count.labels(self.method, self.name, tag).inc(
             resource_usage.db_txn_count
         )
-        response_db_txn_duration.labels(request.method, self.name, tag).inc(
+        response_db_txn_duration.labels(self.method, self.name, tag).inc(
             resource_usage.db_txn_duration_sec
         )
-        response_db_sched_duration.labels(request.method, self.name, tag).inc(
+        response_db_sched_duration.labels(self.method, self.name, tag).inc(
             resource_usage.db_sched_duration_sec
         )
 
-        response_size.labels(request.method, self.name, tag).inc(request.sentLength)
+        response_size.labels(self.method, self.name, tag).inc(sent_bytes)
 
         # We always call this at the end to ensure that we update the metrics
         # regardless of whether a call to /metrics while the request was in
diff --git a/synapse/http/site.py b/synapse/http/site.py
index 2191de9836bd..9579e8cd0d19 100644
--- a/synapse/http/site.py
+++ b/synapse/http/site.py
@@ -288,7 +288,7 @@ def _finished_processing(self):
         )
 
         try:
-            self.request_metrics.stop(self.finish_time, self)
+            self.request_metrics.stop(self.finish_time, self.code, self.sentLength)
         except Exception as e:
             logger.warn("Failed to stop metrics: %r", e)
 

From 06f2dbbb5dbe7cf9efe66376f47008e86a180f3b Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 17 Sep 2018 17:18:26 +0100
Subject: [PATCH 660/830] changelog

---
 changelog.d/3895.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3895.bugfix

diff --git a/changelog.d/3895.bugfix b/changelog.d/3895.bugfix
new file mode 100644
index 000000000000..8b30afab040a
--- /dev/null
+++ b/changelog.d/3895.bugfix
@@ -0,0 +1 @@
+Fix some b'abcd' noise in logs and metrics

From b58714789fa21fb6907d8c46d3155472d09bc8e6 Mon Sep 17 00:00:00 2001
From: Will Hunt <half-shot@molrams.com>
Date: Mon, 17 Sep 2018 17:35:54 +0100
Subject: [PATCH 661/830] make pip happy?

---
 synapse/app/homeserver.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index be9be423f0bc..ea57350d465f 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -71,7 +71,7 @@
 from synapse.util.manhole import manhole
 from synapse.util.module_loader import load_module
 from synapse.util.rlimit import change_resource_limit
-from synapse.util.versionstring import get_version_string
+from synapse.util.\\\\string import get_version_string
 
 logger = logging.getLogger("synapse.app.homeserver")
 
@@ -458,7 +458,9 @@ def phone_stats_home():
         stats["timestamp"] = now
         stats["uptime_seconds"] = uptime
         version = sys.version_info
-        stats["python_version"] = "{}.{}.{}".format(version.major, version.minor, version.micro)
+        stats["python_version"] = "{}.{}.{}".format(
+            version.major, version.minor, version.micro
+        )
         stats["total_users"] = yield hs.get_datastore().count_all_users()
 
         total_nonbridged_users = yield hs.get_datastore().count_nonbridged_users()

From 5baa087312f056fa3730581d66f220f49eed8fdd Mon Sep 17 00:00:00 2001
From: Will Hunt <half-shot@molrams.com>
Date: Mon, 17 Sep 2018 17:37:56 +0100
Subject: [PATCH 662/830] typo

---
 synapse/app/homeserver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index ea57350d465f..3241ded1889f 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -71,7 +71,7 @@
 from synapse.util.manhole import manhole
 from synapse.util.module_loader import load_module
 from synapse.util.rlimit import change_resource_limit
-from synapse.util.\\\\string import get_version_string
+from synapse.util.versionstring import get_version_string
 
 logger = logging.getLogger("synapse.app.homeserver")
 

From 2b41f2ed76a0dca6bc6f578eba9eb6ad011c4cd4 Mon Sep 17 00:00:00 2001
From: Will Hunt <half-shot@molrams.com>
Date: Mon, 17 Sep 2018 17:48:48 +0100
Subject: [PATCH 663/830] Create 3894.feature

---
 changelog.d/3894.feature | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3894.feature

diff --git a/changelog.d/3894.feature b/changelog.d/3894.feature
new file mode 100644
index 000000000000..1ed0cccdb253
--- /dev/null
+++ b/changelog.d/3894.feature
@@ -0,0 +1 @@
+Report "python_version" in the phone home stats

From 4ecdf73fc7a9e1b8357c6f74897b33b32243dd3f Mon Sep 17 00:00:00 2001
From: Aaron Raimist <aaron@raim.ist>
Date: Mon, 17 Sep 2018 22:05:23 -0500
Subject: [PATCH 664/830] Fix typo in README, synaspse -> synapse

Signed-off-by: Aaron Raimist <aaron@raim.ist>
---
 README.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index 0147429aed26..9c0f9c09c811 100644
--- a/README.rst
+++ b/README.rst
@@ -968,7 +968,7 @@ improvement in overall amount, and especially in terms of giving back RAM
 to the OS. To use it, the library must simply be put in the LD_PRELOAD
 environment variable when launching Synapse. On Debian, this can be done
 by installing the ``libjemalloc1`` package and adding this line to
-``/etc/default/matrix-synaspse``::
+``/etc/default/matrix-synapse``::
 
     LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1
 

From 505abb38f04b8b18adb0b5e5a150238f76b3e4fe Mon Sep 17 00:00:00 2001
From: Aaron Raimist <aaron@raim.ist>
Date: Mon, 17 Sep 2018 22:07:19 -0500
Subject: [PATCH 665/830] Add changelog

Signed-off-by: Aaron Raimist <aaron@raim.ist>
---
 changelog.d/3897.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3897.misc

diff --git a/changelog.d/3897.misc b/changelog.d/3897.misc
new file mode 100644
index 000000000000..87e7ac796e3c
--- /dev/null
+++ b/changelog.d/3897.misc
@@ -0,0 +1 @@
+Fix typo in README, synaspse -> synapse
\ No newline at end of file

From 550007cb0eb12c7ddf0510afdb23195c295de396 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 18 Sep 2018 15:02:51 +0100
Subject: [PATCH 666/830] Bump timeout on get_missing_events request

---
 synapse/handlers/federation.py | 31 ++++++++++++++++++++++++++++++-
 1 file changed, 30 insertions(+), 1 deletion(-)

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 0c68e8a472de..f10b46414bf6 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -360,6 +360,35 @@ def _get_missing_events_for_pdu(self, origin, pdu, prevs, min_depth):
         # apparently.
         #
         # see https://github.com/matrix-org/synapse/pull/1744
+        #
+        # ----
+        #
+        # Update richvdh 2018/09/18: There are a number of problems with timing this
+        # request out agressively on the client side:
+        #
+        # - it plays badly with the server-side rate-limiter, which starts tarpitting you
+        #   if you send too many requests at once, so you end up with the server carefully
+        #   working through the backlog of your requests, which you have already timed
+        #   out.
+        #
+        # - for this request in particular, we now (as of
+        #   https://github.com/matrix-org/synapse/pull/3456) reject any PDUs where the
+        #   server can't produce a plausible-looking set of prev_events - so we becone
+        #   much more likely to reject the event.
+        #
+        # - contrary to what it says above, we do *not* fall back to fetching fresh state
+        #   for the room if get_missing_events times out. Rather, we give up processing
+        #   the PDU whose prevs we are missing, which then makes it much more likely that
+        #   we'll end up back here for the *next* PDU in the list, which exacerbates the
+        #   problem.
+        #
+        # - the agressive 10s timeout was introduced to deal with incoming federation
+        #   requests taking 8 hours to process. It's not entirely clear why that was going
+        #   on; certainly there were other issues causing traffic storms which are now
+        #   resolved, and I think in any case we may be more sensible about our locking
+        #   now. We're *certainly* more sensible about our logging.
+        #
+        # All that said: Let's try increasing the timout to 60s and see what happens.
 
         missing_events = yield self.federation_client.get_missing_events(
             origin,
@@ -368,7 +397,7 @@ def _get_missing_events_for_pdu(self, origin, pdu, prevs, min_depth):
             latest_events=[pdu],
             limit=10,
             min_depth=min_depth,
-            timeout=10000,
+            timeout=60000,
         )
 
         logger.info(

From 8565d9a9fe26d445caaa9ab22d326badc6b0150b Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 18 Sep 2018 15:05:26 +0100
Subject: [PATCH 667/830] changelog

---
 changelog.d/3903.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3903.misc

diff --git a/changelog.d/3903.misc b/changelog.d/3903.misc
new file mode 100644
index 000000000000..49b64bf33355
--- /dev/null
+++ b/changelog.d/3903.misc
@@ -0,0 +1 @@
+Increase the timeout when filling missing events in federation requests
\ No newline at end of file

From b3097396e7cf8d29255b4855e9bf06c6b98a549e Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 18 Sep 2018 17:01:12 +0100
Subject: [PATCH 668/830] Set SNI to the server_name, not whatever was in the
 SRV record

Fixes #3843
---
 synapse/http/endpoint.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/synapse/http/endpoint.py b/synapse/http/endpoint.py
index b0c9369519ee..91025037a301 100644
--- a/synapse/http/endpoint.py
+++ b/synapse/http/endpoint.py
@@ -108,7 +108,7 @@ def matrix_federation_endpoint(reactor, destination, tls_client_options_factory=
 
     Args:
         reactor: Twisted reactor.
-        destination (bytes): The name of the server to connect to.
+        destination (unicode): The name of the server to connect to.
         tls_client_options_factory
             (synapse.crypto.context_factory.ClientTLSOptionsFactory):
             Factory which generates TLS options for client connections.
@@ -126,10 +126,17 @@ def matrix_federation_endpoint(reactor, destination, tls_client_options_factory=
         transport_endpoint = HostnameEndpoint
         default_port = 8008
     else:
+        # the SNI string should be the same as the Host header, minus the port.
+        # as per https://github.com/matrix-org/synapse/issues/2525#issuecomment-336896777,
+        # the Host header and SNI should therefore be the server_name of the remote
+        # server.
+        tls_options = tls_client_options_factory.get_options(domain)
+
         def transport_endpoint(reactor, host, port, timeout):
             return wrapClientTLS(
-                tls_client_options_factory.get_options(host),
-                HostnameEndpoint(reactor, host, port, timeout=timeout))
+                tls_options,
+                HostnameEndpoint(reactor, host, port, timeout=timeout),
+            )
         default_port = 8448
 
     if port is None:

From edabc18938c63a7c6ed16840e22be25a243dcff2 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 18 Sep 2018 17:04:20 +0100
Subject: [PATCH 669/830] changelog

---
 changelog.d/3907.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3907.bugfix

diff --git a/changelog.d/3907.bugfix b/changelog.d/3907.bugfix
new file mode 100644
index 000000000000..45e010c0521d
--- /dev/null
+++ b/changelog.d/3907.bugfix
@@ -0,0 +1 @@
+Fix incorrect server-name indication for outgoing federation requests
\ No newline at end of file

From 31c15dcb80c8f11fd03dbb9b0ccff4777dc8e457 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Tue, 18 Sep 2018 18:17:15 +0100
Subject: [PATCH 670/830] Refactor matrixfederationclient to fix logging
 (#3906)

We want to wait until we have read the response body before we log the request
as complete, otherwise a confusing thing happens where the request appears to
have completed, but we later fail it.

To do this, we factor the salient details of a request out to a separate
object, which can then keep track of the txn_id, so that it can be logged.
---
 changelog.d/3906.misc                    |   1 +
 synapse/http/matrixfederationclient.py   | 385 ++++++++++++++---------
 tests/http/test_fedclient.py             |  43 ++-
 tests/replication/slave/storage/_base.py |  35 +--
 tests/server.py                          |  81 +++++
 5 files changed, 359 insertions(+), 186 deletions(-)
 create mode 100644 changelog.d/3906.misc

diff --git a/changelog.d/3906.misc b/changelog.d/3906.misc
new file mode 100644
index 000000000000..11709186d3e4
--- /dev/null
+++ b/changelog.d/3906.misc
@@ -0,0 +1 @@
+Improve logging of outbound federation requests
\ No newline at end of file
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 083484a687ce..6a2d447289ea 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -17,10 +17,12 @@
 import logging
 import random
 import sys
+from io import BytesIO
 
 from six import PY3, string_types
 from six.moves import urllib
 
+import attr
 import treq
 from canonicaljson import encode_canonical_json
 from prometheus_client import Counter
@@ -28,8 +30,9 @@
 
 from twisted.internet import defer, protocol
 from twisted.internet.error import DNSLookupError
+from twisted.internet.task import _EPSILON, Cooperator
 from twisted.web._newclient import ResponseDone
-from twisted.web.client import Agent, HTTPConnectionPool
+from twisted.web.client import Agent, FileBodyProducer, HTTPConnectionPool
 from twisted.web.http_headers import Headers
 
 import synapse.metrics
@@ -41,13 +44,11 @@
     SynapseError,
 )
 from synapse.http.endpoint import matrix_federation_endpoint
-from synapse.util import logcontext
 from synapse.util.async_helpers import timeout_no_seriously
 from synapse.util.logcontext import make_deferred_yieldable
 from synapse.util.metrics import Measure
 
 logger = logging.getLogger(__name__)
-outbound_logger = logging.getLogger("synapse.http.outbound")
 
 outgoing_requests_counter = Counter("synapse_http_matrixfederationclient_requests",
                                     "", ["method"])
@@ -78,6 +79,93 @@ def endpointForURI(self, uri):
         )
 
 
+_next_id = 1
+
+
+@attr.s
+class MatrixFederationRequest(object):
+    method = attr.ib()
+    """HTTP method
+    :type: str
+    """
+
+    path = attr.ib()
+    """HTTP path
+    :type: str
+    """
+
+    destination = attr.ib()
+    """The remote server to send the HTTP request to.
+    :type: str"""
+
+    json = attr.ib(default=None)
+    """JSON to send in the body.
+    :type: dict|None
+    """
+
+    json_callback = attr.ib(default=None)
+    """A callback to generate the JSON.
+    :type: func|None
+    """
+
+    query = attr.ib(default=None)
+    """Query arguments.
+    :type: dict|None
+    """
+
+    txn_id = attr.ib(default=None)
+    """Unique ID for this request (for logging)
+    :type: str|None
+    """
+
+    def __attrs_post_init__(self):
+        global _next_id
+        self.txn_id = "%s-O-%s" % (self.method, _next_id)
+        _next_id = (_next_id + 1) % (MAXINT - 1)
+
+    def get_json(self):
+        if self.json_callback:
+            return self.json_callback()
+        return self.json
+
+
+@defer.inlineCallbacks
+def _handle_json_response(reactor, timeout_sec, request, response):
+    """
+    Reads the JSON body of a response, with a timeout
+
+    Args:
+        reactor (IReactor): twisted reactor, for the timeout
+        timeout_sec (float): number of seconds to wait for response to complete
+        request (MatrixFederationRequest): the request that triggered the response
+        response (IResponse): response to the request
+
+    Returns:
+        dict: parsed JSON response
+    """
+    try:
+        check_content_type_is_json(response.headers)
+        d = treq.json_content(response)
+        d.addTimeout(timeout_sec, reactor)
+        body = yield make_deferred_yieldable(d)
+    except Exception as e:
+        logger.warn(
+            "{%s} [%d] Error reading response: %s",
+            request.txn_id,
+            request.destination,
+            e,
+        )
+        raise
+    logger.info(
+        "{%s} [%d] Completed: %d %s",
+        request.txn_id,
+        request.destination,
+        response.code,
+        response.phrase.decode('ascii', errors='replace'),
+    )
+    defer.returnValue(body)
+
+
 class MatrixFederationHttpClient(object):
     """HTTP client used to talk to other homeservers over the federation
     protocol. Send client certificates and signs requests.
@@ -102,34 +190,35 @@ def __init__(self, hs):
         self.clock = hs.get_clock()
         self._store = hs.get_datastore()
         self.version_string = hs.version_string.encode('ascii')
-        self._next_id = 1
         self.default_timeout = 60
 
-    def _create_url(self, destination, path_bytes, param_bytes, query_bytes):
-        return urllib.parse.urlunparse(
-            (b"matrix", destination, path_bytes, param_bytes, query_bytes, b"")
-        )
+        def schedule(x):
+            reactor.callLater(_EPSILON, x)
+
+        self._cooperator = Cooperator(scheduler=schedule)
 
     @defer.inlineCallbacks
-    def _request(self, destination, method, path,
-                 json=None, json_callback=None,
-                 param_bytes=b"",
-                 query=None, retry_on_dns_fail=True,
-                 timeout=None, long_retries=False,
-                 ignore_backoff=False,
-                 backoff_on_404=False):
+    def _send_request(
+        self,
+        request,
+        retry_on_dns_fail=True,
+        timeout=None,
+        long_retries=False,
+        ignore_backoff=False,
+        backoff_on_404=False
+    ):
         """
-        Creates and sends a request to the given server.
+        Sends a request to the given server.
 
         Args:
-            destination (str): The remote server to send the HTTP request to.
-            method (str): HTTP method
-            path (str): The HTTP path
-            json (dict or None): JSON to send in the body.
-            json_callback (func or None): A callback to generate the JSON.
-            query (dict or None): Query arguments.
+            request (MatrixFederationRequest): details of request to be sent
+
+            timeout (int|None): number of milliseconds to wait for the response headers
+                (including connecting to the server). 60s by default.
+
             ignore_backoff (bool): true to ignore the historical backoff data
                 and try the request anyway.
+
             backoff_on_404 (bool): Back off if we get a 404
 
         Returns:
@@ -154,38 +243,32 @@ def _request(self, destination, method, path,
 
         if (
             self.hs.config.federation_domain_whitelist is not None and
-            destination not in self.hs.config.federation_domain_whitelist
+            request.destination not in self.hs.config.federation_domain_whitelist
         ):
-            raise FederationDeniedError(destination)
+            raise FederationDeniedError(request.destination)
 
         limiter = yield synapse.util.retryutils.get_retry_limiter(
-            destination,
+            request.destination,
             self.clock,
             self._store,
             backoff_on_404=backoff_on_404,
             ignore_backoff=ignore_backoff,
         )
 
-        headers_dict = {}
-        path_bytes = path.encode("ascii")
-        if query:
-            query_bytes = encode_query_args(query)
+        method = request.method
+        destination = request.destination
+        path_bytes = request.path.encode("ascii")
+        if request.query:
+            query_bytes = encode_query_args(request.query)
         else:
             query_bytes = b""
 
         headers_dict = {
             "User-Agent": [self.version_string],
-            "Host": [destination],
+            "Host": [request.destination],
         }
 
         with limiter:
-            url = self._create_url(
-                destination.encode("ascii"), path_bytes, param_bytes, query_bytes
-            ).decode('ascii')
-
-            txn_id = "%s-O-%s" % (method, self._next_id)
-            self._next_id = (self._next_id + 1) % (MAXINT - 1)
-
             # XXX: Would be much nicer to retry only at the transaction-layer
             # (once we have reliable transactions in place)
             if long_retries:
@@ -193,16 +276,19 @@ def _request(self, destination, method, path,
             else:
                 retries_left = MAX_SHORT_RETRIES
 
-            http_url = urllib.parse.urlunparse(
-                (b"", b"", path_bytes, param_bytes, query_bytes, b"")
-            ).decode('ascii')
+            url = urllib.parse.urlunparse((
+                b"matrix", destination.encode("ascii"),
+                path_bytes, None, query_bytes, b"",
+            )).decode('ascii')
+
+            http_url = urllib.parse.urlunparse((
+                b"", b"",
+                path_bytes, None, query_bytes, b"",
+            )).decode('ascii')
 
-            log_result = None
             while True:
                 try:
-                    if json_callback:
-                        json = json_callback()
-
+                    json = request.get_json()
                     if json:
                         data = encode_canonical_json(json)
                         headers_dict["Content-Type"] = ["application/json"]
@@ -213,16 +299,24 @@ def _request(self, destination, method, path,
                         data = None
                         self.sign_request(destination, method, http_url, headers_dict)
 
-                    outbound_logger.info(
+                    logger.info(
                         "{%s} [%s] Sending request: %s %s",
-                        txn_id, destination, method, url
+                        request.txn_id, destination, method, url
                     )
 
+                    if data:
+                        producer = FileBodyProducer(
+                            BytesIO(data),
+                            cooperator=self._cooperator
+                        )
+                    else:
+                        producer = None
+
                     request_deferred = treq.request(
                         method,
                         url,
                         headers=Headers(headers_dict),
-                        data=data,
+                        data=producer,
                         agent=self.agent,
                         reactor=self.hs.get_reactor(),
                         unbuffered=True
@@ -244,33 +338,19 @@ def _request(self, destination, method, path,
                             request_deferred,
                         )
 
-                    log_result = "%d %s" % (
-                        response.code,
-                        response.phrase.decode('ascii', errors='replace'),
-                    )
                     break
                 except Exception as e:
-                    if not retry_on_dns_fail and isinstance(e, DNSLookupError):
-                        logger.warn(
-                            "DNS Lookup failed to %s with %s",
-                            destination,
-                            e
-                        )
-                        log_result = "DNS Lookup failed to %s with %s" % (
-                            destination, e
-                        )
-                        raise
-
                     logger.warn(
-                        "{%s} Sending request failed to %s: %s %s: %s",
-                        txn_id,
+                        "{%s} [%s] Request failed: %s %s: %s",
+                        request.txn_id,
                         destination,
                         method,
                         url,
                         _flatten_response_never_received(e),
                     )
 
-                    log_result = _flatten_response_never_received(e)
+                    if not retry_on_dns_fail and isinstance(e, DNSLookupError):
+                        raise
 
                     if retries_left and not timeout:
                         if long_retries:
@@ -283,33 +363,33 @@ def _request(self, destination, method, path,
                             delay *= random.uniform(0.8, 1.4)
 
                         logger.debug(
-                            "{%s} Waiting %s before sending to %s...",
-                            txn_id,
+                            "{%s} [%s] Waiting %ss before re-sending...",
+                            request.txn_id,
+                            destination,
                             delay,
-                            destination
                         )
 
                         yield self.clock.sleep(delay)
                         retries_left -= 1
                     else:
                         raise
-                finally:
-                    outbound_logger.info(
-                        "{%s} [%s] Result: %s",
-                        txn_id,
-                        destination,
-                        log_result,
-                    )
+
+            logger.info(
+                "{%s} [%s] Got response headers: %d %s",
+                request.txn_id,
+                destination,
+                response.code,
+                response.phrase.decode('ascii', errors='replace'),
+            )
 
             if 200 <= response.code < 300:
                 pass
             else:
                 # :'(
                 # Update transactions table?
-                with logcontext.PreserveLoggingContext():
-                    d = treq.content(response)
-                    d.addTimeout(_sec_timeout, self.hs.get_reactor())
-                    body = yield make_deferred_yieldable(d)
+                d = treq.content(response)
+                d.addTimeout(_sec_timeout, self.hs.get_reactor())
+                body = yield make_deferred_yieldable(d)
                 raise HttpResponseException(
                     response.code, response.phrase, body
                 )
@@ -403,29 +483,26 @@ def put_json(self, destination, path, args={}, data={},
             is not on our federation whitelist
         """
 
-        if not json_data_callback:
-            json_data_callback = lambda: data
-
-        response = yield self._request(
-            destination,
-            "PUT",
-            path,
-            json_callback=json_data_callback,
+        request = MatrixFederationRequest(
+            method="PUT",
+            destination=destination,
+            path=path,
             query=args,
+            json_callback=json_data_callback,
+            json=data,
+        )
+
+        response = yield self._send_request(
+            request,
             long_retries=long_retries,
             timeout=timeout,
             ignore_backoff=ignore_backoff,
             backoff_on_404=backoff_on_404,
         )
 
-        if 200 <= response.code < 300:
-            # We need to update the transactions table to say it was sent?
-            check_content_type_is_json(response.headers)
-
-        with logcontext.PreserveLoggingContext():
-            d = treq.json_content(response)
-            d.addTimeout(self.default_timeout, self.hs.get_reactor())
-            body = yield make_deferred_yieldable(d)
+        body = yield _handle_json_response(
+            self.hs.get_reactor(), self.default_timeout, request, response,
+        )
         defer.returnValue(body)
 
     @defer.inlineCallbacks
@@ -459,31 +536,30 @@ def post_json(self, destination, path, data={}, long_retries=False,
             Fails with ``FederationDeniedError`` if this destination
             is not on our federation whitelist
         """
-        response = yield self._request(
-            destination,
-            "POST",
-            path,
+
+        request = MatrixFederationRequest(
+            method="POST",
+            destination=destination,
+            path=path,
             query=args,
             json=data,
+        )
+
+        response = yield self._send_request(
+            request,
             long_retries=long_retries,
             timeout=timeout,
             ignore_backoff=ignore_backoff,
         )
 
-        if 200 <= response.code < 300:
-            # We need to update the transactions table to say it was sent?
-            check_content_type_is_json(response.headers)
-
-        with logcontext.PreserveLoggingContext():
-            d = treq.json_content(response)
-            if timeout:
-                _sec_timeout = timeout / 1000
-            else:
-                _sec_timeout = self.default_timeout
-
-            d.addTimeout(_sec_timeout, self.hs.get_reactor())
-            body = yield make_deferred_yieldable(d)
+        if timeout:
+            _sec_timeout = timeout / 1000
+        else:
+            _sec_timeout = self.default_timeout
 
+        body = yield _handle_json_response(
+            self.hs.get_reactor(), _sec_timeout, request, response,
+        )
         defer.returnValue(body)
 
     @defer.inlineCallbacks
@@ -519,25 +595,23 @@ def get_json(self, destination, path, args=None, retry_on_dns_fail=True,
 
         logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail)
 
-        response = yield self._request(
-            destination,
-            "GET",
-            path,
+        request = MatrixFederationRequest(
+            method="GET",
+            destination=destination,
+            path=path,
             query=args,
+        )
+
+        response = yield self._send_request(
+            request,
             retry_on_dns_fail=retry_on_dns_fail,
             timeout=timeout,
             ignore_backoff=ignore_backoff,
         )
 
-        if 200 <= response.code < 300:
-            # We need to update the transactions table to say it was sent?
-            check_content_type_is_json(response.headers)
-
-        with logcontext.PreserveLoggingContext():
-            d = treq.json_content(response)
-            d.addTimeout(self.default_timeout, self.hs.get_reactor())
-            body = yield make_deferred_yieldable(d)
-
+        body = yield _handle_json_response(
+            self.hs.get_reactor(), self.default_timeout, request, response,
+        )
         defer.returnValue(body)
 
     @defer.inlineCallbacks
@@ -568,25 +642,23 @@ def delete_json(self, destination, path, long_retries=False,
             Fails with ``FederationDeniedError`` if this destination
             is not on our federation whitelist
         """
-        response = yield self._request(
-            destination,
-            "DELETE",
-            path,
+        request = MatrixFederationRequest(
+            method="DELETE",
+            destination=destination,
+            path=path,
             query=args,
+        )
+
+        response = yield self._send_request(
+            request,
             long_retries=long_retries,
             timeout=timeout,
             ignore_backoff=ignore_backoff,
         )
 
-        if 200 <= response.code < 300:
-            # We need to update the transactions table to say it was sent?
-            check_content_type_is_json(response.headers)
-
-        with logcontext.PreserveLoggingContext():
-            d = treq.json_content(response)
-            d.addTimeout(self.default_timeout, self.hs.get_reactor())
-            body = yield make_deferred_yieldable(d)
-
+        body = yield _handle_json_response(
+            self.hs.get_reactor(), self.default_timeout, request, response,
+        )
         defer.returnValue(body)
 
     @defer.inlineCallbacks
@@ -614,11 +686,15 @@ def get_file(self, destination, path, output_stream, args={},
             Fails with ``FederationDeniedError`` if this destination
             is not on our federation whitelist
         """
-        response = yield self._request(
-            destination,
-            "GET",
-            path,
+        request = MatrixFederationRequest(
+            method="GET",
+            destination=destination,
+            path=path,
             query=args,
+        )
+
+        response = yield self._send_request(
+            request,
             retry_on_dns_fail=retry_on_dns_fail,
             ignore_backoff=ignore_backoff,
         )
@@ -626,14 +702,25 @@ def get_file(self, destination, path, output_stream, args={},
         headers = dict(response.headers.getAllRawHeaders())
 
         try:
-            with logcontext.PreserveLoggingContext():
-                d = _readBodyToFile(response, output_stream, max_size)
-                d.addTimeout(self.default_timeout, self.hs.get_reactor())
-                length = yield make_deferred_yieldable(d)
-        except Exception:
-            logger.exception("Failed to download body")
+            d = _readBodyToFile(response, output_stream, max_size)
+            d.addTimeout(self.default_timeout, self.hs.get_reactor())
+            length = yield make_deferred_yieldable(d)
+        except Exception as e:
+            logger.warn(
+                "{%s} [%d] Error reading response: %s",
+                request.txn_id,
+                request.destination,
+                e,
+            )
             raise
-
+        logger.info(
+            "{%s} [%d] Completed: %d %s [%d bytes]",
+            request.txn_id,
+            request.destination,
+            response.code,
+            response.phrase.decode('ascii', errors='replace'),
+            length,
+        )
         defer.returnValue((length, headers))
 
 
diff --git a/tests/http/test_fedclient.py b/tests/http/test_fedclient.py
index 1c46c9cfeb9e..66c09f63b6b4 100644
--- a/tests/http/test_fedclient.py
+++ b/tests/http/test_fedclient.py
@@ -18,9 +18,14 @@
 from twisted.internet.defer import TimeoutError
 from twisted.internet.error import ConnectingCancelledError, DNSLookupError
 from twisted.web.client import ResponseNeverReceived
+from twisted.web.http import HTTPChannel
 
-from synapse.http.matrixfederationclient import MatrixFederationHttpClient
+from synapse.http.matrixfederationclient import (
+    MatrixFederationHttpClient,
+    MatrixFederationRequest,
+)
 
+from tests.server import FakeTransport
 from tests.unittest import HomeserverTestCase
 
 
@@ -40,7 +45,7 @@ def test_dns_error(self):
         """
         If the DNS raising returns an error, it will bubble up.
         """
-        d = self.cl._request("testserv2:8008", "GET", "foo/bar", timeout=10000)
+        d = self.cl.get_json("testserv2:8008", "foo/bar", timeout=10000)
         self.pump()
 
         f = self.failureResultOf(d)
@@ -51,7 +56,7 @@ def test_client_never_connect(self):
         If the HTTP request is not connected and is timed out, it'll give a
         ConnectingCancelledError.
         """
-        d = self.cl._request("testserv:8008", "GET", "foo/bar", timeout=10000)
+        d = self.cl.get_json("testserv:8008", "foo/bar", timeout=10000)
 
         self.pump()
 
@@ -78,7 +83,7 @@ def test_client_connect_no_response(self):
         If the HTTP request is connected, but gets no response before being
         timed out, it'll give a ResponseNeverReceived.
         """
-        d = self.cl._request("testserv:8008", "GET", "foo/bar", timeout=10000)
+        d = self.cl.get_json("testserv:8008", "foo/bar", timeout=10000)
 
         self.pump()
 
@@ -108,7 +113,12 @@ def test_client_gets_headers(self):
         """
         Once the client gets the headers, _request returns successfully.
         """
-        d = self.cl._request("testserv:8008", "GET", "foo/bar", timeout=10000)
+        request = MatrixFederationRequest(
+            method="GET",
+            destination="testserv:8008",
+            path="foo/bar",
+        )
+        d = self.cl._send_request(request, timeout=10000)
 
         self.pump()
 
@@ -155,3 +165,26 @@ def test_client_headers_no_body(self):
         f = self.failureResultOf(d)
 
         self.assertIsInstance(f.value, TimeoutError)
+
+    def test_client_sends_body(self):
+        self.cl.post_json(
+            "testserv:8008", "foo/bar", timeout=10000,
+            data={"a": "b"}
+        )
+
+        self.pump()
+
+        clients = self.reactor.tcpClients
+        self.assertEqual(len(clients), 1)
+        client = clients[0][2].buildProtocol(None)
+        server = HTTPChannel()
+
+        client.makeConnection(FakeTransport(server, self.reactor))
+        server.makeConnection(FakeTransport(client, self.reactor))
+
+        self.pump(0.1)
+
+        self.assertEqual(len(server.requests), 1)
+        request = server.requests[0]
+        content = request.content.read()
+        self.assertEqual(content, b'{"a":"b"}')
diff --git a/tests/replication/slave/storage/_base.py b/tests/replication/slave/storage/_base.py
index 089cecfbee16..9e9fbbfe930d 100644
--- a/tests/replication/slave/storage/_base.py
+++ b/tests/replication/slave/storage/_base.py
@@ -15,8 +15,6 @@
 
 from mock import Mock, NonCallableMock
 
-import attr
-
 from synapse.replication.tcp.client import (
     ReplicationClientFactory,
     ReplicationClientHandler,
@@ -24,6 +22,7 @@
 from synapse.replication.tcp.resource import ReplicationStreamProtocolFactory
 
 from tests import unittest
+from tests.server import FakeTransport
 
 
 class BaseSlavedStoreTestCase(unittest.HomeserverTestCase):
@@ -56,36 +55,8 @@ def prepare(self, reactor, clock, hs):
         server = server_factory.buildProtocol(None)
         client = client_factory.buildProtocol(None)
 
-        @attr.s
-        class FakeTransport(object):
-
-            other = attr.ib()
-            disconnecting = False
-            buffer = attr.ib(default=b'')
-
-            def registerProducer(self, producer, streaming):
-
-                self.producer = producer
-
-                def _produce():
-                    self.producer.resumeProducing()
-                    reactor.callLater(0.1, _produce)
-
-                reactor.callLater(0.0, _produce)
-
-            def write(self, byt):
-                self.buffer = self.buffer + byt
-
-                if getattr(self.other, "transport") is not None:
-                    self.other.dataReceived(self.buffer)
-                    self.buffer = b""
-
-            def writeSequence(self, seq):
-                for x in seq:
-                    self.write(x)
-
-        client.makeConnection(FakeTransport(server))
-        server.makeConnection(FakeTransport(client))
+        client.makeConnection(FakeTransport(server, reactor))
+        server.makeConnection(FakeTransport(client, reactor))
 
     def replicate(self):
         """Tell the master side of replication that something has happened, and then
diff --git a/tests/server.py b/tests/server.py
index 420ec4e08859..ccea3baa554e 100644
--- a/tests/server.py
+++ b/tests/server.py
@@ -280,3 +280,84 @@ def get_clock():
     clock = ThreadedMemoryReactorClock()
     hs_clock = Clock(clock)
     return (clock, hs_clock)
+
+
+@attr.s
+class FakeTransport(object):
+    """
+    A twisted.internet.interfaces.ITransport implementation which sends all its data
+    straight into an IProtocol object: it exists to connect two IProtocols together.
+
+    To use it, instantiate it with the receiving IProtocol, and then pass it to the
+    sending IProtocol's makeConnection method:
+
+        server = HTTPChannel()
+        client.makeConnection(FakeTransport(server, self.reactor))
+
+    If you want bidirectional communication, you'll need two instances.
+    """
+
+    other = attr.ib()
+    """The Protocol object which will receive any data written to this transport.
+
+    :type: twisted.internet.interfaces.IProtocol
+    """
+
+    _reactor = attr.ib()
+    """Test reactor
+
+    :type: twisted.internet.interfaces.IReactorTime
+    """
+
+    disconnecting = False
+    buffer = attr.ib(default=b'')
+    producer = attr.ib(default=None)
+
+    def getPeer(self):
+        return None
+
+    def getHost(self):
+        return None
+
+    def loseConnection(self):
+        self.disconnecting = True
+
+    def abortConnection(self):
+        self.disconnecting = True
+
+    def pauseProducing(self):
+        self.producer.pauseProducing()
+
+    def unregisterProducer(self):
+        if not self.producer:
+            return
+
+        self.producer = None
+
+    def registerProducer(self, producer, streaming):
+        self.producer = producer
+        self.producerStreaming = streaming
+
+        def _produce():
+            d = self.producer.resumeProducing()
+            d.addCallback(lambda x: self._reactor.callLater(0.1, _produce))
+
+        if not streaming:
+            self._reactor.callLater(0.0, _produce)
+
+    def write(self, byt):
+        self.buffer = self.buffer + byt
+
+        def _write():
+            if getattr(self.other, "transport") is not None:
+                self.other.dataReceived(self.buffer)
+                self.buffer = b""
+                return
+
+            self._reactor.callLater(0.0, _write)
+
+        _write()
+
+    def writeSequence(self, seq):
+        for x in seq:
+            self.write(x)

From a219ce87263ad9be887cf039a04b4a1f06b7b0b8 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Tue, 18 Sep 2018 18:27:37 +0100
Subject: [PATCH 671/830] Use directory server for room joins (#3899)

When we do a join, always try the server we used for the alias lookup first.

Fixes #2418
---
 changelog.d/3899.bugfix         | 1 +
 synapse/handlers/room_member.py | 5 +++++
 2 files changed, 6 insertions(+)
 create mode 100644 changelog.d/3899.bugfix

diff --git a/changelog.d/3899.bugfix b/changelog.d/3899.bugfix
new file mode 100644
index 000000000000..5120e3a82309
--- /dev/null
+++ b/changelog.d/3899.bugfix
@@ -0,0 +1 @@
+When we join a room, always try the server we used for the alias lookup first, to avoid unresponsive and out-of-date servers.
diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index f6436190470a..07fd3e82fcad 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -583,6 +583,11 @@ def lookup_room_alias(self, room_alias):
         room_id = mapping["room_id"]
         servers = mapping["servers"]
 
+        # put the server which owns the alias at the front of the server list.
+        if room_alias.domain in servers:
+            servers.remove(room_alias.domain)
+        servers.insert(0, room_alias.domain)
+
         defer.returnValue((RoomID.from_string(room_id), servers))
 
     @defer.inlineCallbacks

From 35aec19f0ac88e7b16bc6bcc4f4e546b7fc89589 Mon Sep 17 00:00:00 2001
From: Travis Ralston <travpc@gmail.com>
Date: Tue, 18 Sep 2018 15:26:08 -0600
Subject: [PATCH 672/830] Destination is a string

---
 synapse/http/matrixfederationclient.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 6a2d447289ea..530c0245a93b 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -150,14 +150,14 @@ def _handle_json_response(reactor, timeout_sec, request, response):
         body = yield make_deferred_yieldable(d)
     except Exception as e:
         logger.warn(
-            "{%s} [%d] Error reading response: %s",
+            "{%s} [%s] Error reading response: %s",
             request.txn_id,
             request.destination,
             e,
         )
         raise
     logger.info(
-        "{%s} [%d] Completed: %d %s",
+        "{%s} [%s] Completed: %d %s",
         request.txn_id,
         request.destination,
         response.code,
@@ -707,14 +707,14 @@ def get_file(self, destination, path, output_stream, args={},
             length = yield make_deferred_yieldable(d)
         except Exception as e:
             logger.warn(
-                "{%s} [%d] Error reading response: %s",
+                "{%s} [%s] Error reading response: %s",
                 request.txn_id,
                 request.destination,
                 e,
             )
             raise
         logger.info(
-            "{%s} [%d] Completed: %d %s [%d bytes]",
+            "{%s} [%s] Completed: %d %s [%d bytes]",
             request.txn_id,
             request.destination,
             response.code,

From 13b31a9bafcecac13b4c8699f054a1cb5aa67cdd Mon Sep 17 00:00:00 2001
From: Travis Ralston <travpc@gmail.com>
Date: Tue, 18 Sep 2018 15:30:25 -0600
Subject: [PATCH 673/830] Changelog

---
 changelog.d/3909.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3909.misc

diff --git a/changelog.d/3909.misc b/changelog.d/3909.misc
new file mode 100644
index 000000000000..a8d72dacad5a
--- /dev/null
+++ b/changelog.d/3909.misc
@@ -0,0 +1 @@
+Fix log format error in matrixfederationclient.py

From 3f0d8e6b09d33a026df3ef403e1c8a264637d71c Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Wed, 19 Sep 2018 18:14:30 +1000
Subject: [PATCH 674/830] Remove documentation referencing Cygwin (#3873)

---
 .gitignore            |  1 +
 README.rst            | 38 +++++++-------------------------------
 changelog.d/3873.misc |  2 ++
 3 files changed, 10 insertions(+), 31 deletions(-)
 create mode 100644 changelog.d/3873.misc

diff --git a/.gitignore b/.gitignore
index 1718185384a4..a72572523571 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,6 +4,7 @@
 
 .DS_Store
 _trial_temp/
+_trial_temp*/
 logs/
 dbs/
 *.egg
diff --git a/README.rst b/README.rst
index 9c0f9c09c811..5547f617ba23 100644
--- a/README.rst
+++ b/README.rst
@@ -157,7 +157,7 @@ if you prefer.
 
 In case of problems, please see the _`Troubleshooting` section below.
 
-There is an offical synapse image available at 
+There is an offical synapse image available at
 https://hub.docker.com/r/matrixdotorg/synapse/tags/ which can be used with
 the docker-compose file available at `contrib/docker <contrib/docker>`_. Further information on
 this including configuration options is available in the README on
@@ -459,37 +459,13 @@ https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/misc/matrix-
 
 Windows Install
 ---------------
-Synapse can be installed on Cygwin. It requires the following Cygwin packages:
-
-- gcc
-- git
-- libffi-devel
-- openssl (and openssl-devel, python-openssl)
-- python
-- python-setuptools
-
-The content repository requires additional packages and will be unable to process
-uploads without them:
-
-- libjpeg8
-- libjpeg8-devel
-- zlib
-
-If you choose to install Synapse without these packages, you will need to reinstall
-``pillow`` for changes to be applied, e.g. ``pip uninstall pillow`` ``pip install
-pillow --user``
-
-Troubleshooting:
-
-- You may need to upgrade ``setuptools`` to get this to work correctly:
-  ``pip install setuptools --upgrade``.
-- You may encounter errors indicating that ``ffi.h`` is missing, even with
-  ``libffi-devel`` installed. If you do, copy the ``.h`` files:
-  ``cp /usr/lib/libffi-3.0.13/include/*.h /usr/include``
-- You may need to install libsodium from source in order to install PyNacl. If
-  you do, you may need to create a symlink to ``libsodium.a`` so ``ld`` can find
-  it: ``ln -s /usr/local/lib/libsodium.a /usr/lib/libsodium.a``
 
+If you wish to run or develop Synapse on Windows, the Windows Subsystem For
+Linux provides a Linux environment on Windows 10 which is capable of using the
+Debian, Fedora, or source installation methods. More information about WSL can
+be found at https://docs.microsoft.com/en-us/windows/wsl/install-win10 for
+Windows 10 and https://docs.microsoft.com/en-us/windows/wsl/install-on-server
+for Windows Server.
 
 Troubleshooting
 ===============
diff --git a/changelog.d/3873.misc b/changelog.d/3873.misc
new file mode 100644
index 000000000000..8104b5c085f6
--- /dev/null
+++ b/changelog.d/3873.misc
@@ -0,0 +1,2 @@
+Remove documentation regarding installation on Cygwin, the use of WSL is 
+recommended instead.

From 05b9937cd77bd02dc8f38d91ffda3db982a2665f Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 19 Sep 2018 09:17:54 +0100
Subject: [PATCH 675/830] update changelog for #3909

---
 changelog.d/3909.misc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/changelog.d/3909.misc b/changelog.d/3909.misc
index a8d72dacad5a..11709186d3e4 100644
--- a/changelog.d/3909.misc
+++ b/changelog.d/3909.misc
@@ -1 +1 @@
-Fix log format error in matrixfederationclient.py
+Improve logging of outbound federation requests
\ No newline at end of file

From a334e1cacec5dd9c1a983415906d8ffea4a30134 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 19 Sep 2018 10:39:40 +0100
Subject: [PATCH 676/830] Update to use new timeout function everywhere.

The existing deferred timeout helper function (and the one into twisted)
suffer from a bug when a deferred's canceller throws an exception, #3842.

The new helper function doesn't suffer from this problem.
---
 synapse/http/client.py                 |  4 +-
 synapse/http/matrixfederationclient.py | 25 +++++----
 synapse/notifier.py                    | 13 ++---
 synapse/util/async_helpers.py          | 73 +++++++-------------------
 4 files changed, 43 insertions(+), 72 deletions(-)

diff --git a/synapse/http/client.py b/synapse/http/client.py
index ec339a92ade5..3d05f83b8c3f 100644
--- a/synapse/http/client.py
+++ b/synapse/http/client.py
@@ -43,7 +43,7 @@
 from synapse.api.errors import Codes, HttpResponseException, SynapseError
 from synapse.http import cancelled_to_request_timed_out_error, redact_uri
 from synapse.http.endpoint import SpiderEndpoint
-from synapse.util.async_helpers import add_timeout_to_deferred
+from synapse.util.async_helpers import timeout_deferred
 from synapse.util.caches import CACHE_SIZE_FACTOR
 from synapse.util.logcontext import make_deferred_yieldable
 
@@ -99,7 +99,7 @@ def request(self, method, uri, data=b'', headers=None):
             request_deferred = treq.request(
                 method, uri, agent=self.agent, data=data, headers=headers
             )
-            add_timeout_to_deferred(
+            request_deferred = timeout_deferred(
                 request_deferred, 60, self.hs.get_reactor(),
                 cancelled_to_request_timed_out_error,
             )
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 530c0245a93b..14b12cd1c44c 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -44,7 +44,7 @@
     SynapseError,
 )
 from synapse.http.endpoint import matrix_federation_endpoint
-from synapse.util.async_helpers import timeout_no_seriously
+from synapse.util.async_helpers import timeout_deferred
 from synapse.util.logcontext import make_deferred_yieldable
 from synapse.util.metrics import Measure
 
@@ -145,8 +145,14 @@ def _handle_json_response(reactor, timeout_sec, request, response):
     """
     try:
         check_content_type_is_json(response.headers)
+
         d = treq.json_content(response)
-        d.addTimeout(timeout_sec, reactor)
+        d = timeout_deferred(
+            d,
+            timeout=timeout_sec,
+            reactor=reactor,
+        )
+
         body = yield make_deferred_yieldable(d)
     except Exception as e:
         logger.warn(
@@ -321,15 +327,10 @@ def _send_request(
                         reactor=self.hs.get_reactor(),
                         unbuffered=True
                     )
-                    request_deferred.addTimeout(_sec_timeout, self.hs.get_reactor())
 
-                    # Sometimes the timeout above doesn't work, so lets hack yet
-                    # another layer of timeouts in in the vain hope that at some
-                    # point the world made sense and this really really really
-                    # should work.
-                    request_deferred = timeout_no_seriously(
+                    request_deferred = timeout_deferred(
                         request_deferred,
-                        timeout=_sec_timeout * 2,
+                        timeout=_sec_timeout,
                         reactor=self.hs.get_reactor(),
                     )
 
@@ -388,7 +389,11 @@ def _send_request(
                 # :'(
                 # Update transactions table?
                 d = treq.content(response)
-                d.addTimeout(_sec_timeout, self.hs.get_reactor())
+                d = timeout_deferred(
+                    d,
+                    timeout=_sec_timeout,
+                    reactor=self.hs.get_reactor(),
+                )
                 body = yield make_deferred_yieldable(d)
                 raise HttpResponseException(
                     response.code, response.phrase, body
diff --git a/synapse/notifier.py b/synapse/notifier.py
index 82f391481c2c..2d683718fb7b 100644
--- a/synapse/notifier.py
+++ b/synapse/notifier.py
@@ -28,7 +28,7 @@
 from synapse.util.async_helpers import (
     DeferredTimeoutError,
     ObservableDeferred,
-    add_timeout_to_deferred,
+    timeout_deferred,
 )
 from synapse.util.logcontext import PreserveLoggingContext, run_in_background
 from synapse.util.logutils import log_function
@@ -337,7 +337,7 @@ def wait_for_events(self, user_id, timeout, callback, room_ids=None,
                     # Now we wait for the _NotifierUserStream to be told there
                     # is a new token.
                     listener = user_stream.new_listener(prev_token)
-                    add_timeout_to_deferred(
+                    listener.deferred = timeout_deferred(
                         listener.deferred,
                         (end_time - now) / 1000.,
                         self.hs.get_reactor(),
@@ -559,11 +559,12 @@ def wait_for_replication(self, callback, timeout):
             if end_time <= now:
                 break
 
-            add_timeout_to_deferred(
-                listener.deferred.addTimeout,
-                (end_time - now) / 1000.,
-                self.hs.get_reactor(),
+            listener.deferred = timeout_deferred(
+                listener.deferred,
+                timeout=(end_time - now) / 1000.,
+                reactor=self.hs.get_reactor(),
             )
+
             try:
                 with PreserveLoggingContext():
                     yield listener.deferred
diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py
index 40c294612990..2e12bcda9815 100644
--- a/synapse/util/async_helpers.py
+++ b/synapse/util/async_helpers.py
@@ -380,23 +380,25 @@ class DeferredTimeoutError(Exception):
     """
 
 
-def add_timeout_to_deferred(deferred, timeout, reactor, on_timeout_cancel=None):
-    """
-    Add a timeout to a deferred by scheduling it to be cancelled after
-    timeout seconds.
+def _cancelled_to_timed_out_error(value, timeout):
+    if isinstance(value, failure.Failure):
+        value.trap(CancelledError)
+        raise DeferredTimeoutError(timeout, "Deferred")
+    return value
 
-    This is essentially a backport of deferred.addTimeout, which was introduced
-    in twisted 16.5.
 
-    If the deferred gets timed out, it errbacks with a DeferredTimeoutError,
-    unless a cancelable function was passed to its initialization or unless
-    a different on_timeout_cancel callable is provided.
+def timeout_deferred(deferred, timeout, reactor, on_timeout_cancel=None):
+    """The in built twisted `Deferred.addTimeout` fails to time out deferreds
+    that have a canceller that throws exceptions. This method creates a new
+    deferred that wraps and times out the given deferred, correctly handling
+    the case where the given deferred's canceller throws.
 
-    Args:
-        deferred (defer.Deferred): deferred to be timed out
-        timeout (Number): seconds to time out after
-        reactor (twisted.internet.reactor): the Twisted reactor to use
+    NOTE: Unlike `Deferred.addTimeout`, this function returns a new deferred
 
+    Args:
+        deferred (Deferred)
+        timeout (float): Timeout in seconds
+        reactor (twisted.internet.reactor): The twisted reactor to use
         on_timeout_cancel (callable): A callable which is called immediately
             after the deferred times out, and not if this deferred is
             otherwise cancelled before the timeout.
@@ -407,47 +409,9 @@ def add_timeout_to_deferred(deferred, timeout, reactor, on_timeout_cancel=None):
 
             The default callable (if none is provided) will translate a
             CancelledError Failure into a DeferredTimeoutError.
-    """
-    timed_out = [False]
 
-    def time_it_out():
-        timed_out[0] = True
-        deferred.cancel()
-
-    delayed_call = reactor.callLater(timeout, time_it_out)
-
-    def convert_cancelled(value):
-        if timed_out[0]:
-            to_call = on_timeout_cancel or _cancelled_to_timed_out_error
-            return to_call(value, timeout)
-        return value
-
-    deferred.addBoth(convert_cancelled)
-
-    def cancel_timeout(result):
-        # stop the pending call to cancel the deferred if it's been fired
-        if delayed_call.active():
-            delayed_call.cancel()
-        return result
-
-    deferred.addBoth(cancel_timeout)
-
-
-def _cancelled_to_timed_out_error(value, timeout):
-    if isinstance(value, failure.Failure):
-        value.trap(CancelledError)
-        raise DeferredTimeoutError(timeout, "Deferred")
-    return value
-
-
-def timeout_no_seriously(deferred, timeout, reactor):
-    """The in build twisted deferred addTimeout (and the method above)
-    completely fail to time things out under some unknown circumstances.
-
-    Lets try a different way of timing things out and maybe that will make
-    things work?!
-
-    TODO: Kill this with fire.
+    Returns:
+        Deferred
     """
 
     new_d = defer.Deferred()
@@ -466,7 +430,8 @@ def time_it_out():
 
     def convert_cancelled(value):
         if timed_out[0]:
-            return _cancelled_to_timed_out_error(value, timeout)
+            to_call = on_timeout_cancel or _cancelled_to_timed_out_error
+            return to_call(value, timeout)
         return value
 
     deferred.addBoth(convert_cancelled)

From 6bbe3d5732630ad7f58159d267159bda2b5bfb08 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 19 Sep 2018 10:43:13 +0100
Subject: [PATCH 677/830] Newsfile

---
 changelog.d/3910 | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3910

diff --git a/changelog.d/3910 b/changelog.d/3910
new file mode 100644
index 000000000000..22ec2adc33d5
--- /dev/null
+++ b/changelog.d/3910
@@ -0,0 +1 @@
+Fix bug where things occaisonally were not being timed out correctly.

From 6c48aa0256ce7dfd62e72a77b03f5bb2222473ba Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 19 Sep 2018 11:04:42 +0100
Subject: [PATCH 678/830] Run canceller first to allow it to generate correct
 error

---
 synapse/util/async_helpers.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py
index 2e12bcda9815..2b2e85ecb755 100644
--- a/synapse/util/async_helpers.py
+++ b/synapse/util/async_helpers.py
@@ -421,11 +421,14 @@ def timeout_deferred(deferred, timeout, reactor, on_timeout_cancel=None):
     def time_it_out():
         timed_out[0] = True
 
+        try:
+            deferred.cancel()
+        except:   # noqa: E722, if we throw any exception it'll break time outs
+            logger.exception("Canceller failed during timeout")
+
         if not new_d.called:
             new_d.errback(DeferredTimeoutError(timeout, "Deferred"))
 
-        deferred.cancel()
-
     delayed_call = reactor.callLater(timeout, time_it_out)
 
     def convert_cancelled(value):

From 9407bcf37a99ebb72eef73e19632cdcf5964c968 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 19 Sep 2018 11:05:21 +0100
Subject: [PATCH 679/830] Replace custom DeferredTimeoutError with
 defer.TimeoutError

---
 synapse/notifier.py           |  5 ++---
 synapse/util/async_helpers.py | 12 +++---------
 2 files changed, 5 insertions(+), 12 deletions(-)

diff --git a/synapse/notifier.py b/synapse/notifier.py
index 2d683718fb7b..2e7525b7d288 100644
--- a/synapse/notifier.py
+++ b/synapse/notifier.py
@@ -26,7 +26,6 @@
 from synapse.metrics import LaterGauge
 from synapse.types import StreamToken
 from synapse.util.async_helpers import (
-    DeferredTimeoutError,
     ObservableDeferred,
     timeout_deferred,
 )
@@ -354,7 +353,7 @@ def wait_for_events(self, user_id, timeout, callback, room_ids=None,
                     # Update the prev_token to the current_token since nothing
                     # has happened between the old prev_token and the current_token
                     prev_token = current_token
-                except DeferredTimeoutError:
+                except defer.TimeoutError:
                     break
                 except defer.CancelledError:
                     break
@@ -568,7 +567,7 @@ def wait_for_replication(self, callback, timeout):
             try:
                 with PreserveLoggingContext():
                     yield listener.deferred
-            except DeferredTimeoutError:
+            except defer.TimeoutError:
                 break
             except defer.CancelledError:
                 break
diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py
index 2b2e85ecb755..ec7b2c967222 100644
--- a/synapse/util/async_helpers.py
+++ b/synapse/util/async_helpers.py
@@ -374,16 +374,10 @@ def _ctx_manager():
         defer.returnValue(_ctx_manager())
 
 
-class DeferredTimeoutError(Exception):
-    """
-    This error is raised by default when a L{Deferred} times out.
-    """
-
-
 def _cancelled_to_timed_out_error(value, timeout):
     if isinstance(value, failure.Failure):
         value.trap(CancelledError)
-        raise DeferredTimeoutError(timeout, "Deferred")
+        raise defer.TimeoutError(timeout, "Deferred")
     return value
 
 
@@ -408,7 +402,7 @@ def timeout_deferred(deferred, timeout, reactor, on_timeout_cancel=None):
             the timeout.
 
             The default callable (if none is provided) will translate a
-            CancelledError Failure into a DeferredTimeoutError.
+            CancelledError Failure into a defer.TimeoutError.
 
     Returns:
         Deferred
@@ -427,7 +421,7 @@ def time_it_out():
             logger.exception("Canceller failed during timeout")
 
         if not new_d.called:
-            new_d.errback(DeferredTimeoutError(timeout, "Deferred"))
+            new_d.errback(defer.TimeoutError(timeout, "Deferred"))
 
     delayed_call = reactor.callLater(timeout, time_it_out)
 

From 80d2d50f4764d6662ceef86f834f845b539263e8 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 19 Sep 2018 11:19:47 +0100
Subject: [PATCH 680/830] Fixup

---
 changelog.d/{3910 => 3910.bugfix} | 0
 synapse/notifier.py               | 5 +----
 2 files changed, 1 insertion(+), 4 deletions(-)
 rename changelog.d/{3910 => 3910.bugfix} (100%)

diff --git a/changelog.d/3910 b/changelog.d/3910.bugfix
similarity index 100%
rename from changelog.d/3910
rename to changelog.d/3910.bugfix
diff --git a/synapse/notifier.py b/synapse/notifier.py
index 2e7525b7d288..f1d92c1395d2 100644
--- a/synapse/notifier.py
+++ b/synapse/notifier.py
@@ -25,10 +25,7 @@
 from synapse.handlers.presence import format_user_presence_state
 from synapse.metrics import LaterGauge
 from synapse.types import StreamToken
-from synapse.util.async_helpers import (
-    ObservableDeferred,
-    timeout_deferred,
-)
+from synapse.util.async_helpers import ObservableDeferred, timeout_deferred
 from synapse.util.logcontext import PreserveLoggingContext, run_in_background
 from synapse.util.logutils import log_function
 from synapse.util.metrics import Measure

From b9158ac2bf46e15107162be136964c17bd8a948c Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 19 Sep 2018 14:16:34 +0100
Subject: [PATCH 681/830] Remove get_destination_retry_timings cache

Currently we rely on the master to invalidate this cache promptly.
However, after having moved most federation endpoints off of master this
no longer happens, causing outbound fedeariont to get blackholed.

Fixes #3798
---
 synapse/storage/transactions.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py
index 0c42bd332227..7d51f9612ab2 100644
--- a/synapse/storage/transactions.py
+++ b/synapse/storage/transactions.py
@@ -156,7 +156,6 @@ def delivered_txn(self, transaction_id, destination, code, response_dict):
         """
         pass
 
-    @cached(max_entries=10000)
     def get_destination_retry_timings(self, destination):
         """Gets the current retry timings (if any) for a given destination.
 
@@ -212,10 +211,6 @@ def _set_destination_retry_timings(self, txn, destination,
                                        retry_last_ts, retry_interval):
         self.database_engine.lock_table(txn, "destinations")
 
-        self._invalidate_cache_and_stream(
-            txn, self.get_destination_retry_timings, (destination,)
-        )
-
         # We need to be careful here as the data may have changed from under us
         # due to a worker setting the timings.
 

From 83ee5592a5fae9335a0c4b62e3f2dea870296ac5 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 19 Sep 2018 14:20:16 +0100
Subject: [PATCH 682/830] Newsfile

---
 changelog.d/3914.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3914.bugfix

diff --git a/changelog.d/3914.bugfix b/changelog.d/3914.bugfix
new file mode 100644
index 000000000000..60fb8ec1ae6a
--- /dev/null
+++ b/changelog.d/3914.bugfix
@@ -0,0 +1 @@
+Fix bug where outboud federation would stop talking to some servers when using workers

From 392a54128c03128255ee538e17ed9bd87b95d4ca Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 19 Sep 2018 14:37:49 +0100
Subject: [PATCH 683/830] pep8

---
 synapse/storage/transactions.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py
index 7d51f9612ab2..449bdc1e49d9 100644
--- a/synapse/storage/transactions.py
+++ b/synapse/storage/transactions.py
@@ -23,7 +23,6 @@
 from twisted.internet import defer
 
 from synapse.metrics.background_process_metrics import run_as_background_process
-from synapse.util.caches.descriptors import cached
 
 from ._base import SQLBaseStore, db_to_json
 

From bbab6ebfd9257ba9ad5e576c4b9b8bdea1c166cf Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 19 Sep 2018 14:45:14 +0100
Subject: [PATCH 684/830] Fix up changelog and remove spurious comment

---
 changelog.d/3914.bugfix         | 2 +-
 synapse/storage/transactions.py | 2 --
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/changelog.d/3914.bugfix b/changelog.d/3914.bugfix
index 60fb8ec1ae6a..27e6bad59095 100644
--- a/changelog.d/3914.bugfix
+++ b/changelog.d/3914.bugfix
@@ -1 +1 @@
-Fix bug where outboud federation would stop talking to some servers when using workers
+Fix bug where outbound federation would stop talking to some servers when using workers
diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py
index 449bdc1e49d9..baf0379a6834 100644
--- a/synapse/storage/transactions.py
+++ b/synapse/storage/transactions.py
@@ -196,8 +196,6 @@ def set_destination_retry_timings(self, destination,
             retry_interval (int) - how long until next retry in ms
         """
 
-        # XXX: we could chose to not bother persisting this if our cache thinks
-        # this is a NOOP
         return self.runInteraction(
             "set_destination_retry_timings",
             self._set_destination_retry_timings,

From 1a24d4effaeadaa14e30d56c1a710fff296c2391 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 5 Sep 2018 16:22:26 +0100
Subject: [PATCH 685/830] Newsfile

---
 changelog.d/3794.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3794.misc

diff --git a/changelog.d/3794.misc b/changelog.d/3794.misc
new file mode 100644
index 000000000000..6b98c9609b18
--- /dev/null
+++ b/changelog.d/3794.misc
@@ -0,0 +1 @@
+Speed up calculation of typing updates for replication

From 642199570c57fd37b856c3808e08fe2923001b55 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Wed, 19 Sep 2018 17:28:18 +0100
Subject: [PATCH 686/830] Improve the logging when handling a federation
 transaction (#3904)

Let's try to rationalise the logging that happens when we are processing an
incoming transaction, to make it easier to figure out what is going wrong when
they take ages. In particular:

- make everything start with a [room_id event_id] prefix
- make sure we log a warning when catching exceptions rather than just turning
  them into other, more cryptic, exceptions.
---
 changelog.d/3904.misc          |   1 +
 synapse/handlers/federation.py | 164 ++++++++++++++++++++++-----------
 synapse/util/retryutils.py     |   2 +-
 3 files changed, 111 insertions(+), 56 deletions(-)
 create mode 100644 changelog.d/3904.misc

diff --git a/changelog.d/3904.misc b/changelog.d/3904.misc
new file mode 100644
index 000000000000..1e3c8e1706ca
--- /dev/null
+++ b/changelog.d/3904.misc
@@ -0,0 +1 @@
+Improve the logging when handling a federation transaction
\ No newline at end of file
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index f10b46414bf6..8d6bd7976d74 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -69,6 +69,27 @@
 logger = logging.getLogger(__name__)
 
 
+def shortstr(iterable, maxitems=5):
+    """If iterable has maxitems or fewer, return the stringification of a list
+    containing those items.
+
+    Otherwise, return the stringification of a a list with the first maxitems items,
+    followed by "...".
+
+    Args:
+        iterable (Iterable): iterable to truncate
+        maxitems (int): number of items to return before truncating
+
+    Returns:
+        unicode
+    """
+
+    items = list(itertools.islice(iterable, maxitems + 1))
+    if len(items) <= maxitems:
+        return str(items)
+    return u"[" + u", ".join(repr(r) for r in items[:maxitems]) + u", ...]"
+
+
 class FederationHandler(BaseHandler):
     """Handles events that originated from federation.
         Responsible for:
@@ -114,7 +135,6 @@ def __init__(self, hs):
         self._room_pdu_linearizer = Linearizer("fed_room_pdu")
 
     @defer.inlineCallbacks
-    @log_function
     def on_receive_pdu(
             self, origin, pdu, get_missing=True, sent_to_us_directly=False,
     ):
@@ -130,9 +150,17 @@ def on_receive_pdu(
         Returns (Deferred): completes with None
         """
 
+        room_id = pdu.room_id
+        event_id = pdu.event_id
+
+        logger.info(
+            "[%s %s] handling received PDU: %s",
+            room_id, event_id, pdu,
+        )
+
         # We reprocess pdus when we have seen them only as outliers
         existing = yield self.store.get_event(
-            pdu.event_id,
+            event_id,
             allow_none=True,
             allow_rejected=True,
         )
@@ -147,7 +175,7 @@ def on_receive_pdu(
             )
         )
         if already_seen:
-            logger.debug("Already seen pdu %s", pdu.event_id)
+            logger.debug("[%s %s]: Already seen pdu", room_id, event_id)
             return
 
         # do some initial sanity-checking of the event. In particular, make
@@ -156,6 +184,7 @@ def on_receive_pdu(
         try:
             self._sanity_check_event(pdu)
         except SynapseError as err:
+            logger.warn("[%s %s] Received event failed sanity checks", room_id, event_id)
             raise FederationError(
                 "ERROR",
                 err.code,
@@ -165,10 +194,12 @@ def on_receive_pdu(
 
         # If we are currently in the process of joining this room, then we
         # queue up events for later processing.
-        if pdu.room_id in self.room_queues:
-            logger.info("Ignoring PDU %s for room %s from %s for now; join "
-                        "in progress", pdu.event_id, pdu.room_id, origin)
-            self.room_queues[pdu.room_id].append((pdu, origin))
+        if room_id in self.room_queues:
+            logger.info(
+                "[%s %s] Queuing PDU from %s for now: join in progress",
+                room_id, event_id, origin,
+            )
+            self.room_queues[room_id].append((pdu, origin))
             return
 
         # If we're no longer in the room just ditch the event entirely. This
@@ -179,7 +210,7 @@ def on_receive_pdu(
         # we should check if we *are* in fact in the room. If we are then we
         # can magically rejoin the room.
         is_in_room = yield self.auth.check_host_in_room(
-            pdu.room_id,
+            room_id,
             self.server_name
         )
         if not is_in_room:
@@ -188,8 +219,8 @@ def on_receive_pdu(
             )
             if was_in_room:
                 logger.info(
-                    "Ignoring PDU %s for room %s from %s as we've left the room!",
-                    pdu.event_id, pdu.room_id, origin,
+                    "[%s %s] Ignoring PDU from %s as we've left the room",
+                    room_id, event_id, origin,
                 )
                 defer.returnValue(None)
 
@@ -204,8 +235,8 @@ def on_receive_pdu(
             )
 
             logger.debug(
-                "_handle_new_pdu min_depth for %s: %d",
-                pdu.room_id, min_depth
+                "[%s %s] min_depth: %d",
+                room_id, event_id, min_depth,
             )
 
             prevs = {e_id for e_id, _ in pdu.prev_events}
@@ -218,17 +249,18 @@ def on_receive_pdu(
                 # send to the clients.
                 pdu.internal_metadata.outlier = True
             elif min_depth and pdu.depth > min_depth:
-                if get_missing and prevs - seen:
+                missing_prevs = prevs - seen
+                if get_missing and missing_prevs:
                     # If we're missing stuff, ensure we only fetch stuff one
                     # at a time.
                     logger.info(
-                        "Acquiring lock for room %r to fetch %d missing events: %r...",
-                        pdu.room_id, len(prevs - seen), list(prevs - seen)[:5],
+                        "[%s %s] Acquiring room lock to fetch %d missing prev_events: %s",
+                        room_id, event_id, len(missing_prevs), shortstr(missing_prevs),
                     )
                     with (yield self._room_pdu_linearizer.queue(pdu.room_id)):
                         logger.info(
-                            "Acquired lock for room %r to fetch %d missing events",
-                            pdu.room_id, len(prevs - seen),
+                            "[%s %s] Acquired room lock to fetch %d missing prev_events",
+                            room_id, event_id, len(missing_prevs),
                         )
 
                         yield self._get_missing_events_for_pdu(
@@ -241,19 +273,23 @@ def on_receive_pdu(
 
                         if not prevs - seen:
                             logger.info(
-                                "Found all missing prev events for %s", pdu.event_id
+                                "[%s %s] Found all missing prev_events",
+                                room_id, event_id,
                             )
-                elif prevs - seen:
+                elif missing_prevs:
                     logger.info(
-                        "Not fetching %d missing events for room %r,event %s: %r...",
-                        len(prevs - seen), pdu.room_id, pdu.event_id,
-                        list(prevs - seen)[:5],
+                        "[%s %s] Not recursively fetching %d missing prev_events: %s",
+                        room_id, event_id, len(missing_prevs), shortstr(missing_prevs),
                     )
 
             if sent_to_us_directly and prevs - seen:
                 # If they have sent it to us directly, and the server
                 # isn't telling us about the auth events that it's
                 # made a message referencing, we explode
+                logger.warn(
+                    "[%s %s] Failed to fetch %d prev events: rejecting",
+                    room_id, event_id, len(prevs - seen),
+                )
                 raise FederationError(
                     "ERROR",
                     403,
@@ -270,15 +306,19 @@ def on_receive_pdu(
                 auth_chains = set()
                 try:
                     # Get the state of the events we know about
-                    ours = yield self.store.get_state_groups(pdu.room_id, list(seen))
+                    ours = yield self.store.get_state_groups(room_id, list(seen))
                     state_groups.append(ours)
 
                     # Ask the remote server for the states we don't
                     # know about
                     for p in prevs - seen:
+                        logger.info(
+                            "[%s %s] Requesting state at missing prev_event %s",
+                            room_id, event_id, p,
+                        )
                         state, got_auth_chain = (
                             yield self.federation_client.get_state_for_room(
-                                origin, pdu.room_id, p
+                                origin, room_id, p,
                             )
                         )
                         auth_chains.update(got_auth_chain)
@@ -291,19 +331,24 @@ def fetch(ev_ids):
                             ev_ids, get_prev_content=False, check_redacted=False
                         )
 
-                    room_version = yield self.store.get_room_version(pdu.room_id)
+                    room_version = yield self.store.get_room_version(room_id)
                     state_map = yield resolve_events_with_factory(
-                        room_version, state_groups, {pdu.event_id: pdu}, fetch
+                        room_version, state_groups, {event_id: pdu}, fetch
                     )
 
                     state = (yield self.store.get_events(state_map.values())).values()
                     auth_chain = list(auth_chains)
                 except Exception:
+                    logger.warn(
+                        "[%s %s] Error attempting to resolve state at missing "
+                        "prev_events",
+                        room_id, event_id, exc_info=True,
+                    )
                     raise FederationError(
                         "ERROR",
                         403,
                         "We can't get valid state history.",
-                        affected=pdu.event_id,
+                        affected=event_id,
                     )
 
         yield self._process_received_pdu(
@@ -322,15 +367,16 @@ def _get_missing_events_for_pdu(self, origin, pdu, prevs, min_depth):
             prevs (set(str)): List of event ids which we are missing
             min_depth (int): Minimum depth of events to return.
         """
-        # We recalculate seen, since it may have changed.
+
+        room_id = pdu.room_id
+        event_id = pdu.event_id
+
         seen = yield self.store.have_seen_events(prevs)
 
         if not prevs - seen:
             return
 
-        latest = yield self.store.get_latest_event_ids_in_room(
-            pdu.room_id
-        )
+        latest = yield self.store.get_latest_event_ids_in_room(room_id)
 
         # We add the prev events that we have seen to the latest
         # list to ensure the remote server doesn't give them to us
@@ -338,8 +384,8 @@ def _get_missing_events_for_pdu(self, origin, pdu, prevs, min_depth):
         latest |= seen
 
         logger.info(
-            "Missing %d events for room %r pdu %s: %r...",
-            len(prevs - seen), pdu.room_id, pdu.event_id, list(prevs - seen)[:5]
+            "[%s %s]: Requesting %d prev_events: %s",
+            room_id, event_id, len(prevs - seen), shortstr(prevs - seen)
         )
 
         # XXX: we set timeout to 10s to help workaround
@@ -392,7 +438,7 @@ def _get_missing_events_for_pdu(self, origin, pdu, prevs, min_depth):
 
         missing_events = yield self.federation_client.get_missing_events(
             origin,
-            pdu.room_id,
+            room_id,
             earliest_events_ids=list(latest),
             latest_events=[pdu],
             limit=10,
@@ -401,37 +447,46 @@ def _get_missing_events_for_pdu(self, origin, pdu, prevs, min_depth):
         )
 
         logger.info(
-            "Got %d events: %r...",
-            len(missing_events), [e.event_id for e in missing_events[:5]]
+            "[%s %s]: Got %d prev_events: %s",
+            room_id, event_id, len(missing_events), shortstr(missing_events),
         )
 
         # We want to sort these by depth so we process them and
         # tell clients about them in order.
         missing_events.sort(key=lambda x: x.depth)
 
-        for e in missing_events:
-            logger.info("Handling found event %s", e.event_id)
+        for ev in missing_events:
+            logger.info(
+                "[%s %s] Handling received prev_event %s",
+                room_id, event_id, ev.event_id,
+            )
             try:
                 yield self.on_receive_pdu(
                     origin,
-                    e,
+                    ev,
                     get_missing=False
                 )
             except FederationError as e:
                 if e.code == 403:
-                    logger.warn("Event %s failed history check.")
+                    logger.warn(
+                        "[%s %s] Received prev_event %s failed history check.",
+                        room_id, event_id, ev.event_id,
+                    )
                 else:
                     raise
 
-    @log_function
     @defer.inlineCallbacks
-    def _process_received_pdu(self, origin, pdu, state, auth_chain):
+    def _process_received_pdu(self, origin, event, state, auth_chain):
         """ Called when we have a new pdu. We need to do auth checks and put it
         through the StateHandler.
         """
-        event = pdu
+        room_id = event.room_id
+        event_id = event.event_id
 
-        logger.debug("Processing event: %s", event)
+        logger.debug(
+            "[%s %s] Processing event: %s",
+            room_id, event_id, event,
+        )
 
         # FIXME (erikj): Awful hack to make the case where we are not currently
         # in the room work
@@ -440,15 +495,16 @@ def _process_received_pdu(self, origin, pdu, state, auth_chain):
         # event.
         if state and auth_chain and not event.internal_metadata.is_outlier():
             is_in_room = yield self.auth.check_host_in_room(
-                event.room_id,
+                room_id,
                 self.server_name
             )
         else:
             is_in_room = True
+
         if not is_in_room:
             logger.info(
-                "Got event for room we're not in: %r %r",
-                event.room_id, event.event_id
+                "[%s %s] Got event for room we're not in",
+                room_id, event_id,
             )
 
             try:
@@ -460,7 +516,7 @@ def _process_received_pdu(self, origin, pdu, state, auth_chain):
                     "ERROR",
                     e.code,
                     e.msg,
-                    affected=event.event_id,
+                    affected=event_id,
                 )
 
         else:
@@ -509,12 +565,12 @@ def _process_received_pdu(self, origin, pdu, state, auth_chain):
                     affected=event.event_id,
                 )
 
-        room = yield self.store.get_room(event.room_id)
+        room = yield self.store.get_room(room_id)
 
         if not room:
             try:
                 yield self.store.store_room(
-                    room_id=event.room_id,
+                    room_id=room_id,
                     room_creator_user_id="",
                     is_public=False,
                 )
@@ -542,7 +598,7 @@ def _process_received_pdu(self, origin, pdu, state, auth_chain):
 
                 if newly_joined:
                     user = UserID.from_string(event.state_key)
-                    yield self.user_joined_room(user, event.room_id)
+                    yield self.user_joined_room(user, room_id)
 
     @log_function
     @defer.inlineCallbacks
@@ -1459,12 +1515,10 @@ def get_persisted_pdu(self, origin, event_id):
         else:
             defer.returnValue(None)
 
-    @log_function
     def get_min_depth_for_context(self, context):
         return self.store.get_min_depth(context)
 
     @defer.inlineCallbacks
-    @log_function
     def _handle_new_event(self, origin, event, state=None, auth_events=None,
                           backfilled=False):
         context = yield self._prep_event(
@@ -1664,8 +1718,8 @@ def _prep_event(self, origin, event, state=None, auth_events=None):
             )
         except AuthError as e:
             logger.warn(
-                "Rejecting %s because %s",
-                event.event_id, e.msg
+                "[%s %s] Rejecting: %s",
+                event.room_id, event.event_id, e.msg
             )
 
             context.rejected = RejectedReason.AUTH_ERROR
diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py
index 8a3a06fd743e..26cce7d197d6 100644
--- a/synapse/util/retryutils.py
+++ b/synapse/util/retryutils.py
@@ -188,7 +188,7 @@ def __exit__(self, exc_type, exc_val, exc_tb):
             else:
                 self.retry_interval = self.min_retry_interval
 
-            logger.debug(
+            logger.info(
                 "Connection to %s was unsuccessful (%s(%s)); backoff now %i",
                 self.destination, exc_type, exc_val, self.retry_interval
             )

From aeca5a5ed5816a7797a0807142a41e14baac80f6 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Thu, 20 Sep 2018 16:28:18 +1000
Subject: [PATCH 687/830] Add a regression test for logging on failed
 connections (#3912)

---
 changelog.d/3912.misc |  1 +
 synapse/http/site.py  |  4 +--
 tests/test_server.py  | 74 +++++++++++++++++++++++++++++++++++++++++--
 tests/unittest.py     |  3 +-
 4 files changed, 77 insertions(+), 5 deletions(-)
 create mode 100644 changelog.d/3912.misc

diff --git a/changelog.d/3912.misc b/changelog.d/3912.misc
new file mode 100644
index 000000000000..87d73697ea20
--- /dev/null
+++ b/changelog.d/3912.misc
@@ -0,0 +1 @@
+Add a regression test for logging failed HTTP requests on Python 3.
\ No newline at end of file
diff --git a/synapse/http/site.py b/synapse/http/site.py
index 9579e8cd0d19..d41d6729340a 100644
--- a/synapse/http/site.py
+++ b/synapse/http/site.py
@@ -75,9 +75,9 @@ def __repr__(self):
         return '<%s at 0x%x method=%r uri=%r clientproto=%r site=%r>' % (
             self.__class__.__name__,
             id(self),
-            self.method,
+            self.method.decode('ascii', errors='replace'),
             self.get_redacted_uri(),
-            self.clientproto,
+            self.clientproto.decode('ascii', errors='replace'),
             self.site.site_tag,
         )
 
diff --git a/tests/test_server.py b/tests/test_server.py
index ef74544e93cc..4045fdadc3fa 100644
--- a/tests/test_server.py
+++ b/tests/test_server.py
@@ -1,14 +1,35 @@
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
 import re
 
+from six import StringIO
+
 from twisted.internet.defer import Deferred
-from twisted.test.proto_helpers import MemoryReactorClock
+from twisted.python.failure import Failure
+from twisted.test.proto_helpers import AccumulatingProtocol, MemoryReactorClock
+from twisted.web.resource import Resource
+from twisted.web.server import NOT_DONE_YET
 
 from synapse.api.errors import Codes, SynapseError
 from synapse.http.server import JsonResource
+from synapse.http.site import SynapseSite, logger
 from synapse.util import Clock
 
 from tests import unittest
-from tests.server import make_request, render, setup_test_homeserver
+from tests.server import FakeTransport, make_request, render, setup_test_homeserver
 
 
 class JsonResourceTests(unittest.TestCase):
@@ -121,3 +142,52 @@ def _callback(request, **kwargs):
         self.assertEqual(channel.result["code"], b'400')
         self.assertEqual(channel.json_body["error"], "Unrecognized request")
         self.assertEqual(channel.json_body["errcode"], "M_UNRECOGNIZED")
+
+
+class SiteTestCase(unittest.HomeserverTestCase):
+    def test_lose_connection(self):
+        """
+        We log the URI correctly redacted when we lose the connection.
+        """
+
+        class HangingResource(Resource):
+            """
+            A Resource that strategically hangs, as if it were processing an
+            answer.
+            """
+
+            def render(self, request):
+                return NOT_DONE_YET
+
+        # Set up a logging handler that we can inspect afterwards
+        output = StringIO()
+        handler = logging.StreamHandler(output)
+        logger.addHandler(handler)
+        old_level = logger.level
+        logger.setLevel(10)
+        self.addCleanup(logger.setLevel, old_level)
+        self.addCleanup(logger.removeHandler, handler)
+
+        # Make a resource and a Site, the resource will hang and allow us to
+        # time out the request while it's 'processing'
+        base_resource = Resource()
+        base_resource.putChild(b'', HangingResource())
+        site = SynapseSite("test", "site_tag", {}, base_resource, "1.0")
+
+        server = site.buildProtocol(None)
+        client = AccumulatingProtocol()
+        client.makeConnection(FakeTransport(server, self.reactor))
+        server.makeConnection(FakeTransport(client, self.reactor))
+
+        # Send a request with an access token that will get redacted
+        server.dataReceived(b"GET /?access_token=bar HTTP/1.0\r\n\r\n")
+        self.pump()
+
+        # Lose the connection
+        e = Failure(Exception("Failed123"))
+        server.connectionLost(e)
+        handler.flush()
+
+        # Our access token is redacted and the failure reason is logged.
+        self.assertIn("/?access_token=<redacted>", output.getvalue())
+        self.assertIn("Failed123", output.getvalue())
diff --git a/tests/unittest.py b/tests/unittest.py
index a3d39920db0a..56f3dca394d4 100644
--- a/tests/unittest.py
+++ b/tests/unittest.py
@@ -219,7 +219,8 @@ def make_homeserver(self, reactor, clock):
 
         Function to be overridden in subclasses.
         """
-        raise NotImplementedError()
+        hs = self.setup_test_homeserver()
+        return hs
 
     def prepare(self, reactor, clock, homeserver):
         """

From 741571cf22b342f2cfaf5c47f324397af7072bb3 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Thu, 20 Sep 2018 18:12:45 +1000
Subject: [PATCH 688/830] Add a way to run tests in PostgreSQL in Docker
 (#3699)

---
 .gitignore                |  1 +
 CONTRIBUTING.rst          | 38 ++++++++++++++++++++++++++++++--------
 MANIFEST.in               |  1 +
 changelog.d/3699.misc     |  2 ++
 docker/Dockerfile-pgtests | 12 ++++++++++++
 docker/run_pg_tests.sh    | 20 ++++++++++++++++++++
 test_postgresql.sh        | 12 ++++++++++++
 tox.ini                   |  2 +-
 8 files changed, 79 insertions(+), 9 deletions(-)
 create mode 100644 changelog.d/3699.misc
 create mode 100644 docker/Dockerfile-pgtests
 create mode 100755 docker/run_pg_tests.sh
 create mode 100755 test_postgresql.sh

diff --git a/.gitignore b/.gitignore
index a72572523571..3b2252ad8a5a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
 *.pyc
 .*.swp
 *~
+*.lock
 
 .DS_Store
 _trial_temp/
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
index f9de78a4602f..6ef7d48dc78c 100644
--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -30,12 +30,28 @@ use github's pull request workflow to review the contribution, and either ask
 you to make any refinements needed or merge it and make them ourselves. The
 changes will then land on master when we next do a release.
 
-We use `Jenkins <http://matrix.org/jenkins>`_ and
-`Travis <https://travis-ci.org/matrix-org/synapse>`_ for continuous
-integration. All pull requests to synapse get automatically tested by Travis;
-the Jenkins builds require an adminstrator to start them. If your change
-breaks the build, this will be shown in github, so please keep an eye on the
-pull request for feedback.
+We use `CircleCI <https://circleci.com/gh/matrix-org>`_ and `Travis CI 
+<https://travis-ci.org/matrix-org/synapse>`_ for continuous integration. All
+pull requests to synapse get automatically tested by Travis and CircleCI.
+If your change breaks the build, this will be shown in GitHub, so please
+keep an eye on the pull request for feedback.
+
+To run unit tests in a local development environment, you can use:
+
+- ``tox -e py27`` (requires tox to be installed by ``pip install tox``) for
+  SQLite-backed Synapse on Python 2.7.
+- ``tox -e py35`` for SQLite-backed Synapse on Python 3.5.
+- ``tox -e py36`` for SQLite-backed Synapse on Python 3.6.
+- ``tox -e py27-postgres`` for PostgreSQL-backed Synapse on Python 2.7
+  (requires a running local PostgreSQL with access to create databases).
+- ``./test_postgresql.sh`` for PostgreSQL-backed Synapse on Python 2.7
+  (requires Docker). Entirely self-contained, recommended if you don't want to
+  set up PostgreSQL yourself.
+
+Docker images are available for running the integration tests (SyTest) locally,
+see the `documentation in the SyTest repo
+<https://github.com/matrix-org/sytest/blob/develop/docker/README.md>`_ for more
+information.
 
 Code style
 ~~~~~~~~~~
@@ -77,7 +93,8 @@ AUTHORS.rst file for the project in question. Please feel free to include a
 change to AUTHORS.rst in your pull request to list yourself and a short
 description of the area(s) you've worked on. Also, we sometimes have swag to
 give away to contributors - if you feel that Matrix-branded apparel is missing
-from your life, please mail us your shipping address to matrix at matrix.org and we'll try to fix it :)
+from your life, please mail us your shipping address to matrix at matrix.org and
+we'll try to fix it :)
 
 Sign off
 ~~~~~~~~
@@ -144,4 +161,9 @@ flag to ``git commit``, which uses the name and email set in your
 Conclusion
 ~~~~~~~~~~
 
-That's it!  Matrix is a very open and collaborative project as you might expect given our obsession with open communication.  If we're going to successfully matrix together all the fragmented communication technologies out there we are reliant on contributions and collaboration from the community to do so.  So please get involved - and we hope you have as much fun hacking on Matrix as we do!
+That's it!  Matrix is a very open and collaborative project as you might expect
+given our obsession with open communication.  If we're going to successfully
+matrix together all the fragmented communication technologies out there we are
+reliant on contributions and collaboration from the community to do so.  So
+please get involved - and we hope you have as much fun hacking on Matrix as we
+do!
diff --git a/MANIFEST.in b/MANIFEST.in
index e0826ba544c8..47ae5a77b9d2 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -28,6 +28,7 @@ exclude jenkins*.sh
 exclude jenkins*
 exclude Dockerfile
 exclude .dockerignore
+exclude test_postgresql.sh
 recursive-exclude jenkins *.sh
 
 include pyproject.toml
diff --git a/changelog.d/3699.misc b/changelog.d/3699.misc
new file mode 100644
index 000000000000..437efbd98f13
--- /dev/null
+++ b/changelog.d/3699.misc
@@ -0,0 +1,2 @@
+Unit tests can now be run under PostgreSQL in Docker using 
+``test_postgresql.sh``.
diff --git a/docker/Dockerfile-pgtests b/docker/Dockerfile-pgtests
new file mode 100644
index 000000000000..7da8eeb9eb0e
--- /dev/null
+++ b/docker/Dockerfile-pgtests
@@ -0,0 +1,12 @@
+# Use the Sytest image that comes with a lot of the build dependencies
+# pre-installed
+FROM matrixdotorg/sytest:latest
+
+# The Sytest image doesn't come with python, so install that
+RUN apt-get -qq install -y python python-dev python-pip
+
+# We need tox to run the tests in run_pg_tests.sh
+RUN pip install tox
+
+ADD run_pg_tests.sh /pg_tests.sh
+ENTRYPOINT /pg_tests.sh
diff --git a/docker/run_pg_tests.sh b/docker/run_pg_tests.sh
new file mode 100755
index 000000000000..e77424c41a37
--- /dev/null
+++ b/docker/run_pg_tests.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# This script runs the PostgreSQL tests inside a Docker container. It expects
+# the relevant source files to be mounted into /src (done automatically by the
+# caller script). It will set up the database, run it, and then use the tox
+# configuration to run the tests.
+
+set -e
+
+# Set PGUSER so Synapse's tests know what user to connect to the database with
+export PGUSER=postgres
+
+# Initialise & start the database
+su -c '/usr/lib/postgresql/9.6/bin/initdb -D /var/lib/postgresql/data -E "UTF-8" --lc-collate="en_US.UTF-8" --lc-ctype="en_US.UTF-8" --username=postgres' postgres
+su -c '/usr/lib/postgresql/9.6/bin/pg_ctl -w -D /var/lib/postgresql/data start' postgres
+
+# Run the tests
+cd /src
+export TRIAL_FLAGS="-j 4"
+tox --workdir=/tmp -e py27-postgres
diff --git a/test_postgresql.sh b/test_postgresql.sh
new file mode 100755
index 000000000000..1ffcaabd3133
--- /dev/null
+++ b/test_postgresql.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# This script builds the Docker image to run the PostgreSQL tests, and then runs
+# the tests.
+
+set -e
+
+# Build, and tag
+docker build docker/ -f docker/Dockerfile-pgtests -t synapsepgtests
+
+# Run, mounting the current directory into /src
+docker run --rm -it -v $(pwd)\:/src synapsepgtests
diff --git a/tox.ini b/tox.ini
index 80ac9324df76..88875c331862 100644
--- a/tox.ini
+++ b/tox.ini
@@ -4,7 +4,7 @@ envlist = packaging, py27, py36, pep8, check_isort
 [base]
 deps =
     coverage
-    Twisted>=15.1
+    Twisted>=17.1
     mock
     python-subunit
     junitxml

From 1f3f5fcf52efdf5215dc6c7cd3805b1357bf8236 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Thu, 20 Sep 2018 20:14:34 +1000
Subject: [PATCH 689/830] Fix client IPs being broken on Python 3 (#3908)

---
 .travis.yml                      |   5 +
 changelog.d/3908.bugfix          |   1 +
 synapse/http/site.py             |   2 +-
 synapse/storage/client_ips.py    |  34 +++---
 tests/server.py                  |   8 +-
 tests/storage/test_client_ips.py | 202 +++++++++++++++++++++++++------
 tests/unittest.py                |   7 +-
 tests/utils.py                   |  27 ++++-
 tox.ini                          |  10 ++
 9 files changed, 238 insertions(+), 58 deletions(-)
 create mode 100644 changelog.d/3908.bugfix

diff --git a/.travis.yml b/.travis.yml
index b3ee66da8ff1..b6faca4b9258 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -31,6 +31,11 @@ matrix:
   - python: 3.6
     env: TOX_ENV=py36
 
+  - python: 3.6
+    env: TOX_ENV=py36-postgres TRIAL_FLAGS="-j 4"
+    services:
+      - postgresql
+
   - python: 3.6
     env: TOX_ENV=check_isort
 
diff --git a/changelog.d/3908.bugfix b/changelog.d/3908.bugfix
new file mode 100644
index 000000000000..518aee6c4d2e
--- /dev/null
+++ b/changelog.d/3908.bugfix
@@ -0,0 +1 @@
+Fix adding client IPs to the database failing on Python 3.
\ No newline at end of file
diff --git a/synapse/http/site.py b/synapse/http/site.py
index d41d6729340a..50be2de3bbc7 100644
--- a/synapse/http/site.py
+++ b/synapse/http/site.py
@@ -308,7 +308,7 @@ def getClientIP(self):
             C{b"-"}.
         """
         return self.requestHeaders.getRawHeaders(
-            b"x-forwarded-for", [b"-"])[0].split(b",")[0].strip()
+            b"x-forwarded-for", [b"-"])[0].split(b",")[0].strip().decode('ascii')
 
 
 class SynapseRequestFactory(object):
diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py
index 8fc678fa67be..9ad17b7c2597 100644
--- a/synapse/storage/client_ips.py
+++ b/synapse/storage/client_ips.py
@@ -119,21 +119,25 @@ def _update_client_ips_batch_txn(self, txn, to_update):
         for entry in iteritems(to_update):
             (user_id, access_token, ip), (user_agent, device_id, last_seen) = entry
 
-            self._simple_upsert_txn(
-                txn,
-                table="user_ips",
-                keyvalues={
-                    "user_id": user_id,
-                    "access_token": access_token,
-                    "ip": ip,
-                    "user_agent": user_agent,
-                    "device_id": device_id,
-                },
-                values={
-                    "last_seen": last_seen,
-                },
-                lock=False,
-            )
+            try:
+                self._simple_upsert_txn(
+                    txn,
+                    table="user_ips",
+                    keyvalues={
+                        "user_id": user_id,
+                        "access_token": access_token,
+                        "ip": ip,
+                        "user_agent": user_agent,
+                        "device_id": device_id,
+                    },
+                    values={
+                        "last_seen": last_seen,
+                    },
+                    lock=False,
+                )
+            except Exception as e:
+                # Failed to upsert, log and continue
+                logger.error("Failed to insert client IP %r: %r", entry, e)
 
     @defer.inlineCallbacks
     def get_last_client_ip_by_device(self, user_id, device_id):
diff --git a/tests/server.py b/tests/server.py
index ccea3baa554e..7bee58dff162 100644
--- a/tests/server.py
+++ b/tests/server.py
@@ -98,7 +98,7 @@ def info(self, *args, **kwargs):
         return FakeLogger()
 
 
-def make_request(method, path, content=b"", access_token=None):
+def make_request(method, path, content=b"", access_token=None, request=SynapseRequest):
     """
     Make a web request using the given method and path, feed it the
     content, and return the Request and the Channel underneath.
@@ -120,14 +120,16 @@ def make_request(method, path, content=b"", access_token=None):
     site = FakeSite()
     channel = FakeChannel()
 
-    req = SynapseRequest(site, channel)
+    req = request(site, channel)
     req.process = lambda: b""
     req.content = BytesIO(content)
 
     if access_token:
         req.requestHeaders.addRawHeader(b"Authorization", b"Bearer " + access_token)
 
-    req.requestHeaders.addRawHeader(b"X-Forwarded-For", b"127.0.0.1")
+    if content:
+        req.requestHeaders.addRawHeader(b"Content-Type", b"application/json")
+
     req.requestReceived(method, path, b"1.1")
 
     return req, channel
diff --git a/tests/storage/test_client_ips.py b/tests/storage/test_client_ips.py
index c9b02a062be5..2ffbb9f14f0f 100644
--- a/tests/storage/test_client_ips.py
+++ b/tests/storage/test_client_ips.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
+# Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,35 +13,45 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+import hashlib
+import hmac
+import json
+
 from mock import Mock
 
 from twisted.internet import defer
 
-import tests.unittest
-import tests.utils
+from synapse.http.site import XForwardedForRequest
+from synapse.rest.client.v1 import admin, login
+
+from tests import unittest
 
 
-class ClientIpStoreTestCase(tests.unittest.TestCase):
-    def __init__(self, *args, **kwargs):
-        super(ClientIpStoreTestCase, self).__init__(*args, **kwargs)
-        self.store = None  # type: synapse.storage.DataStore
-        self.clock = None  # type: tests.utils.MockClock
+class ClientIpStoreTestCase(unittest.HomeserverTestCase):
+    def make_homeserver(self, reactor, clock):
+        hs = self.setup_test_homeserver()
+        return hs
 
-    @defer.inlineCallbacks
-    def setUp(self):
-        self.hs = yield tests.utils.setup_test_homeserver(self.addCleanup)
+    def prepare(self, hs, reactor, clock):
         self.store = self.hs.get_datastore()
-        self.clock = self.hs.get_clock()
 
-    @defer.inlineCallbacks
     def test_insert_new_client_ip(self):
-        self.clock.now = 12345678
+        self.reactor.advance(12345678)
+
         user_id = "@user:id"
-        yield self.store.insert_client_ip(
-            user_id, "access_token", "ip", "user_agent", "device_id"
+        self.get_success(
+            self.store.insert_client_ip(
+                user_id, "access_token", "ip", "user_agent", "device_id"
+            )
         )
 
-        result = yield self.store.get_last_client_ip_by_device(user_id, "device_id")
+        # Trigger the storage loop
+        self.reactor.advance(10)
+
+        result = self.get_success(
+            self.store.get_last_client_ip_by_device(user_id, "device_id")
+        )
 
         r = result[(user_id, "device_id")]
         self.assertDictContainsSubset(
@@ -55,18 +66,18 @@ def test_insert_new_client_ip(self):
             r,
         )
 
-    @defer.inlineCallbacks
     def test_disabled_monthly_active_user(self):
         self.hs.config.limit_usage_by_mau = False
         self.hs.config.max_mau_value = 50
         user_id = "@user:server"
-        yield self.store.insert_client_ip(
-            user_id, "access_token", "ip", "user_agent", "device_id"
+        self.get_success(
+            self.store.insert_client_ip(
+                user_id, "access_token", "ip", "user_agent", "device_id"
+            )
         )
-        active = yield self.store.user_last_seen_monthly_active(user_id)
+        active = self.get_success(self.store.user_last_seen_monthly_active(user_id))
         self.assertFalse(active)
 
-    @defer.inlineCallbacks
     def test_adding_monthly_active_user_when_full(self):
         self.hs.config.limit_usage_by_mau = True
         self.hs.config.max_mau_value = 50
@@ -76,38 +87,159 @@ def test_adding_monthly_active_user_when_full(self):
         self.store.get_monthly_active_count = Mock(
             return_value=defer.succeed(lots_of_users)
         )
-        yield self.store.insert_client_ip(
-            user_id, "access_token", "ip", "user_agent", "device_id"
+        self.get_success(
+            self.store.insert_client_ip(
+                user_id, "access_token", "ip", "user_agent", "device_id"
+            )
         )
-        active = yield self.store.user_last_seen_monthly_active(user_id)
+        active = self.get_success(self.store.user_last_seen_monthly_active(user_id))
         self.assertFalse(active)
 
-    @defer.inlineCallbacks
     def test_adding_monthly_active_user_when_space(self):
         self.hs.config.limit_usage_by_mau = True
         self.hs.config.max_mau_value = 50
         user_id = "@user:server"
-        active = yield self.store.user_last_seen_monthly_active(user_id)
+        active = self.get_success(self.store.user_last_seen_monthly_active(user_id))
         self.assertFalse(active)
 
-        yield self.store.insert_client_ip(
-            user_id, "access_token", "ip", "user_agent", "device_id"
+        # Trigger the saving loop
+        self.reactor.advance(10)
+
+        self.get_success(
+            self.store.insert_client_ip(
+                user_id, "access_token", "ip", "user_agent", "device_id"
+            )
         )
-        active = yield self.store.user_last_seen_monthly_active(user_id)
+        active = self.get_success(self.store.user_last_seen_monthly_active(user_id))
         self.assertTrue(active)
 
-    @defer.inlineCallbacks
     def test_updating_monthly_active_user_when_space(self):
         self.hs.config.limit_usage_by_mau = True
         self.hs.config.max_mau_value = 50
         user_id = "@user:server"
-        yield self.store.register(user_id=user_id, token="123", password_hash=None)
+        self.get_success(
+            self.store.register(user_id=user_id, token="123", password_hash=None)
+        )
 
-        active = yield self.store.user_last_seen_monthly_active(user_id)
+        active = self.get_success(self.store.user_last_seen_monthly_active(user_id))
         self.assertFalse(active)
 
-        yield self.store.insert_client_ip(
-            user_id, "access_token", "ip", "user_agent", "device_id"
+        # Trigger the saving loop
+        self.reactor.advance(10)
+
+        self.get_success(
+            self.store.insert_client_ip(
+                user_id, "access_token", "ip", "user_agent", "device_id"
+            )
         )
-        active = yield self.store.user_last_seen_monthly_active(user_id)
+        active = self.get_success(self.store.user_last_seen_monthly_active(user_id))
         self.assertTrue(active)
+
+
+class ClientIpAuthTestCase(unittest.HomeserverTestCase):
+
+    servlets = [admin.register_servlets, login.register_servlets]
+
+    def make_homeserver(self, reactor, clock):
+        hs = self.setup_test_homeserver()
+        return hs
+
+    def prepare(self, hs, reactor, clock):
+        self.hs.config.registration_shared_secret = u"shared"
+        self.store = self.hs.get_datastore()
+
+        # Create the user
+        request, channel = self.make_request("GET", "/_matrix/client/r0/admin/register")
+        self.render(request)
+        nonce = channel.json_body["nonce"]
+
+        want_mac = hmac.new(key=b"shared", digestmod=hashlib.sha1)
+        want_mac.update(nonce.encode('ascii') + b"\x00bob\x00abc123\x00admin")
+        want_mac = want_mac.hexdigest()
+
+        body = json.dumps(
+            {
+                "nonce": nonce,
+                "username": "bob",
+                "password": "abc123",
+                "admin": True,
+                "mac": want_mac,
+            }
+        )
+        request, channel = self.make_request(
+            "POST", "/_matrix/client/r0/admin/register", body.encode('utf8')
+        )
+        self.render(request)
+
+        self.assertEqual(channel.code, 200)
+        self.user_id = channel.json_body["user_id"]
+
+    def test_request_with_xforwarded(self):
+        """
+        The IP in X-Forwarded-For is entered into the client IPs table.
+        """
+        self._runtest(
+            {b"X-Forwarded-For": b"127.9.0.1"},
+            "127.9.0.1",
+            {"request": XForwardedForRequest},
+        )
+
+    def test_request_from_getPeer(self):
+        """
+        The IP returned by getPeer is entered into the client IPs table, if
+        there's no X-Forwarded-For header.
+        """
+        self._runtest({}, "127.0.0.1", {})
+
+    def _runtest(self, headers, expected_ip, make_request_args):
+        device_id = "bleb"
+
+        body = json.dumps(
+            {
+                "type": "m.login.password",
+                "user": "bob",
+                "password": "abc123",
+                "device_id": device_id,
+            }
+        )
+        request, channel = self.make_request(
+            "POST", "/_matrix/client/r0/login", body.encode('utf8'), **make_request_args
+        )
+        self.render(request)
+        self.assertEqual(channel.code, 200)
+        access_token = channel.json_body["access_token"].encode('ascii')
+
+        # Advance to a known time
+        self.reactor.advance(123456 - self.reactor.seconds())
+
+        request, channel = self.make_request(
+            "GET",
+            "/_matrix/client/r0/admin/users/" + self.user_id,
+            body.encode('utf8'),
+            access_token=access_token,
+            **make_request_args
+        )
+        request.requestHeaders.addRawHeader(b"User-Agent", b"Mozzila pizza")
+
+        # Add the optional headers
+        for h, v in headers.items():
+            request.requestHeaders.addRawHeader(h, v)
+        self.render(request)
+
+        # Advance so the save loop occurs
+        self.reactor.advance(100)
+
+        result = self.get_success(
+            self.store.get_last_client_ip_by_device(self.user_id, device_id)
+        )
+        r = result[(self.user_id, device_id)]
+        self.assertDictContainsSubset(
+            {
+                "user_id": self.user_id,
+                "device_id": device_id,
+                "ip": expected_ip,
+                "user_agent": "Mozzila pizza",
+                "last_seen": 123456100,
+            },
+            r,
+        )
diff --git a/tests/unittest.py b/tests/unittest.py
index 56f3dca394d4..ef905e6389de 100644
--- a/tests/unittest.py
+++ b/tests/unittest.py
@@ -26,6 +26,7 @@
 from twisted.trial import unittest
 
 from synapse.http.server import JsonResource
+from synapse.http.site import SynapseRequest
 from synapse.server import HomeServer
 from synapse.types import UserID, create_requester
 from synapse.util.logcontext import LoggingContextFilter
@@ -237,7 +238,9 @@ def prepare(self, reactor, clock, homeserver):
         Function to optionally be overridden in subclasses.
         """
 
-    def make_request(self, method, path, content=b""):
+    def make_request(
+        self, method, path, content=b"", access_token=None, request=SynapseRequest
+    ):
         """
         Create a SynapseRequest at the path using the method and containing the
         given content.
@@ -255,7 +258,7 @@ def make_request(self, method, path, content=b""):
         if isinstance(content, dict):
             content = json.dumps(content).encode('utf8')
 
-        return make_request(method, path, content)
+        return make_request(method, path, content, access_token, request)
 
     def render(self, request):
         """
diff --git a/tests/utils.py b/tests/utils.py
index 215226debfec..aaed1149c31e 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -16,7 +16,9 @@
 import atexit
 import hashlib
 import os
+import time
 import uuid
+import warnings
 from inspect import getcallargs
 
 from mock import Mock, patch
@@ -237,20 +239,41 @@ def is_threepid_reserved(threepid):
         else:
             # We need to do cleanup on PostgreSQL
             def cleanup():
+                import psycopg2
+
                 # Close all the db pools
                 hs.get_db_pool().close()
 
+                dropped = False
+
                 # Drop the test database
                 db_conn = db_engine.module.connect(
                     database=POSTGRES_BASE_DB, user=POSTGRES_USER
                 )
                 db_conn.autocommit = True
                 cur = db_conn.cursor()
-                cur.execute("DROP DATABASE IF EXISTS %s;" % (test_db,))
-                db_conn.commit()
+
+                # Try a few times to drop the DB. Some things may hold on to the
+                # database for a few more seconds due to flakiness, preventing
+                # us from dropping it when the test is over. If we can't drop
+                # it, warn and move on.
+                for x in range(5):
+                    try:
+                        cur.execute("DROP DATABASE IF EXISTS %s;" % (test_db,))
+                        db_conn.commit()
+                        dropped = True
+                    except psycopg2.OperationalError as e:
+                        warnings.warn(
+                            "Couldn't drop old db: " + str(e), category=UserWarning
+                        )
+                        time.sleep(0.5)
+
                 cur.close()
                 db_conn.close()
 
+                if not dropped:
+                    warnings.warn("Failed to drop old DB.", category=UserWarning)
+
             if not LEAVE_DB:
                 # Register the cleanup hook
                 cleanup_func(cleanup)
diff --git a/tox.ini b/tox.ini
index 88875c331862..e4db563b4bfd 100644
--- a/tox.ini
+++ b/tox.ini
@@ -70,6 +70,16 @@ usedevelop=true
 [testenv:py36]
 usedevelop=true
 
+[testenv:py36-postgres]
+usedevelop=true
+deps =
+    {[base]deps}
+     psycopg2
+setenv =
+    {[base]setenv}
+    SYNAPSE_POSTGRES = 1
+
+
 [testenv:packaging]
 deps =
     check-manifest

From 703de4ec13b95f0c8f2e04bf8c20b3906b010592 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 20 Sep 2018 13:06:55 +0100
Subject: [PATCH 690/830] Comments and interface cleanup for on_receive_pdu

Add some informative comments about what's going on here.

Also, `sent_to_us_directly` and `get_missing` were doing the same thing (apart
from in `_handle_queued_pdus`, which looks like a bug), so let's get rid of
`get_missing` and use `sent_to_us_directly` consistently.
---
 synapse/federation/federation_server.py |  2 +-
 synapse/handlers/federation.py          | 69 ++++++++++++++++---------
 2 files changed, 47 insertions(+), 24 deletions(-)

diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index dbee404ea72a..9a571e4fc7d8 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -618,7 +618,7 @@ def _handle_received_pdu(self, origin, pdu):
             )
 
         yield self.handler.on_receive_pdu(
-            origin, pdu, get_missing=True, sent_to_us_directly=True,
+            origin, pdu, sent_to_us_directly=True,
         )
 
     def __str__(self):
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 8d6bd7976d74..2ccdc3bfa71a 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -136,7 +136,7 @@ def __init__(self, hs):
 
     @defer.inlineCallbacks
     def on_receive_pdu(
-            self, origin, pdu, get_missing=True, sent_to_us_directly=False,
+            self, origin, pdu, sent_to_us_directly=False,
     ):
         """ Process a PDU received via a federation /send/ transaction, or
         via backfill of missing prev_events
@@ -145,7 +145,8 @@ def on_receive_pdu(
             origin (str): server which initiated the /send/ transaction. Will
                 be used to fetch missing events or state.
             pdu (FrozenEvent): received PDU
-            get_missing (bool): True if we should fetch missing prev_events
+            sent_to_us_directly (bool): True if this event was pushed to us; False if
+                we pulled it as the result of a missing prev_event.
 
         Returns (Deferred): completes with None
         """
@@ -250,7 +251,7 @@ def on_receive_pdu(
                 pdu.internal_metadata.outlier = True
             elif min_depth and pdu.depth > min_depth:
                 missing_prevs = prevs - seen
-                if get_missing and missing_prevs:
+                if sent_to_us_directly and missing_prevs:
                     # If we're missing stuff, ensure we only fetch stuff one
                     # at a time.
                     logger.info(
@@ -282,24 +283,46 @@ def on_receive_pdu(
                         room_id, event_id, len(missing_prevs), shortstr(missing_prevs),
                     )
 
-            if sent_to_us_directly and prevs - seen:
-                # If they have sent it to us directly, and the server
-                # isn't telling us about the auth events that it's
-                # made a message referencing, we explode
-                logger.warn(
-                    "[%s %s] Failed to fetch %d prev events: rejecting",
-                    room_id, event_id, len(prevs - seen),
-                )
-                raise FederationError(
-                    "ERROR",
-                    403,
-                    (
-                        "Your server isn't divulging details about prev_events "
-                        "referenced in this event."
-                    ),
-                    affected=pdu.event_id,
-                )
-            elif prevs - seen:
+            if prevs - seen:
+                # We've still not been able to get all of the prev_events for this event.
+                #
+                # In this case, we need to fall back to asking another server in the
+                # federation for the state at this event. That's ok provided we then
+                # resolve the state against other bits of the DAG before using it (which
+                # will ensure that you can't just take over a room by sending an event,
+                # withholding its prev_events, and declaring yourself to be an admin in
+                # the subsequent state request).
+                #
+                # Now, if we're pulling this event as a missing prev_event, then clearly
+                # this event is not going to become the only forward-extremity and we are
+                # guaranteed to resolve its state against our existing forward
+                # extremities, so that should be fine.
+                #
+                # On the other hand, if this event was pushed to us, it is possible for
+                # it to become the only forward-extremity in the room, and we would then
+                # trust its state to be the state for the whole room. This is very bad.
+                # Further, if the event was pushed to us, there is no excuse for us not to
+                # have all the prev_events. We therefore reject any such events.
+                #
+                # XXX this really feels like it could/should be merged with the above,
+                # but there is an interaction with min_depth that I'm not really
+                # following.
+
+                if sent_to_us_directly:
+                    logger.warn(
+                        "[%s %s] Failed to fetch %d prev events: rejecting",
+                        room_id, event_id, len(prevs - seen),
+                    )
+                    raise FederationError(
+                        "ERROR",
+                        403,
+                        (
+                            "Your server isn't divulging details about prev_events "
+                            "referenced in this event."
+                        ),
+                        affected=pdu.event_id,
+                    )
+
                 # Calculate the state of the previous events, and
                 # de-conflict them to find the current state.
                 state_groups = []
@@ -464,7 +487,7 @@ def _get_missing_events_for_pdu(self, origin, pdu, prevs, min_depth):
                 yield self.on_receive_pdu(
                     origin,
                     ev,
-                    get_missing=False
+                    sent_to_us_directly=False,
                 )
             except FederationError as e:
                 if e.code == 403:
@@ -1112,7 +1135,7 @@ def _handle_queued_pdus(self, room_queue):
             try:
                 logger.info("Processing queued PDU %s which was received "
                             "while we were joining %s", p.event_id, p.room_id)
-                yield self.on_receive_pdu(origin, p)
+                yield self.on_receive_pdu(origin, p, sent_to_us_directly=True)
             except Exception as e:
                 logger.warn(
                     "Error handling queued PDU %s from %s: %s",

From 6bb9a4b1d6666ac918b3809158c3210bde630671 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 20 Sep 2018 13:15:20 +0100
Subject: [PATCH 691/830] changelog

---
 changelog.d/3924.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3924.misc

diff --git a/changelog.d/3924.misc b/changelog.d/3924.misc
new file mode 100644
index 000000000000..8d010e0bd9c5
--- /dev/null
+++ b/changelog.d/3924.misc
@@ -0,0 +1 @@
+Comments and interface cleanup for on_receive_pdu
\ No newline at end of file

From b28a7ed5030148139af6c6c03e635b068a6eae15 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 20 Sep 2018 13:44:20 +0100
Subject: [PATCH 692/830] Fix spurious exceptions when client closes conncetion

If a HTTP handler throws an exception while processing a request we
automatically write a JSON error response. If the handler had already
started writing a response twisted throws an exception.

We should check for this case and simple abort the connection if there
was an error after the response had started being written.
---
 synapse/http/server.py | 49 ++++++++++++++++++++++++++++++------------
 1 file changed, 35 insertions(+), 14 deletions(-)

diff --git a/synapse/http/server.py b/synapse/http/server.py
index 2d5c23e6739b..b4b25cab19b8 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -84,10 +84,21 @@ def wrapped_request_handler(self, request):
             logger.info(
                 "%s SynapseError: %s - %s", request, code, e.msg
             )
-            respond_with_json(
-                request, code, e.error_dict(), send_cors=True,
-                pretty_print=_request_user_agent_is_curl(request),
-            )
+
+            # Only respond with an error response if we haven't already started
+            # writing, otherwise lets just kill the connection
+            if request.startedWriting:
+                if request.transport:
+                    try:
+                        request.transport.abortConnection()
+                    except Exception:
+                        # abortConnection throws if the connection is already closed
+                        pass
+            else:
+                respond_with_json(
+                    request, code, e.error_dict(), send_cors=True,
+                    pretty_print=_request_user_agent_is_curl(request),
+                )
 
         except Exception:
             # failure.Failure() fishes the original Failure out
@@ -100,16 +111,26 @@ def wrapped_request_handler(self, request):
                 request,
                 f.getTraceback().rstrip(),
             )
-            respond_with_json(
-                request,
-                500,
-                {
-                    "error": "Internal server error",
-                    "errcode": Codes.UNKNOWN,
-                },
-                send_cors=True,
-                pretty_print=_request_user_agent_is_curl(request),
-            )
+            # Only respond with an error response if we haven't already started
+            # writing, otherwise lets just kill the connection
+            if request.startedWriting:
+                if request.transport:
+                    try:
+                        request.transport.abortConnection()
+                    except Exception:
+                        # abortConnection throws if the connection is already closed
+                        pass
+            else:
+                respond_with_json(
+                    request,
+                    500,
+                    {
+                        "error": "Internal server error",
+                        "errcode": Codes.UNKNOWN,
+                    },
+                    send_cors=True,
+                    pretty_print=_request_user_agent_is_curl(request),
+                )
 
     return wrap_async_request_handler(wrapped_request_handler)
 

From 13f6f1624b21aab82fc7f0edf90a61ce635b91be Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 20 Sep 2018 13:52:09 +0100
Subject: [PATCH 693/830] Newsfile

---
 changelog.d/3925.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3925.misc

diff --git a/changelog.d/3925.misc b/changelog.d/3925.misc
new file mode 100644
index 000000000000..3e41f78ff570
--- /dev/null
+++ b/changelog.d/3925.misc
@@ -0,0 +1 @@
+Fix spurious exceptions when remote http client closes conncetion

From 8dfb33d325a7b231668973338894d08f0d5436c9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Christian=20Gr=C3=BCnhage?=
 <jan.christian@gruenhage.xyz>
Date: Wed, 19 Sep 2018 13:48:41 +0200
Subject: [PATCH 694/830] make python 3 work in the docker container

---
 .dockerignore     | 1 +
 docker/Dockerfile | 4 +++-
 docker/start.py   | 3 ++-
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/.dockerignore b/.dockerignore
index 0180602e56fc..81e8f6d50a13 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -2,6 +2,7 @@ Dockerfile
 .travis.yml
 .gitignore
 demo/etc
+synctl
 tox.ini
 .git/*
 .tox/*
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 20d3fe3bd891..ad0ae0b734eb 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -1,4 +1,5 @@
-FROM docker.io/python:2-alpine3.8
+ARG PYTHON_VERSION=2
+FROM docker.io/python:${PYTHON_VERSION}-alpine3.8
 
 COPY . /synapse
 
@@ -12,6 +13,7 @@ RUN apk add --no-cache --virtual .build_deps \
         postgresql-dev \
         zlib-dev \
  && cd /synapse \
+ && sed -i 's/\["synctl"\] + //' setup.py \
  && apk add --no-cache --virtual .runtime_deps \
  	libffi \
         libjpeg-turbo \
diff --git a/docker/start.py b/docker/start.py
index 90e8b9c51aa2..346df8c87f06 100755
--- a/docker/start.py
+++ b/docker/start.py
@@ -5,6 +5,7 @@
 import sys
 import subprocess
 import glob
+import codecs
 
 # Utility functions
 convert = lambda src, dst, environ: open(dst, "w").write(jinja2.Template(open(src).read()).render(**environ))
@@ -23,7 +24,7 @@ def generate_secrets(environ, secrets):
                 with open(filename) as handle: value = handle.read()
             else:
                 print("Generating a random secret for {}".format(name))
-                value = os.urandom(32).encode("hex")
+                value = codecs.encode(os.urandom(32), "hex").decode()
                 with open(filename, "w") as handle: handle.write(value)
             environ[secret] = value
 

From 3af7a95a9d8b4cf16632e4b1b2c84ced89001f0c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Christian=20Gr=C3=BCnhage?=
 <jan.christian@gruenhage.xyz>
Date: Wed, 19 Sep 2018 13:51:46 +0200
Subject: [PATCH 695/830] add changelog

---
 changelog.d/3911.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3911.misc

diff --git a/changelog.d/3911.misc b/changelog.d/3911.misc
new file mode 100644
index 000000000000..e31311d52048
--- /dev/null
+++ b/changelog.d/3911.misc
@@ -0,0 +1 @@
+Fix the docker image building on python 3

From 9ea408441f527064aeafb5a7076a10a15f2c580c Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 20 Sep 2018 16:15:21 +0100
Subject: [PATCH 696/830] Handle exceptions thrown by background tasks

Fixes #3921
---
 synapse/metrics/background_process_metrics.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/synapse/metrics/background_process_metrics.py b/synapse/metrics/background_process_metrics.py
index 167167be0af3..e5293587e31e 100644
--- a/synapse/metrics/background_process_metrics.py
+++ b/synapse/metrics/background_process_metrics.py
@@ -191,6 +191,8 @@ def run():
 
             try:
                 yield func(*args, **kwargs)
+            except Exception:
+                logger.exception("Background process '%s' threw an exception", desc)
             finally:
                 proc.update_metrics()
 

From 168491c412775918b3c02dc8b10c4e3b7a50b674 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 20 Sep 2018 16:16:52 +0100
Subject: [PATCH 697/830] Newsfile

---
 changelog.d/3927.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3927.misc

diff --git a/changelog.d/3927.misc b/changelog.d/3927.misc
new file mode 100644
index 000000000000..4bd8e25f67bb
--- /dev/null
+++ b/changelog.d/3927.misc
@@ -0,0 +1 @@
+Log exceptions thrown by background tasks

From 94ae1dea3cf746ba5f26b1ab57f945cb2e734ba1 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 20 Sep 2018 17:05:34 +0100
Subject: [PATCH 698/830] Add missing logger

---
 synapse/metrics/background_process_metrics.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/synapse/metrics/background_process_metrics.py b/synapse/metrics/background_process_metrics.py
index e5293587e31e..173908299c4a 100644
--- a/synapse/metrics/background_process_metrics.py
+++ b/synapse/metrics/background_process_metrics.py
@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import logging
 import threading
 
 import six
@@ -23,6 +24,9 @@
 
 from synapse.util.logcontext import LoggingContext, PreserveLoggingContext
 
+logger = logging.getLogger(__name__)
+
+
 _background_process_start_count = Counter(
     "synapse_background_process_start_count",
     "Number of background processes started",

From a2ddaa90f21074a659abde45b712ab39324e97e4 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@arasphere.net>
Date: Thu, 20 Sep 2018 21:21:54 +0100
Subject: [PATCH 699/830] Always LL ourselves if we're in a room to simplify
 clients (#3916)

Should fix https://github.com/vector-im/riot-web/issues/7209
---
 changelog.d/3916.feature | 1 +
 synapse/handlers/sync.py | 4 ++++
 2 files changed, 5 insertions(+)
 create mode 100644 changelog.d/3916.feature

diff --git a/changelog.d/3916.feature b/changelog.d/3916.feature
new file mode 100644
index 000000000000..13282d992b9a
--- /dev/null
+++ b/changelog.d/3916.feature
@@ -0,0 +1 @@
+Always LL ourselves if we're in a room
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 9bca4e70670d..b598916b215c 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -713,6 +713,10 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                     )
                 ]
 
+                # always make sure we LL ourselves so we know we're in the room
+                # (if we are), to fix https://github.com/vector-im/riot-web/issues/7209
+                types.append((EventTypes.Member, sync_config.user.to_string()))
+
                 # only apply the filtering to room members
                 filtered_types = [EventTypes.Member]
 

From 8601c24287c452c0d9a803b130e0f68cf6169f6a Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 21 Sep 2018 14:19:46 +0100
Subject: [PATCH 700/830] Fix some instances of ExpiringCache not expiring
 cache items

ExpiringCache required that `start()` be called before it would actually
start expiring entries. A number of places didn't do that.

This PR removes `start` from ExpiringCache, and automatically starts
backround reaping process on creation instead.
---
 synapse/app/appservice.py                     |  1 -
 synapse/app/client_reader.py                  |  1 -
 synapse/app/event_creator.py                  |  1 -
 synapse/app/federation_reader.py              |  1 -
 synapse/app/federation_sender.py              |  1 -
 synapse/app/frontend_proxy.py                 |  1 -
 synapse/app/homeserver.py                     |  1 -
 synapse/app/media_repository.py               |  1 -
 synapse/app/pusher.py                         |  1 -
 synapse/app/synchrotron.py                    |  1 -
 synapse/app/user_dir.py                       |  1 -
 synapse/federation/federation_client.py       | 28 ++++++++-----------
 synapse/rest/media/v1/preview_url_resource.py |  1 -
 synapse/state/__init__.py                     |  9 ------
 synapse/util/caches/expiringcache.py          |  1 -
 tests/util/test_expiring_cache.py             |  1 -
 16 files changed, 12 insertions(+), 39 deletions(-)

diff --git a/synapse/app/appservice.py b/synapse/app/appservice.py
index 86b5067400e3..02039f7e793c 100644
--- a/synapse/app/appservice.py
+++ b/synapse/app/appservice.py
@@ -172,7 +172,6 @@ def start(config_options):
 
     def start():
         ps.get_datastore().start_profiling()
-        ps.get_state_handler().start_caching()
 
     reactor.callWhenRunning(start)
 
diff --git a/synapse/app/client_reader.py b/synapse/app/client_reader.py
index ce2b113dbb97..4c73c637bb71 100644
--- a/synapse/app/client_reader.py
+++ b/synapse/app/client_reader.py
@@ -181,7 +181,6 @@ def start(config_options):
     ss.start_listening(config.worker_listeners)
 
     def start():
-        ss.get_state_handler().start_caching()
         ss.get_datastore().start_profiling()
 
     reactor.callWhenRunning(start)
diff --git a/synapse/app/event_creator.py b/synapse/app/event_creator.py
index f98e456ea0b5..bc82197b2a83 100644
--- a/synapse/app/event_creator.py
+++ b/synapse/app/event_creator.py
@@ -199,7 +199,6 @@ def start(config_options):
     ss.start_listening(config.worker_listeners)
 
     def start():
-        ss.get_state_handler().start_caching()
         ss.get_datastore().start_profiling()
 
     reactor.callWhenRunning(start)
diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py
index 60f597350501..18ca71ef99e8 100644
--- a/synapse/app/federation_reader.py
+++ b/synapse/app/federation_reader.py
@@ -168,7 +168,6 @@ def start(config_options):
     ss.start_listening(config.worker_listeners)
 
     def start():
-        ss.get_state_handler().start_caching()
         ss.get_datastore().start_profiling()
 
     reactor.callWhenRunning(start)
diff --git a/synapse/app/federation_sender.py b/synapse/app/federation_sender.py
index 60dd09aac30c..6501c5779230 100644
--- a/synapse/app/federation_sender.py
+++ b/synapse/app/federation_sender.py
@@ -201,7 +201,6 @@ def start(config_options):
 
     def start():
         ps.get_datastore().start_profiling()
-        ps.get_state_handler().start_caching()
 
     reactor.callWhenRunning(start)
     _base.start_worker_reactor("synapse-federation-sender", config)
diff --git a/synapse/app/frontend_proxy.py b/synapse/app/frontend_proxy.py
index 8c0b9c67b07f..b076fbe522b0 100644
--- a/synapse/app/frontend_proxy.py
+++ b/synapse/app/frontend_proxy.py
@@ -258,7 +258,6 @@ def start(config_options):
     ss.start_listening(config.worker_listeners)
 
     def start():
-        ss.get_state_handler().start_caching()
         ss.get_datastore().start_profiling()
 
     reactor.callWhenRunning(start)
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 3241ded1889f..8c5d858b0b65 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -384,7 +384,6 @@ def setup(config_options):
 
     def start():
         hs.get_pusherpool().start()
-        hs.get_state_handler().start_caching()
         hs.get_datastore().start_profiling()
         hs.get_datastore().start_doing_background_updates()
         hs.get_federation_client().start_get_pdu_cache()
diff --git a/synapse/app/media_repository.py b/synapse/app/media_repository.py
index e3dbb3b4e6bb..992d182dba42 100644
--- a/synapse/app/media_repository.py
+++ b/synapse/app/media_repository.py
@@ -168,7 +168,6 @@ def start(config_options):
     ss.start_listening(config.worker_listeners)
 
     def start():
-        ss.get_state_handler().start_caching()
         ss.get_datastore().start_profiling()
 
     reactor.callWhenRunning(start)
diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py
index 244c604de91c..2ec4c7defb98 100644
--- a/synapse/app/pusher.py
+++ b/synapse/app/pusher.py
@@ -228,7 +228,6 @@ def start(config_options):
     def start():
         ps.get_pusherpool().start()
         ps.get_datastore().start_profiling()
-        ps.get_state_handler().start_caching()
 
     reactor.callWhenRunning(start)
 
diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py
index 6662340797dc..df81b7bcbe11 100644
--- a/synapse/app/synchrotron.py
+++ b/synapse/app/synchrotron.py
@@ -435,7 +435,6 @@ def start(config_options):
 
     def start():
         ss.get_datastore().start_profiling()
-        ss.get_state_handler().start_caching()
 
     reactor.callWhenRunning(start)
 
diff --git a/synapse/app/user_dir.py b/synapse/app/user_dir.py
index 96ffcaf073d4..b383e79c1ccd 100644
--- a/synapse/app/user_dir.py
+++ b/synapse/app/user_dir.py
@@ -229,7 +229,6 @@ def start(config_options):
 
     def start():
         ps.get_datastore().start_profiling()
-        ps.get_state_handler().start_caching()
 
     reactor.callWhenRunning(start)
 
diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index fe67b2ff42a9..5a92428f56c2 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -66,6 +66,14 @@ def __init__(self, hs):
         self.state = hs.get_state_handler()
         self.transport_layer = hs.get_federation_transport_client()
 
+        self._get_pdu_cache = ExpiringCache(
+            cache_name="get_pdu_cache",
+            clock=self._clock,
+            max_len=1000,
+            expiry_ms=120 * 1000,
+            reset_expiry_on_get=False,
+        )
+
     def _clear_tried_cache(self):
         """Clear pdu_destination_tried cache"""
         now = self._clock.time_msec()
@@ -82,17 +90,6 @@ def _clear_tried_cache(self):
             if destination_dict:
                 self.pdu_destination_tried[event_id] = destination_dict
 
-    def start_get_pdu_cache(self):
-        self._get_pdu_cache = ExpiringCache(
-            cache_name="get_pdu_cache",
-            clock=self._clock,
-            max_len=1000,
-            expiry_ms=120 * 1000,
-            reset_expiry_on_get=False,
-        )
-
-        self._get_pdu_cache.start()
-
     @log_function
     def make_query(self, destination, query_type, args,
                    retry_on_dns_fail=False, ignore_backoff=False):
@@ -229,10 +226,9 @@ def get_pdu(self, destinations, event_id, outlier=False, timeout=None):
 
         # TODO: Rate limit the number of times we try and get the same event.
 
-        if self._get_pdu_cache:
-            ev = self._get_pdu_cache.get(event_id)
-            if ev:
-                defer.returnValue(ev)
+        ev = self._get_pdu_cache.get(event_id)
+        if ev:
+            defer.returnValue(ev)
 
         pdu_attempts = self.pdu_destination_tried.setdefault(event_id, {})
 
@@ -285,7 +281,7 @@ def get_pdu(self, destinations, event_id, outlier=False, timeout=None):
                 )
                 continue
 
-        if self._get_pdu_cache is not None and signed_pdu:
+        if signed_pdu:
             self._get_pdu_cache[event_id] = signed_pdu
 
         defer.returnValue(signed_pdu)
diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py
index cad2dec33a2b..af01040a38da 100644
--- a/synapse/rest/media/v1/preview_url_resource.py
+++ b/synapse/rest/media/v1/preview_url_resource.py
@@ -79,7 +79,6 @@ def __init__(self, hs, media_repo, media_storage):
             # don't spider URLs more often than once an hour
             expiry_ms=60 * 60 * 1000,
         )
-        self._cache.start()
 
         self._cleaner_loop = self.clock.looping_call(
             self._start_expire_url_cache_data, 10 * 1000,
diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py
index d7ae22a661f6..b22495c1f974 100644
--- a/synapse/state/__init__.py
+++ b/synapse/state/__init__.py
@@ -95,10 +95,6 @@ def __init__(self, hs):
         self.hs = hs
         self._state_resolution_handler = hs.get_state_resolution_handler()
 
-    def start_caching(self):
-        # TODO: remove this shim
-        self._state_resolution_handler.start_caching()
-
     @defer.inlineCallbacks
     def get_current_state(self, room_id, event_type=None, state_key="",
                           latest_event_ids=None):
@@ -428,9 +424,6 @@ def __init__(self, hs):
         self._state_cache = None
         self.resolve_linearizer = Linearizer(name="state_resolve_lock")
 
-    def start_caching(self):
-        logger.debug("start_caching")
-
         self._state_cache = ExpiringCache(
             cache_name="state_cache",
             clock=self.clock,
@@ -440,8 +433,6 @@ def start_caching(self):
             reset_expiry_on_get=True,
         )
 
-        self._state_cache.start()
-
     @defer.inlineCallbacks
     @log_function
     def resolve_state_groups(
diff --git a/synapse/util/caches/expiringcache.py b/synapse/util/caches/expiringcache.py
index ce85b2ae1179..921a9c5b29a5 100644
--- a/synapse/util/caches/expiringcache.py
+++ b/synapse/util/caches/expiringcache.py
@@ -58,7 +58,6 @@ def __init__(self, cache_name, clock, max_len=0, expiry_ms=0,
 
         self.metrics = register_cache("expiring", cache_name, self)
 
-    def start(self):
         if not self._expiry_ms:
             # Don't bother starting the loop if things never expire
             return
diff --git a/tests/util/test_expiring_cache.py b/tests/util/test_expiring_cache.py
index 5cbada4edaff..50bc7702d20b 100644
--- a/tests/util/test_expiring_cache.py
+++ b/tests/util/test_expiring_cache.py
@@ -65,7 +65,6 @@ def test_iterable_eviction(self):
     def test_time_eviction(self):
         clock = MockClock()
         cache = ExpiringCache("test", clock, expiry_ms=1000)
-        cache.start()
 
         cache["key"] = 1
         clock.advance_time(0.5)

From d3f80cbc9c157303484ae6aac70c60ed2bbec19d Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 21 Sep 2018 14:24:39 +0100
Subject: [PATCH 701/830] Newsfile

---
 changelog.d/3932.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3932.bugfix

diff --git a/changelog.d/3932.bugfix b/changelog.d/3932.bugfix
new file mode 100644
index 000000000000..7578414ede72
--- /dev/null
+++ b/changelog.d/3932.bugfix
@@ -0,0 +1 @@
+Fix some instances of ExpiringCache not expiring cache items

From 79eded1ae4f0d341918f5c58076f1e60cd400e8a Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 21 Sep 2018 14:52:21 +0100
Subject: [PATCH 702/830] Make ExpiringCache slightly more performant

---
 synapse/util/caches/expiringcache.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/synapse/util/caches/expiringcache.py b/synapse/util/caches/expiringcache.py
index 921a9c5b29a5..48ca2d634d4f 100644
--- a/synapse/util/caches/expiringcache.py
+++ b/synapse/util/caches/expiringcache.py
@@ -16,6 +16,8 @@
 import logging
 from collections import OrderedDict
 
+from six import iteritems
+
 from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.util.caches import register_cache
 
@@ -127,7 +129,7 @@ def _prune_cache(self):
 
         keys_to_delete = set()
 
-        for key, cache_entry in self._cache.items():
+        for key, cache_entry in iteritems(self._cache):
             if now - cache_entry.time > self._expiry_ms:
                 keys_to_delete.add(key)
 
@@ -149,6 +151,8 @@ def __len__(self):
 
 
 class _CacheEntry(object):
+    __slots__ = ["time", "value"]
+
     def __init__(self, time, value):
         self.time = time
         self.value = value

From fdd1a62e8d09ddccbe685fe7d7840990a9c06241 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 21 Sep 2018 14:55:47 +0100
Subject: [PATCH 703/830] Add a five minute cache to
 get_destination_retry_timings

Hopefully helps with #3931
---
 synapse/storage/transactions.py      | 23 ++++++++++++++++++++++-
 synapse/util/caches/expiringcache.py | 13 +++++++++++++
 2 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py
index baf0379a6834..ab54977a7592 100644
--- a/synapse/storage/transactions.py
+++ b/synapse/storage/transactions.py
@@ -23,6 +23,7 @@
 from twisted.internet import defer
 
 from synapse.metrics.background_process_metrics import run_as_background_process
+from synapse.util.caches.expiringcache import ExpiringCache
 
 from ._base import SQLBaseStore, db_to_json
 
@@ -49,6 +50,8 @@
     )
 )
 
+SENTINEL = object()
+
 
 class TransactionStore(SQLBaseStore):
     """A collection of queries for handling PDUs.
@@ -59,6 +62,12 @@ def __init__(self, db_conn, hs):
 
         self._clock.looping_call(self._start_cleanup_transactions, 30 * 60 * 1000)
 
+        self._destination_retry_cache = ExpiringCache(
+            cache_name="get_destination_retry_timings",
+            clock=self._clock,
+            expiry_ms=5 * 60 * 1000,
+        )
+
     def get_received_txn_response(self, transaction_id, origin):
         """For an incoming transaction from a given origin, check if we have
         already responded to it. If so, return the response code and response
@@ -155,6 +164,7 @@ def delivered_txn(self, transaction_id, destination, code, response_dict):
         """
         pass
 
+    @defer.inlineCallbacks
     def get_destination_retry_timings(self, destination):
         """Gets the current retry timings (if any) for a given destination.
 
@@ -165,10 +175,20 @@ def get_destination_retry_timings(self, destination):
             None if not retrying
             Otherwise a dict for the retry scheme
         """
-        return self.runInteraction(
+
+        result = self._destination_retry_cache.get(destination, SENTINEL)
+        if result is not SENTINEL:
+            defer.returnValue(result)
+
+        result = yield self.runInteraction(
             "get_destination_retry_timings",
             self._get_destination_retry_timings, destination)
 
+        # We don't hugely care about race conditions between getting and
+        # invalidating the cache, since we time out fairly quickly anyway.
+        self._destination_retry_cache[destination] = result
+        defer.returnValue(result)
+
     def _get_destination_retry_timings(self, txn, destination):
         result = self._simple_select_one_txn(
             txn,
@@ -196,6 +216,7 @@ def set_destination_retry_timings(self, destination,
             retry_interval (int) - how long until next retry in ms
         """
 
+        self._destination_retry_cache.pop(destination)
         return self.runInteraction(
             "set_destination_retry_timings",
             self._set_destination_retry_timings,
diff --git a/synapse/util/caches/expiringcache.py b/synapse/util/caches/expiringcache.py
index 48ca2d634d4f..346669e4ce37 100644
--- a/synapse/util/caches/expiringcache.py
+++ b/synapse/util/caches/expiringcache.py
@@ -24,6 +24,9 @@
 logger = logging.getLogger(__name__)
 
 
+SENTINEL = object()
+
+
 class ExpiringCache(object):
     def __init__(self, cache_name, clock, max_len=0, expiry_ms=0,
                  reset_expiry_on_get=False, iterable=False):
@@ -102,6 +105,16 @@ def __getitem__(self, key):
 
         return entry.value
 
+    def pop(self, key, default=None):
+        value = self._cache.pop(key, SENTINEL)
+        if value is SENTINEL:
+            return default
+
+        if self.iterable:
+            self._size_estimate -= len(value.value)
+
+        return value
+
     def __contains__(self, key):
         return key in self._cache
 

From 5230bc147159d6ffbe04d27e695064cac44166d2 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 21 Sep 2018 15:05:37 +0100
Subject: [PATCH 704/830] Newsfile

---
 changelog.d/3933.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3933.misc

diff --git a/changelog.d/3933.misc b/changelog.d/3933.misc
new file mode 100644
index 000000000000..6545871f55eb
--- /dev/null
+++ b/changelog.d/3933.misc
@@ -0,0 +1 @@
+Add a cache to get_destination_retry_timings

From 19dc676d1aaf66a45c9a1810132952f7ae3ca2aa Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 21 Sep 2018 16:25:42 +0100
Subject: [PATCH 705/830] Fix ExpiringCache.__len__ to be accurate

It used to try and produce an estimate, which was sometimes negative.
This caused metrics to be sad, so lets always just calculate it from
scratch.
---
 synapse/util/caches/expiringcache.py | 21 +++++++++------------
 1 file changed, 9 insertions(+), 12 deletions(-)

diff --git a/synapse/util/caches/expiringcache.py b/synapse/util/caches/expiringcache.py
index 346669e4ce37..8ac56d85ff36 100644
--- a/synapse/util/caches/expiringcache.py
+++ b/synapse/util/caches/expiringcache.py
@@ -16,7 +16,7 @@
 import logging
 from collections import OrderedDict
 
-from six import iteritems
+from six import iteritems, itervalues
 
 from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.util.caches import register_cache
@@ -59,8 +59,6 @@ def __init__(self, cache_name, clock, max_len=0, expiry_ms=0,
 
         self.iterable = iterable
 
-        self._size_estimate = 0
-
         self.metrics = register_cache("expiring", cache_name, self)
 
         if not self._expiry_ms:
@@ -79,16 +77,11 @@ def __setitem__(self, key, value):
         now = self._clock.time_msec()
         self._cache[key] = _CacheEntry(now, value)
 
-        if self.iterable:
-            self._size_estimate += len(value)
-
         # Evict if there are now too many items
         while self._max_len and len(self) > self._max_len:
             _key, value = self._cache.popitem(last=False)
             if self.iterable:
-                removed_len = len(value.value)
-                self.metrics.inc_evictions(removed_len)
-                self._size_estimate -= removed_len
+                self.metrics.inc_evictions(len(value.value))
             else:
                 self.metrics.inc_evictions()
 
@@ -111,7 +104,9 @@ def pop(self, key, default=None):
             return default
 
         if self.iterable:
-            self._size_estimate -= len(value.value)
+            self.metrics.inc_evictions(len(value.value))
+        else:
+            self.metrics.inc_evictions()
 
         return value
 
@@ -149,7 +144,9 @@ def _prune_cache(self):
         for k in keys_to_delete:
             value = self._cache.pop(k)
             if self.iterable:
-                self._size_estimate -= len(value.value)
+                self.metrics.inc_evictions(len(value.value))
+            else:
+                self.metrics.inc_evictions()
 
         logger.debug(
             "[%s] _prune_cache before: %d, after len: %d",
@@ -158,7 +155,7 @@ def _prune_cache(self):
 
     def __len__(self):
         if self.iterable:
-            return self._size_estimate
+            return sum(len(entry.value) for entry in itervalues(self._cache))
         else:
             return len(self._cache)
 

From e302f40e20df3b2fdb1a3fd18abd4164455e8ac7 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Mon, 24 Sep 2018 23:40:05 +1000
Subject: [PATCH 706/830] update version

---
 synapse/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/__init__.py b/synapse/__init__.py
index f6a194dfc2e0..58244a5dd407 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -27,4 +27,4 @@
 except ImportError:
     pass
 
-__version__ = "0.33.5rc1"
+__version__ = "0.33.5"

From e3aa2c0b75b062efc78388abba0f8b49666d6c54 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Mon, 24 Sep 2018 23:40:27 +1000
Subject: [PATCH 707/830] towncrier

---
 CHANGES.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/CHANGES.md b/CHANGES.md
index 00dbed702164..58fb9acddaa9 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,9 @@
+Synapse 0.33.5 (2018-09-24)
+===========================
+
+No significant changes.
+
+
 Synapse 0.33.5rc1 (2018-09-17)
 ==============================
 

From 7d3f639844f50061367db25cac0bd4d189648678 Mon Sep 17 00:00:00 2001
From: Oleg Girko <ol@infoserver.lv>
Date: Mon, 24 Sep 2018 16:51:59 +0100
Subject: [PATCH 708/830] Fix compatibility issue with older Twisted in tests.

Older Twisted (18.4.0) returns TimeoutError instead of
ConnectingCancelledError when connection times out.
This change allows tests to be compatible with this behaviour.

Signed-off-by: Oleg Girko <ol@infoserver.lv>
---
 tests/http/test_fedclient.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/http/test_fedclient.py b/tests/http/test_fedclient.py
index 1c46c9cfeb9e..d45d2bcb169f 100644
--- a/tests/http/test_fedclient.py
+++ b/tests/http/test_fedclient.py
@@ -49,7 +49,7 @@ def test_dns_error(self):
     def test_client_never_connect(self):
         """
         If the HTTP request is not connected and is timed out, it'll give a
-        ConnectingCancelledError.
+        ConnectingCancelledError or TimeoutError.
         """
         d = self.cl._request("testserv:8008", "GET", "foo/bar", timeout=10000)
 
@@ -71,7 +71,7 @@ def test_client_never_connect(self):
         self.reactor.advance(10.5)
         f = self.failureResultOf(d)
 
-        self.assertIsInstance(f.value, ConnectingCancelledError)
+        self.assertIsInstance(f.value, (ConnectingCancelledError, TimeoutError))
 
     def test_client_connect_no_response(self):
         """

From e37c221b9792caf5a451079d9ce995609fa624f9 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Tue, 25 Sep 2018 02:51:55 +1000
Subject: [PATCH 709/830] changelog for 3940

---
 changelog.d/3940.misc | 2 ++
 1 file changed, 2 insertions(+)
 create mode 100644 changelog.d/3940.misc

diff --git a/changelog.d/3940.misc b/changelog.d/3940.misc
new file mode 100644
index 000000000000..a39d2bde3ed6
--- /dev/null
+++ b/changelog.d/3940.misc
@@ -0,0 +1,2 @@
+Fix incompatibility with older Twisted version in tests. Thanks 
+@OlegGirko!

From 6b6cb32297b29a1dc6c415150b8a9a37b38349bf Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Tue, 25 Sep 2018 02:54:34 +1000
Subject: [PATCH 710/830] bump version

---
 synapse/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/__init__.py b/synapse/__init__.py
index 58244a5dd407..b1f7a89fba5e 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -27,4 +27,4 @@
 except ImportError:
     pass
 
-__version__ = "0.33.5"
+__version__ = "0.33.5.1"

From fc691ca97ca171b81faef83e116a2c11a6816e1b Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Tue, 25 Sep 2018 02:54:59 +1000
Subject: [PATCH 711/830] changelog

---
 CHANGES.md            | 10 ++++++++++
 changelog.d/3940.misc |  2 --
 2 files changed, 10 insertions(+), 2 deletions(-)
 delete mode 100644 changelog.d/3940.misc

diff --git a/CHANGES.md b/CHANGES.md
index 58fb9acddaa9..45d3cdb1314b 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,13 @@
+Synapse 0.33.5.1 (2018-09-25)
+=============================
+
+Internal Changes
+----------------
+
+- Fix incompatibility with older Twisted version in tests. Thanks 
+  @OlegGirko! ([\#3940](https://github.com/matrix-org/synapse/issues/3940))
+
+
 Synapse 0.33.5 (2018-09-24)
 ===========================
 
diff --git a/changelog.d/3940.misc b/changelog.d/3940.misc
deleted file mode 100644
index a39d2bde3ed6..000000000000
--- a/changelog.d/3940.misc
+++ /dev/null
@@ -1,2 +0,0 @@
-Fix incompatibility with older Twisted version in tests. Thanks 
-@OlegGirko!

From 787d22ed6cbc8f78fed7e316d345e54b7c6a93da Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@arasphere.net>
Date: Tue, 25 Sep 2018 00:49:26 +0100
Subject: [PATCH 712/830] Only lazy load self-members on initial sync

Given we have disabled lazy loading for incr syncs in #3840, we can make self-LL more efficient by only doing it on initial sync.  Also adds a bounds check for if/when we change our mind, so that we don't try to include LL members on sync responses with no timeline.
---
 changelog.d/3936.bugfix  |  1 +
 synapse/handlers/sync.py | 13 ++++++++-----
 2 files changed, 9 insertions(+), 5 deletions(-)
 create mode 100644 changelog.d/3936.bugfix

diff --git a/changelog.d/3936.bugfix b/changelog.d/3936.bugfix
new file mode 100644
index 000000000000..49b02b9e2781
--- /dev/null
+++ b/changelog.d/3936.bugfix
@@ -0,0 +1 @@
+Fix out-of-bounds error when LLing yourself
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index b598916b215c..c7d69d9d805c 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -713,10 +713,6 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
                     )
                 ]
 
-                # always make sure we LL ourselves so we know we're in the room
-                # (if we are), to fix https://github.com/vector-im/riot-web/issues/7209
-                types.append((EventTypes.Member, sync_config.user.to_string()))
-
                 # only apply the filtering to room members
                 filtered_types = [EventTypes.Member]
 
@@ -726,6 +722,13 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
             }
 
             if full_state:
+                if lazy_load_members:
+                    # always make sure we LL ourselves so we know we're in the room
+                    # (if we are) to fix https://github.com/vector-im/riot-web/issues/7209
+                    # We only need apply this on full state syncs given we disabled
+                    # LL for incr syncs in #3840.
+                    types.append((EventTypes.Member, sync_config.user.to_string()))
+
                 if batch:
                     current_state_ids = yield self.store.get_state_ids_for_event(
                         batch.events[-1].event_id, types=types,
@@ -794,7 +797,7 @@ def compute_state_delta(self, room_id, batch, sync_config, since_token, now_toke
             else:
                 state_ids = {}
                 if lazy_load_members:
-                    if types:
+                    if types and batch.events:
                         # We're returning an incremental sync, with no
                         # "gap" since the previous sync, so normally there would be
                         # no state to return.

From a9d84f4e44783bc75dd931093339636732bfbf6c Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 25 Sep 2018 10:43:39 +0100
Subject: [PATCH 713/830] We require attrs 16.0.0

Ref: https://github.com/matrix-org/synapse/issues/3945
---
 synapse/python_dependencies.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index 0d8de600cfa7..c779f69fa07f 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -58,7 +58,9 @@
     "phonenumbers>=8.2.0": ["phonenumbers"],
     "six": ["six"],
     "prometheus_client": ["prometheus_client"],
-    "attrs": ["attr"],
+
+    # we use attr.s(slots), which arrived in 16.0.0
+    "attrs>=16.0.0": ["attr>=16.0.0"],
     "netaddr>=0.7.18": ["netaddr"],
 }
 

From 45a1053d447903aae9b75d9a86016e71034917b9 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 25 Sep 2018 10:45:34 +0100
Subject: [PATCH 714/830] changelog

---
 changelog.d/3947.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3947.misc

diff --git a/changelog.d/3947.misc b/changelog.d/3947.misc
new file mode 100644
index 000000000000..5a9a22bed992
--- /dev/null
+++ b/changelog.d/3947.misc
@@ -0,0 +1 @@
+Require attrs 16.0.0 or later

From c53336986d9208925438a8e2366bac7d1267a95b Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 25 Sep 2018 11:19:27 +0100
Subject: [PATCH 715/830] Move synctl into top dir to avoid a symlink

symlinks apparently break setuptools on python3 and alpine
(https://bugs.python.org/issue31940), so let's stop using a symlink and just
use the file directly.
---
 synapse/app/synctl.py | 284 -----------------------------------------
 synctl                | 285 +++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 284 insertions(+), 285 deletions(-)
 delete mode 100755 synapse/app/synctl.py
 mode change 120000 => 100755 synctl

diff --git a/synapse/app/synctl.py b/synapse/app/synctl.py
deleted file mode 100755
index d658f967ba87..000000000000
--- a/synapse/app/synctl.py
+++ /dev/null
@@ -1,284 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright 2014-2016 OpenMarket Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import argparse
-import collections
-import errno
-import glob
-import os
-import os.path
-import signal
-import subprocess
-import sys
-import time
-
-from six import iteritems
-
-import yaml
-
-SYNAPSE = [sys.executable, "-B", "-m", "synapse.app.homeserver"]
-
-GREEN = "\x1b[1;32m"
-YELLOW = "\x1b[1;33m"
-RED = "\x1b[1;31m"
-NORMAL = "\x1b[m"
-
-
-def pid_running(pid):
-    try:
-        os.kill(pid, 0)
-        return True
-    except OSError as err:
-        if err.errno == errno.EPERM:
-            return True
-        return False
-
-
-def write(message, colour=NORMAL, stream=sys.stdout):
-    if colour == NORMAL:
-        stream.write(message + "\n")
-    else:
-        stream.write(colour + message + NORMAL + "\n")
-
-
-def abort(message, colour=RED, stream=sys.stderr):
-    write(message, colour, stream)
-    sys.exit(1)
-
-
-def start(configfile):
-    write("Starting ...")
-    args = SYNAPSE
-    args.extend(["--daemonize", "-c", configfile])
-
-    try:
-        subprocess.check_call(args)
-        write("started synapse.app.homeserver(%r)" %
-              (configfile,), colour=GREEN)
-    except subprocess.CalledProcessError as e:
-        write(
-            "error starting (exit code: %d); see above for logs" % e.returncode,
-            colour=RED,
-        )
-
-
-def start_worker(app, configfile, worker_configfile):
-    args = [
-        "python", "-B",
-        "-m", app,
-        "-c", configfile,
-        "-c", worker_configfile
-    ]
-
-    try:
-        subprocess.check_call(args)
-        write("started %s(%r)" % (app, worker_configfile), colour=GREEN)
-    except subprocess.CalledProcessError as e:
-        write(
-            "error starting %s(%r) (exit code: %d); see above for logs" % (
-                app, worker_configfile, e.returncode,
-            ),
-            colour=RED,
-        )
-
-
-def stop(pidfile, app):
-    if os.path.exists(pidfile):
-        pid = int(open(pidfile).read())
-        try:
-            os.kill(pid, signal.SIGTERM)
-            write("stopped %s" % (app,), colour=GREEN)
-        except OSError as err:
-            if err.errno == errno.ESRCH:
-                write("%s not running" % (app,), colour=YELLOW)
-            elif err.errno == errno.EPERM:
-                abort("Cannot stop %s: Operation not permitted" % (app,))
-            else:
-                abort("Cannot stop %s: Unknown error" % (app,))
-
-
-Worker = collections.namedtuple("Worker", [
-    "app", "configfile", "pidfile", "cache_factor"
-])
-
-
-def main():
-
-    parser = argparse.ArgumentParser()
-
-    parser.add_argument(
-        "action",
-        choices=["start", "stop", "restart"],
-        help="whether to start, stop or restart the synapse",
-    )
-    parser.add_argument(
-        "configfile",
-        nargs="?",
-        default="homeserver.yaml",
-        help="the homeserver config file, defaults to homeserver.yaml",
-    )
-    parser.add_argument(
-        "-w", "--worker",
-        metavar="WORKERCONFIG",
-        help="start or stop a single worker",
-    )
-    parser.add_argument(
-        "-a", "--all-processes",
-        metavar="WORKERCONFIGDIR",
-        help="start or stop all the workers in the given directory"
-             " and the main synapse process",
-    )
-
-    options = parser.parse_args()
-
-    if options.worker and options.all_processes:
-        write(
-            'Cannot use "--worker" with "--all-processes"',
-            stream=sys.stderr
-        )
-        sys.exit(1)
-
-    configfile = options.configfile
-
-    if not os.path.exists(configfile):
-        write(
-            "No config file found\n"
-            "To generate a config file, run '%s -c %s --generate-config"
-            " --server-name=<server name>'\n" % (
-                " ".join(SYNAPSE), options.configfile
-            ),
-            stream=sys.stderr,
-        )
-        sys.exit(1)
-
-    with open(configfile) as stream:
-        config = yaml.load(stream)
-
-    pidfile = config["pid_file"]
-    cache_factor = config.get("synctl_cache_factor")
-    start_stop_synapse = True
-
-    if cache_factor:
-        os.environ["SYNAPSE_CACHE_FACTOR"] = str(cache_factor)
-
-    cache_factors = config.get("synctl_cache_factors", {})
-    for cache_name, factor in iteritems(cache_factors):
-        os.environ["SYNAPSE_CACHE_FACTOR_" + cache_name.upper()] = str(factor)
-
-    worker_configfiles = []
-    if options.worker:
-        start_stop_synapse = False
-        worker_configfile = options.worker
-        if not os.path.exists(worker_configfile):
-            write(
-                "No worker config found at %r" % (worker_configfile,),
-                stream=sys.stderr,
-            )
-            sys.exit(1)
-        worker_configfiles.append(worker_configfile)
-
-    if options.all_processes:
-        # To start the main synapse with -a you need to add a worker file
-        # with worker_app == "synapse.app.homeserver"
-        start_stop_synapse = False
-        worker_configdir = options.all_processes
-        if not os.path.isdir(worker_configdir):
-            write(
-                "No worker config directory found at %r" % (worker_configdir,),
-                stream=sys.stderr,
-            )
-            sys.exit(1)
-        worker_configfiles.extend(sorted(glob.glob(
-            os.path.join(worker_configdir, "*.yaml")
-        )))
-
-    workers = []
-    for worker_configfile in worker_configfiles:
-        with open(worker_configfile) as stream:
-            worker_config = yaml.load(stream)
-        worker_app = worker_config["worker_app"]
-        if worker_app == "synapse.app.homeserver":
-            # We need to special case all of this to pick up options that may
-            # be set in the main config file or in this worker config file.
-            worker_pidfile = (
-                worker_config.get("pid_file")
-                or pidfile
-            )
-            worker_cache_factor = worker_config.get("synctl_cache_factor") or cache_factor
-            daemonize = worker_config.get("daemonize") or config.get("daemonize")
-            assert daemonize, "Main process must have daemonize set to true"
-
-            # The master process doesn't support using worker_* config.
-            for key in worker_config:
-                if key == "worker_app":  # But we allow worker_app
-                    continue
-                assert not key.startswith("worker_"), \
-                    "Main process cannot use worker_* config"
-        else:
-            worker_pidfile = worker_config["worker_pid_file"]
-            worker_daemonize = worker_config["worker_daemonize"]
-            assert worker_daemonize, "In config %r: expected '%s' to be True" % (
-                worker_configfile, "worker_daemonize")
-            worker_cache_factor = worker_config.get("synctl_cache_factor")
-        workers.append(Worker(
-            worker_app, worker_configfile, worker_pidfile, worker_cache_factor,
-        ))
-
-    action = options.action
-
-    if action == "stop" or action == "restart":
-        for worker in workers:
-            stop(worker.pidfile, worker.app)
-
-        if start_stop_synapse:
-            stop(pidfile, "synapse.app.homeserver")
-
-    # Wait for synapse to actually shutdown before starting it again
-    if action == "restart":
-        running_pids = []
-        if start_stop_synapse and os.path.exists(pidfile):
-            running_pids.append(int(open(pidfile).read()))
-        for worker in workers:
-            if os.path.exists(worker.pidfile):
-                running_pids.append(int(open(worker.pidfile).read()))
-        if len(running_pids) > 0:
-            write("Waiting for process to exit before restarting...")
-            for running_pid in running_pids:
-                while pid_running(running_pid):
-                    time.sleep(0.2)
-            write("All processes exited; now restarting...")
-
-    if action == "start" or action == "restart":
-        if start_stop_synapse:
-            # Check if synapse is already running
-            if os.path.exists(pidfile) and pid_running(int(open(pidfile).read())):
-                abort("synapse.app.homeserver already running")
-            start(configfile)
-
-        for worker in workers:
-            if worker.cache_factor:
-                os.environ["SYNAPSE_CACHE_FACTOR"] = str(worker.cache_factor)
-
-            start_worker(worker.app, configfile, worker.configfile)
-
-            if cache_factor:
-                os.environ["SYNAPSE_CACHE_FACTOR"] = str(cache_factor)
-            else:
-                os.environ.pop("SYNAPSE_CACHE_FACTOR", None)
-
-
-if __name__ == "__main__":
-    main()
diff --git a/synctl b/synctl
deleted file mode 120000
index 1bdceda20adc..000000000000
--- a/synctl
+++ /dev/null
@@ -1 +0,0 @@
-./synapse/app/synctl.py
\ No newline at end of file
diff --git a/synctl b/synctl
new file mode 100755
index 000000000000..d658f967ba87
--- /dev/null
+++ b/synctl
@@ -0,0 +1,284 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright 2014-2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import collections
+import errno
+import glob
+import os
+import os.path
+import signal
+import subprocess
+import sys
+import time
+
+from six import iteritems
+
+import yaml
+
+SYNAPSE = [sys.executable, "-B", "-m", "synapse.app.homeserver"]
+
+GREEN = "\x1b[1;32m"
+YELLOW = "\x1b[1;33m"
+RED = "\x1b[1;31m"
+NORMAL = "\x1b[m"
+
+
+def pid_running(pid):
+    try:
+        os.kill(pid, 0)
+        return True
+    except OSError as err:
+        if err.errno == errno.EPERM:
+            return True
+        return False
+
+
+def write(message, colour=NORMAL, stream=sys.stdout):
+    if colour == NORMAL:
+        stream.write(message + "\n")
+    else:
+        stream.write(colour + message + NORMAL + "\n")
+
+
+def abort(message, colour=RED, stream=sys.stderr):
+    write(message, colour, stream)
+    sys.exit(1)
+
+
+def start(configfile):
+    write("Starting ...")
+    args = SYNAPSE
+    args.extend(["--daemonize", "-c", configfile])
+
+    try:
+        subprocess.check_call(args)
+        write("started synapse.app.homeserver(%r)" %
+              (configfile,), colour=GREEN)
+    except subprocess.CalledProcessError as e:
+        write(
+            "error starting (exit code: %d); see above for logs" % e.returncode,
+            colour=RED,
+        )
+
+
+def start_worker(app, configfile, worker_configfile):
+    args = [
+        "python", "-B",
+        "-m", app,
+        "-c", configfile,
+        "-c", worker_configfile
+    ]
+
+    try:
+        subprocess.check_call(args)
+        write("started %s(%r)" % (app, worker_configfile), colour=GREEN)
+    except subprocess.CalledProcessError as e:
+        write(
+            "error starting %s(%r) (exit code: %d); see above for logs" % (
+                app, worker_configfile, e.returncode,
+            ),
+            colour=RED,
+        )
+
+
+def stop(pidfile, app):
+    if os.path.exists(pidfile):
+        pid = int(open(pidfile).read())
+        try:
+            os.kill(pid, signal.SIGTERM)
+            write("stopped %s" % (app,), colour=GREEN)
+        except OSError as err:
+            if err.errno == errno.ESRCH:
+                write("%s not running" % (app,), colour=YELLOW)
+            elif err.errno == errno.EPERM:
+                abort("Cannot stop %s: Operation not permitted" % (app,))
+            else:
+                abort("Cannot stop %s: Unknown error" % (app,))
+
+
+Worker = collections.namedtuple("Worker", [
+    "app", "configfile", "pidfile", "cache_factor"
+])
+
+
+def main():
+
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "action",
+        choices=["start", "stop", "restart"],
+        help="whether to start, stop or restart the synapse",
+    )
+    parser.add_argument(
+        "configfile",
+        nargs="?",
+        default="homeserver.yaml",
+        help="the homeserver config file, defaults to homeserver.yaml",
+    )
+    parser.add_argument(
+        "-w", "--worker",
+        metavar="WORKERCONFIG",
+        help="start or stop a single worker",
+    )
+    parser.add_argument(
+        "-a", "--all-processes",
+        metavar="WORKERCONFIGDIR",
+        help="start or stop all the workers in the given directory"
+             " and the main synapse process",
+    )
+
+    options = parser.parse_args()
+
+    if options.worker and options.all_processes:
+        write(
+            'Cannot use "--worker" with "--all-processes"',
+            stream=sys.stderr
+        )
+        sys.exit(1)
+
+    configfile = options.configfile
+
+    if not os.path.exists(configfile):
+        write(
+            "No config file found\n"
+            "To generate a config file, run '%s -c %s --generate-config"
+            " --server-name=<server name>'\n" % (
+                " ".join(SYNAPSE), options.configfile
+            ),
+            stream=sys.stderr,
+        )
+        sys.exit(1)
+
+    with open(configfile) as stream:
+        config = yaml.load(stream)
+
+    pidfile = config["pid_file"]
+    cache_factor = config.get("synctl_cache_factor")
+    start_stop_synapse = True
+
+    if cache_factor:
+        os.environ["SYNAPSE_CACHE_FACTOR"] = str(cache_factor)
+
+    cache_factors = config.get("synctl_cache_factors", {})
+    for cache_name, factor in iteritems(cache_factors):
+        os.environ["SYNAPSE_CACHE_FACTOR_" + cache_name.upper()] = str(factor)
+
+    worker_configfiles = []
+    if options.worker:
+        start_stop_synapse = False
+        worker_configfile = options.worker
+        if not os.path.exists(worker_configfile):
+            write(
+                "No worker config found at %r" % (worker_configfile,),
+                stream=sys.stderr,
+            )
+            sys.exit(1)
+        worker_configfiles.append(worker_configfile)
+
+    if options.all_processes:
+        # To start the main synapse with -a you need to add a worker file
+        # with worker_app == "synapse.app.homeserver"
+        start_stop_synapse = False
+        worker_configdir = options.all_processes
+        if not os.path.isdir(worker_configdir):
+            write(
+                "No worker config directory found at %r" % (worker_configdir,),
+                stream=sys.stderr,
+            )
+            sys.exit(1)
+        worker_configfiles.extend(sorted(glob.glob(
+            os.path.join(worker_configdir, "*.yaml")
+        )))
+
+    workers = []
+    for worker_configfile in worker_configfiles:
+        with open(worker_configfile) as stream:
+            worker_config = yaml.load(stream)
+        worker_app = worker_config["worker_app"]
+        if worker_app == "synapse.app.homeserver":
+            # We need to special case all of this to pick up options that may
+            # be set in the main config file or in this worker config file.
+            worker_pidfile = (
+                worker_config.get("pid_file")
+                or pidfile
+            )
+            worker_cache_factor = worker_config.get("synctl_cache_factor") or cache_factor
+            daemonize = worker_config.get("daemonize") or config.get("daemonize")
+            assert daemonize, "Main process must have daemonize set to true"
+
+            # The master process doesn't support using worker_* config.
+            for key in worker_config:
+                if key == "worker_app":  # But we allow worker_app
+                    continue
+                assert not key.startswith("worker_"), \
+                    "Main process cannot use worker_* config"
+        else:
+            worker_pidfile = worker_config["worker_pid_file"]
+            worker_daemonize = worker_config["worker_daemonize"]
+            assert worker_daemonize, "In config %r: expected '%s' to be True" % (
+                worker_configfile, "worker_daemonize")
+            worker_cache_factor = worker_config.get("synctl_cache_factor")
+        workers.append(Worker(
+            worker_app, worker_configfile, worker_pidfile, worker_cache_factor,
+        ))
+
+    action = options.action
+
+    if action == "stop" or action == "restart":
+        for worker in workers:
+            stop(worker.pidfile, worker.app)
+
+        if start_stop_synapse:
+            stop(pidfile, "synapse.app.homeserver")
+
+    # Wait for synapse to actually shutdown before starting it again
+    if action == "restart":
+        running_pids = []
+        if start_stop_synapse and os.path.exists(pidfile):
+            running_pids.append(int(open(pidfile).read()))
+        for worker in workers:
+            if os.path.exists(worker.pidfile):
+                running_pids.append(int(open(worker.pidfile).read()))
+        if len(running_pids) > 0:
+            write("Waiting for process to exit before restarting...")
+            for running_pid in running_pids:
+                while pid_running(running_pid):
+                    time.sleep(0.2)
+            write("All processes exited; now restarting...")
+
+    if action == "start" or action == "restart":
+        if start_stop_synapse:
+            # Check if synapse is already running
+            if os.path.exists(pidfile) and pid_running(int(open(pidfile).read())):
+                abort("synapse.app.homeserver already running")
+            start(configfile)
+
+        for worker in workers:
+            if worker.cache_factor:
+                os.environ["SYNAPSE_CACHE_FACTOR"] = str(worker.cache_factor)
+
+            start_worker(worker.app, configfile, worker.configfile)
+
+            if cache_factor:
+                os.environ["SYNAPSE_CACHE_FACTOR"] = str(cache_factor)
+            else:
+                os.environ.pop("SYNAPSE_CACHE_FACTOR", None)
+
+
+if __name__ == "__main__":
+    main()

From bd469adaa9f3ae079e06d3510f61da537c5ee1cd Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 25 Sep 2018 11:22:17 +0100
Subject: [PATCH 716/830] changelog

---
 changelog.d/3948.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3948.misc

diff --git a/changelog.d/3948.misc b/changelog.d/3948.misc
new file mode 100644
index 000000000000..a93edbd1c354
--- /dev/null
+++ b/changelog.d/3948.misc
@@ -0,0 +1 @@
+Fix incompatibility with python3 on alpine
\ No newline at end of file

From 6cf261930ac4c5f08cd3d6bf1f5a37a2889f6e7b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Farnaud?=
 <remjey@users.noreply.github.com>
Date: Tue, 25 Sep 2018 12:55:02 +0200
Subject: [PATCH 717/830] added "media-src: 'self'" to CSP for resources
 (#3578)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Synapse doesn’t allow for media resources to be played directly from
Chrome. It is a problem for users on other networks (e.g. IRC)
communicating with Matrix users through a gateway. The gateway sends
them the raw URL for the resource when a Matrix user uploads a video
and the video cannot be played directly in Chrome using that URL.

Chrome argues it is not authorized to play the video because of the
Content Security Policy. Chrome checks for the "media-src" policy which
is missing, and defauts to the "default-src" policy which is "none".

As Synapse already sends "object-src: 'self'" I thought it wouldn’t be
a problem to add "media-src: 'self'" to the CSP to fix this problem.
---
 changelog.d/3578.bugfix                    | 1 +
 synapse/rest/media/v1/download_resource.py | 1 +
 2 files changed, 2 insertions(+)
 create mode 100644 changelog.d/3578.bugfix

diff --git a/changelog.d/3578.bugfix b/changelog.d/3578.bugfix
new file mode 100644
index 000000000000..9c52b6fa7e92
--- /dev/null
+++ b/changelog.d/3578.bugfix
@@ -0,0 +1 @@
+Fix problem when playing media from Chrome using direct URL (thanks @remjey!)
diff --git a/synapse/rest/media/v1/download_resource.py b/synapse/rest/media/v1/download_resource.py
index ca90964d1dfd..f911b120b154 100644
--- a/synapse/rest/media/v1/download_resource.py
+++ b/synapse/rest/media/v1/download_resource.py
@@ -52,6 +52,7 @@ def _async_render_GET(self, request):
             b" script-src 'none';"
             b" plugin-types application/pdf;"
             b" style-src 'unsafe-inline';"
+            b" media-src 'self';"
             b" object-src 'self';"
         )
         server_name, media_id, name = parse_media_id(request)

From 0649306fde79c56e9668682ab7285fdc2e5483c9 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 25 Sep 2018 13:29:28 +0100
Subject: [PATCH 718/830] Update grafana dashboard

---
 contrib/grafana/synapse.json | 592 ++++++++++++++++++++++++-----------
 1 file changed, 417 insertions(+), 175 deletions(-)

diff --git a/contrib/grafana/synapse.json b/contrib/grafana/synapse.json
index c58612594ab8..dc3f4a1d1c71 100644
--- a/contrib/grafana/synapse.json
+++ b/contrib/grafana/synapse.json
@@ -14,7 +14,7 @@
       "type": "grafana",
       "id": "grafana",
       "name": "Grafana",
-      "version": "5.2.0"
+      "version": "5.2.4"
     },
     {
       "type": "panel",
@@ -54,7 +54,7 @@
   "gnetId": null,
   "graphTooltip": 0,
   "id": null,
-  "iteration": 1533598785368,
+  "iteration": 1537878047048,
   "links": [
     {
       "asDropdown": true,
@@ -86,7 +86,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "$datasource",
       "fill": 1,
       "gridPos": {
         "h": 9,
@@ -118,7 +118,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "process_cpu_seconds:rate2m{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+          "expr": "rate(process_cpu_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{job}}-{{index}} ",
@@ -179,7 +179,7 @@
         "mode": "spectrum"
       },
       "dataFormat": "tsbuckets",
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "$datasource",
       "gridPos": {
         "h": 9,
         "w": 12,
@@ -525,7 +525,7 @@
             "x": 0,
             "y": 25
           },
-          "id": 48,
+          "id": 50,
           "legend": {
             "avg": false,
             "current": false,
@@ -549,8 +549,9 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "rate(synapse_storage_schedule_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_schedule_time_count[$bucket_size])",
+              "expr": "rate(python_twisted_reactor_tick_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(python_twisted_reactor_tick_time_count[$bucket_size])",
               "format": "time_series",
+              "interval": "",
               "intervalFactor": 2,
               "legendFormat": "{{job}}-{{index}}",
               "refId": "A",
@@ -560,7 +561,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Avg time waiting for db conn",
+          "title": "Avg reactor tick time",
           "tooltip": {
             "shared": true,
             "sort": 0,
@@ -576,12 +577,11 @@
           },
           "yaxes": [
             {
-              "decimals": null,
               "format": "s",
-              "label": "",
+              "label": null,
               "logBase": 1,
               "max": null,
-              "min": "0",
+              "min": null,
               "show": true
             },
             {
@@ -604,6 +604,7 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
+          "description": "Shows the time in which the given percentage of reactor ticks completed, over the sampled timespan",
           "fill": 1,
           "gridPos": {
             "h": 7,
@@ -611,7 +612,7 @@
             "x": 12,
             "y": 25
           },
-          "id": 49,
+          "id": 105,
           "legend": {
             "avg": false,
             "current": false,
@@ -629,33 +630,47 @@
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
-          "seriesOverrides": [
-            {
-              "alias": "/^up/",
-              "legend": false,
-              "yaxis": 2
-            }
-          ],
+          "seriesOverrides": [],
           "spaceLength": 10,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
-              "expr": "scrape_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+              "expr": "histogram_quantile(0.99, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 2,
-              "legendFormat": "{{job}}-{{index}}",
+              "legendFormat": "{{job}}-{{index}} 99%",
               "refId": "A",
               "step": 20
+            },
+            {
+              "expr": "histogram_quantile(0.95, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} 95%",
+              "refId": "B"
+            },
+            {
+              "expr": "histogram_quantile(0.90, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} 90%",
+              "refId": "C"
+            },
+            {
+              "expr": "",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "refId": "D"
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Prometheus scrape time",
+          "title": "Reactor tick quantiles",
           "tooltip": {
-            "shared": true,
+            "shared": false,
             "sort": 0,
             "value_type": "individual"
           },
@@ -673,16 +688,15 @@
               "label": null,
               "logBase": 1,
               "max": null,
-              "min": "0",
+              "min": null,
               "show": true
             },
             {
-              "decimals": 0,
-              "format": "none",
-              "label": "",
+              "format": "short",
+              "label": null,
               "logBase": 1,
-              "max": "0",
-              "min": "-1",
+              "max": null,
+              "min": null,
               "show": false
             }
           ],
@@ -697,14 +711,14 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
-          "fill": 1,
+          "fill": 0,
           "gridPos": {
             "h": 7,
             "w": 12,
             "x": 0,
             "y": 32
           },
-          "id": 50,
+          "id": 53,
           "legend": {
             "avg": false,
             "current": false,
@@ -728,19 +742,17 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "rate(python_twisted_reactor_tick_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(python_twisted_reactor_tick_time_count[$bucket_size])",
+              "expr": "min_over_time(up{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
               "format": "time_series",
-              "interval": "",
               "intervalFactor": 2,
               "legendFormat": "{{job}}-{{index}}",
-              "refId": "A",
-              "step": 20
+              "refId": "A"
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Avg reactor tick time",
+          "title": "Up",
           "tooltip": {
             "shared": true,
             "sort": 0,
@@ -756,7 +768,7 @@
           },
           "yaxes": [
             {
-              "format": "s",
+              "format": "short",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -769,7 +781,7 @@
               "logBase": 1,
               "max": null,
               "min": null,
-              "show": false
+              "show": true
             }
           ],
           "yaxis": {
@@ -783,26 +795,19 @@
           "dashLength": 10,
           "dashes": false,
           "datasource": "$datasource",
-          "editable": true,
-          "error": false,
           "fill": 1,
-          "grid": {},
           "gridPos": {
             "h": 7,
             "w": 12,
             "x": 12,
             "y": 32
           },
-          "id": 5,
+          "id": 49,
           "legend": {
-            "alignAsTable": false,
             "avg": false,
             "current": false,
-            "hideEmpty": false,
-            "hideZero": false,
             "max": false,
             "min": false,
-            "rightSide": false,
             "show": true,
             "total": false,
             "values": false
@@ -817,10 +822,9 @@
           "renderer": "flot",
           "seriesOverrides": [
             {
-              "alias": "/user/"
-            },
-            {
-              "alias": "/system/"
+              "alias": "/^up/",
+              "legend": false,
+              "yaxis": 2
             }
           ],
           "spaceLength": 10,
@@ -828,44 +832,19 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "rate(process_cpu_system_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "{{job}}-{{index}} system ",
-              "metric": "",
-              "refId": "B",
-              "step": 20
-            },
-            {
-              "expr": "rate(process_cpu_user_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "expr": "scrape_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
               "format": "time_series",
-              "hide": false,
               "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{job}}-{{index}} user",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}}",
               "refId": "A",
               "step": 20
             }
           ],
-          "thresholds": [
-            {
-              "colorMode": "custom",
-              "line": true,
-              "lineColor": "rgba(216, 200, 27, 0.27)",
-              "op": "gt",
-              "value": 0.5
-            },
-            {
-              "colorMode": "custom",
-              "line": true,
-              "lineColor": "rgba(234, 112, 112, 0.22)",
-              "op": "gt",
-              "value": 0.8
-            }
-          ],
+          "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "CPU",
+          "title": "Prometheus scrape time",
           "tooltip": {
             "shared": true,
             "sort": 0,
@@ -881,20 +860,21 @@
           },
           "yaxes": [
             {
-              "decimals": null,
-              "format": "percentunit",
-              "label": "",
+              "format": "s",
+              "label": null,
               "logBase": 1,
-              "max": "1.2",
-              "min": 0,
+              "max": null,
+              "min": "0",
               "show": true
             },
             {
-              "format": "short",
+              "decimals": 0,
+              "format": "none",
+              "label": "",
               "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
+              "max": "0",
+              "min": "-1",
+              "show": false
             }
           ],
           "yaxis": {
@@ -907,20 +887,27 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS}",
-          "fill": 0,
+          "datasource": "$datasource",
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {},
           "gridPos": {
             "h": 7,
             "w": 12,
             "x": 0,
             "y": 39
           },
-          "id": 53,
+          "id": 5,
           "legend": {
+            "alignAsTable": false,
             "avg": false,
             "current": false,
+            "hideEmpty": false,
+            "hideZero": false,
             "max": false,
             "min": false,
+            "rightSide": false,
             "show": true,
             "total": false,
             "values": false
@@ -933,23 +920,57 @@
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
-          "seriesOverrides": [],
+          "seriesOverrides": [
+            {
+              "alias": "/user/"
+            },
+            {
+              "alias": "/system/"
+            }
+          ],
           "spaceLength": 10,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
-              "expr": "min_over_time(up{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "expr": "rate(process_cpu_system_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
               "format": "time_series",
-              "intervalFactor": 2,
-              "legendFormat": "{{job}}-{{index}}",
-              "refId": "A"
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} system ",
+              "metric": "",
+              "refId": "B",
+              "step": 20
+            },
+            {
+              "expr": "rate(process_cpu_user_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} user",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [
+            {
+              "colorMode": "custom",
+              "line": true,
+              "lineColor": "rgba(216, 200, 27, 0.27)",
+              "op": "gt",
+              "value": 0.5
+            },
+            {
+              "colorMode": "custom",
+              "line": true,
+              "lineColor": "rgba(234, 112, 112, 0.22)",
+              "op": "gt",
+              "value": 0.8
             }
           ],
-          "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Up",
+          "title": "CPU",
           "tooltip": {
             "shared": true,
             "sort": 0,
@@ -965,16 +986,16 @@
           },
           "yaxes": [
             {
-              "format": "short",
-              "label": null,
+              "decimals": null,
+              "format": "percentunit",
+              "label": "",
               "logBase": 1,
-              "max": null,
-              "min": null,
+              "max": "1.2",
+              "min": 0,
               "show": true
             },
             {
               "format": "short",
-              "label": null,
               "logBase": 1,
               "max": null,
               "min": null,
@@ -1013,7 +1034,7 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 49
+            "y": 47
           },
           "id": 40,
           "legend": {
@@ -1098,7 +1119,7 @@
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 49
+            "y": 47
           },
           "id": 46,
           "legend": {
@@ -1187,7 +1208,7 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 56
+            "y": 54
           },
           "id": 44,
           "legend": {
@@ -1276,7 +1297,7 @@
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 56
+            "y": 54
           },
           "id": 45,
           "legend": {
@@ -1383,7 +1404,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 48
+            "y": 62
           },
           "id": 4,
           "legend": {
@@ -1490,7 +1511,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 48
+            "y": 62
           },
           "id": 32,
           "legend": {
@@ -1578,7 +1599,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 56
+            "y": 70
           },
           "id": 23,
           "legend": {
@@ -1688,7 +1709,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 56
+            "y": 70
           },
           "id": 52,
           "legend": {
@@ -1795,7 +1816,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 64
+            "y": 78
           },
           "id": 7,
           "legend": {
@@ -1886,7 +1907,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 64
+            "y": 78
           },
           "id": 47,
           "legend": {
@@ -1969,13 +1990,13 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS}",
+          "datasource": "$datasource",
           "fill": 1,
           "gridPos": {
             "h": 9,
             "w": 12,
             "x": 0,
-            "y": 72
+            "y": 86
           },
           "id": 103,
           "legend": {
@@ -2069,13 +2090,13 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS}",
+          "datasource": "$datasource",
           "fill": 1,
           "gridPos": {
             "h": 9,
             "w": 12,
             "x": 0,
-            "y": 23
+            "y": 49
           },
           "id": 99,
           "legend": {
@@ -2154,13 +2175,13 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS}",
+          "datasource": "$datasource",
           "fill": 1,
           "gridPos": {
             "h": 9,
             "w": 12,
             "x": 12,
-            "y": 23
+            "y": 49
           },
           "id": 101,
           "legend": {
@@ -2186,17 +2207,24 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "rate(synapse_background_process_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "expr": "rate(synapse_background_process_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) +  rate(synapse_background_process_db_sched_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
               "format": "time_series",
+              "hide": false,
               "intervalFactor": 1,
               "legendFormat": "{{job}}-{{index}} {{name}}",
               "refId": "A"
+            },
+            {
+              "expr": "",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "refId": "B"
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "DB usage by background jobs",
+          "title": "DB usage by background jobs (including scheduling time)",
           "tooltip": {
             "shared": true,
             "sort": 0,
@@ -2252,13 +2280,13 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS}",
+          "datasource": "$datasource",
           "fill": 1,
           "gridPos": {
             "h": 9,
             "w": 12,
             "x": 0,
-            "y": 25
+            "y": 64
           },
           "id": 79,
           "legend": {
@@ -2336,13 +2364,13 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS}",
+          "datasource": "$datasource",
           "fill": 1,
           "gridPos": {
             "h": 9,
             "w": 12,
             "x": 12,
-            "y": 25
+            "y": 64
           },
           "id": 83,
           "legend": {
@@ -2447,7 +2475,7 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 23
+            "y": 65
           },
           "id": 51,
           "legend": {
@@ -2545,6 +2573,194 @@
       },
       "id": 58,
       "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "fill": 1,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 24
+          },
+          "id": 48,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_storage_schedule_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_schedule_time_count[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Avg time waiting for db conn",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "s",
+              "label": "",
+              "logBase": 1,
+              "max": null,
+              "min": "0",
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": false
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "description": "Shows the time in which the given percentage of database queries were scheduled, over the sampled timespan",
+          "fill": 1,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 24
+          },
+          "id": 104,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "histogram_quantile(0.99, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 1,
+              "legendFormat": "{{job}} {{index}} 99%",
+              "refId": "A",
+              "step": 20
+            },
+            {
+              "expr": "histogram_quantile(0.95, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}} {{index}} 95%",
+              "refId": "B"
+            },
+            {
+              "expr": "histogram_quantile(0.90, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}} {{index}} 90%",
+              "refId": "C"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Db scheduling time quantiles",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "s",
+              "label": "",
+              "logBase": 1,
+              "max": null,
+              "min": "0",
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": false
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
         {
           "aliasColors": {},
           "bars": false,
@@ -2559,7 +2775,7 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 25
+            "y": 31
           },
           "id": 10,
           "legend": {
@@ -2648,7 +2864,7 @@
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 25
+            "y": 31
           },
           "id": 11,
           "legend": {
@@ -2672,11 +2888,11 @@
           "renderer": "flot",
           "seriesOverrides": [],
           "spaceLength": 10,
-          "stack": false,
+          "stack": true,
           "steppedLine": true,
           "targets": [
             {
-              "expr": "topk(5, rate(synapse_storage_transaction_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+              "expr": "rate(synapse_storage_transaction_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
               "format": "time_series",
               "instant": false,
               "interval": "",
@@ -2753,7 +2969,7 @@
             "h": 13,
             "w": 12,
             "x": 0,
-            "y": 17
+            "y": 60
           },
           "id": 12,
           "legend": {
@@ -2841,7 +3057,7 @@
             "h": 13,
             "w": 12,
             "x": 12,
-            "y": 17
+            "y": 60
           },
           "id": 26,
           "legend": {
@@ -2929,7 +3145,7 @@
             "h": 13,
             "w": 12,
             "x": 0,
-            "y": 30
+            "y": 73
           },
           "id": 13,
           "legend": {
@@ -3017,7 +3233,7 @@
             "h": 13,
             "w": 12,
             "x": 12,
-            "y": 30
+            "y": 73
           },
           "id": 27,
           "legend": {
@@ -3105,7 +3321,7 @@
             "h": 13,
             "w": 12,
             "x": 0,
-            "y": 43
+            "y": 86
           },
           "id": 28,
           "legend": {
@@ -3192,7 +3408,7 @@
             "h": 13,
             "w": 12,
             "x": 12,
-            "y": 43
+            "y": 86
           },
           "id": 25,
           "legend": {
@@ -3295,7 +3511,7 @@
             "h": 10,
             "w": 12,
             "x": 0,
-            "y": 55
+            "y": 68
           },
           "id": 1,
           "legend": {
@@ -3387,7 +3603,7 @@
             "h": 10,
             "w": 12,
             "x": 12,
-            "y": 55
+            "y": 68
           },
           "id": 8,
           "legend": {
@@ -3477,7 +3693,7 @@
             "h": 10,
             "w": 12,
             "x": 0,
-            "y": 65
+            "y": 78
           },
           "id": 38,
           "legend": {
@@ -3563,7 +3779,7 @@
             "h": 10,
             "w": 12,
             "x": 12,
-            "y": 65
+            "y": 78
           },
           "id": 39,
           "legend": {
@@ -3643,13 +3859,13 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS}",
+          "datasource": "$datasource",
           "fill": 1,
           "gridPos": {
             "h": 9,
             "w": 12,
             "x": 0,
-            "y": 75
+            "y": 88
           },
           "id": 65,
           "legend": {
@@ -3745,13 +3961,13 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS}",
+          "datasource": "$datasource",
           "fill": 1,
           "gridPos": {
             "h": 9,
             "w": 12,
             "x": 0,
-            "y": 90
+            "y": 27
           },
           "id": 91,
           "legend": {
@@ -3841,7 +4057,7 @@
             "h": 9,
             "w": 12,
             "x": 12,
-            "y": 90
+            "y": 27
           },
           "id": 21,
           "legend": {
@@ -3920,13 +4136,13 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS}",
+          "datasource": "$datasource",
           "fill": 1,
           "gridPos": {
             "h": 9,
             "w": 12,
             "x": 0,
-            "y": 99
+            "y": 36
           },
           "id": 89,
           "legend": {
@@ -4006,13 +4222,13 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS}",
+          "datasource": "$datasource",
           "fill": 1,
           "gridPos": {
             "h": 9,
             "w": 12,
             "x": 12,
-            "y": 99
+            "y": 36
           },
           "id": 93,
           "legend": {
@@ -4027,7 +4243,7 @@
           "lines": true,
           "linewidth": 1,
           "links": [],
-          "nullPointMode": "null",
+          "nullPointMode": "connected",
           "percentage": false,
           "pointradius": 5,
           "points": false,
@@ -4090,13 +4306,13 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS}",
+          "datasource": "$datasource",
           "fill": 1,
           "gridPos": {
             "h": 9,
             "w": 12,
             "x": 0,
-            "y": 108
+            "y": 45
           },
           "id": 95,
           "legend": {
@@ -4189,7 +4405,7 @@
             "h": 9,
             "w": 12,
             "x": 12,
-            "y": 108
+            "y": 45
           },
           "heatmap": {},
           "highlightCards": true,
@@ -4251,13 +4467,13 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS}",
+          "datasource": "$datasource",
           "fill": 1,
           "gridPos": {
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 19
+            "y": 97
           },
           "id": 2,
           "legend": {
@@ -4357,20 +4573,24 @@
               "min": null,
               "show": true
             }
-          ]
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
         },
         {
           "aliasColors": {},
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS}",
+          "datasource": "$datasource",
           "fill": 1,
           "gridPos": {
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 19
+            "y": 97
           },
           "id": 41,
           "legend": {
@@ -4439,20 +4659,24 @@
               "min": null,
               "show": true
             }
-          ]
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
         },
         {
           "aliasColors": {},
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS}",
+          "datasource": "$datasource",
           "fill": 1,
           "gridPos": {
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 26
+            "y": 104
           },
           "id": 42,
           "legend": {
@@ -4520,20 +4744,24 @@
               "min": null,
               "show": true
             }
-          ]
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
         },
         {
           "aliasColors": {},
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS}",
+          "datasource": "$datasource",
           "fill": 1,
           "gridPos": {
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 26
+            "y": 104
           },
           "id": 43,
           "legend": {
@@ -4601,7 +4829,11 @@
               "min": null,
               "show": true
             }
-          ]
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
         }
       ],
       "repeat": null,
@@ -4623,7 +4855,7 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS}",
+          "datasource": "$datasource",
           "fill": 1,
           "gridPos": {
             "h": 9,
@@ -4644,7 +4876,7 @@
           "lines": true,
           "linewidth": 1,
           "links": [],
-          "nullPointMode": "null",
+          "nullPointMode": "connected",
           "percentage": false,
           "pointradius": 5,
           "points": false,
@@ -4708,7 +4940,7 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS}",
+          "datasource": "$datasource",
           "fill": 1,
           "gridPos": {
             "h": 9,
@@ -4729,7 +4961,7 @@
           "lines": true,
           "linewidth": 1,
           "links": [],
-          "nullPointMode": "null",
+          "nullPointMode": "connected",
           "percentage": false,
           "pointradius": 5,
           "points": false,
@@ -4856,9 +5088,19 @@
             "selected": false,
             "text": "5m",
             "value": "5m"
+          },
+          {
+            "selected": false,
+            "text": "10m",
+            "value": "10m"
+          },
+          {
+            "selected": false,
+            "text": "15m",
+            "value": "15m"
           }
         ],
-        "query": "30s,1m,2m,5m",
+        "query": "30s,1m,2m,5m,10m,15m",
         "refresh": 2,
         "type": "interval"
       },
@@ -4872,7 +5114,7 @@
         "multi": false,
         "name": "instance",
         "options": [],
-        "query": "label_values(process_cpu_user_seconds_total{job=~\"synapse.*\"}, instance)",
+        "query": "label_values(synapse_util_metrics_block_ru_utime_seconds, instance)",
         "refresh": 2,
         "regex": "",
         "sort": 0,
@@ -4895,7 +5137,7 @@
         "multiFormat": "regex values",
         "name": "job",
         "options": [],
-        "query": "label_values(process_cpu_user_seconds_total{job=~\"synapse.*\"}, job)",
+        "query": "label_values(synapse_util_metrics_block_ru_utime_seconds, job)",
         "refresh": 2,
         "refresh_on_load": false,
         "regex": "",
@@ -4919,7 +5161,7 @@
         "multiFormat": "regex values",
         "name": "index",
         "options": [],
-        "query": "label_values(process_cpu_user_seconds_total{job=~\"synapse.*\"}, index)",
+        "query": "label_values(synapse_util_metrics_block_ru_utime_seconds, index)",
         "refresh": 2,
         "refresh_on_load": false,
         "regex": "",
@@ -4965,5 +5207,5 @@
   "timezone": "",
   "title": "Synapse",
   "uid": "000000012",
-  "version": 127
+  "version": 3
 }
\ No newline at end of file

From e7fa032126855f783b9246e2f03d8b62b58f5521 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Christian=20Gr=C3=BCnhage?=
 <jan.christian@gruenhage.xyz>
Date: Tue, 25 Sep 2018 14:33:19 +0200
Subject: [PATCH 719/830] Update .dockerignore

---
 .dockerignore | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.dockerignore b/.dockerignore
index 81e8f6d50a13..0180602e56fc 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -2,7 +2,6 @@ Dockerfile
 .travis.yml
 .gitignore
 demo/etc
-synctl
 tox.ini
 .git/*
 .tox/*

From df55a943cadaba60b9171f2e70f62dc4b38fbd78 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Christian=20Gr=C3=BCnhage?=
 <jan.christian@gruenhage.xyz>
Date: Tue, 25 Sep 2018 14:33:38 +0200
Subject: [PATCH 720/830] Update Dockerfile

---
 docker/Dockerfile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index ad0ae0b734eb..1d00defc2dcd 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -13,7 +13,6 @@ RUN apk add --no-cache --virtual .build_deps \
         postgresql-dev \
         zlib-dev \
  && cd /synapse \
- && sed -i 's/\["synctl"\] + //' setup.py \
  && apk add --no-cache --virtual .runtime_deps \
  	libffi \
         libjpeg-turbo \

From 3baf6e1667f0c257f288a28bd542a93c2445fd30 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 21 Sep 2018 16:25:42 +0100
Subject: [PATCH 721/830] Fix ExpiringCache.__len__ to be accurate

It used to try and produce an estimate, which was sometimes negative.
This caused metrics to be sad, so lets always just calculate it from
scratch.

(This appears to have been a longstanding bug, but one which has been made more
of a problem by #3932 and #3933).

(This was originally done by Erik as part of #3933. I'm cherry-picking it
because really it's a fix in its own right)
---
 synapse/util/caches/expiringcache.py | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/synapse/util/caches/expiringcache.py b/synapse/util/caches/expiringcache.py
index 921a9c5b29a5..9af4ec4aa80f 100644
--- a/synapse/util/caches/expiringcache.py
+++ b/synapse/util/caches/expiringcache.py
@@ -16,6 +16,8 @@
 import logging
 from collections import OrderedDict
 
+from six import itervalues
+
 from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.util.caches import register_cache
 
@@ -54,8 +56,6 @@ def __init__(self, cache_name, clock, max_len=0, expiry_ms=0,
 
         self.iterable = iterable
 
-        self._size_estimate = 0
-
         self.metrics = register_cache("expiring", cache_name, self)
 
         if not self._expiry_ms:
@@ -74,16 +74,11 @@ def __setitem__(self, key, value):
         now = self._clock.time_msec()
         self._cache[key] = _CacheEntry(now, value)
 
-        if self.iterable:
-            self._size_estimate += len(value)
-
         # Evict if there are now too many items
         while self._max_len and len(self) > self._max_len:
             _key, value = self._cache.popitem(last=False)
             if self.iterable:
-                removed_len = len(value.value)
-                self.metrics.inc_evictions(removed_len)
-                self._size_estimate -= removed_len
+                self.metrics.inc_evictions(len(value.value))
             else:
                 self.metrics.inc_evictions()
 
@@ -134,7 +129,9 @@ def _prune_cache(self):
         for k in keys_to_delete:
             value = self._cache.pop(k)
             if self.iterable:
-                self._size_estimate -= len(value.value)
+                self.metrics.inc_evictions(len(value.value))
+            else:
+                self.metrics.inc_evictions()
 
         logger.debug(
             "[%s] _prune_cache before: %d, after len: %d",
@@ -143,7 +140,7 @@ def _prune_cache(self):
 
     def __len__(self):
         if self.iterable:
-            return self._size_estimate
+            return sum(len(entry.value) for entry in itervalues(self._cache))
         else:
             return len(self._cache)
 

From 8ac9fa73752b35e0d4feaeef33c5dcca018fca07 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 26 Sep 2018 12:35:10 +0100
Subject: [PATCH 722/830] changelog

---
 changelog.d/3956.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3956.bugfix

diff --git a/changelog.d/3956.bugfix b/changelog.d/3956.bugfix
new file mode 100644
index 000000000000..b0828c9fc632
--- /dev/null
+++ b/changelog.d/3956.bugfix
@@ -0,0 +1 @@
+Fix exceptions from metrics handler
\ No newline at end of file

From 7ee94fc1baf6c4e2970232383d76277d8378712b Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 26 Sep 2018 12:40:20 +0100
Subject: [PATCH 723/830] Log which cache is throwing exceptions

---
 synapse/util/caches/__init__.py | 27 +++++++++++++++++----------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/synapse/util/caches/__init__.py b/synapse/util/caches/__init__.py
index 7b065b195ed3..f37d5bec0864 100644
--- a/synapse/util/caches/__init__.py
+++ b/synapse/util/caches/__init__.py
@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import logging
 import os
 
 import six
@@ -20,6 +21,8 @@
 
 from prometheus_client.core import REGISTRY, Gauge, GaugeMetricFamily
 
+logger = logging.getLogger(__name__)
+
 CACHE_SIZE_FACTOR = float(os.environ.get("SYNAPSE_CACHE_FACTOR", 0.5))
 
 
@@ -76,16 +79,20 @@ def describe(self):
             return []
 
         def collect(self):
-            if cache_type == "response_cache":
-                response_cache_size.labels(cache_name).set(len(cache))
-                response_cache_hits.labels(cache_name).set(self.hits)
-                response_cache_evicted.labels(cache_name).set(self.evicted_size)
-                response_cache_total.labels(cache_name).set(self.hits + self.misses)
-            else:
-                cache_size.labels(cache_name).set(len(cache))
-                cache_hits.labels(cache_name).set(self.hits)
-                cache_evicted.labels(cache_name).set(self.evicted_size)
-                cache_total.labels(cache_name).set(self.hits + self.misses)
+            try:
+                if cache_type == "response_cache":
+                    response_cache_size.labels(cache_name).set(len(cache))
+                    response_cache_hits.labels(cache_name).set(self.hits)
+                    response_cache_evicted.labels(cache_name).set(self.evicted_size)
+                    response_cache_total.labels(cache_name).set(self.hits + self.misses)
+                else:
+                    cache_size.labels(cache_name).set(len(cache))
+                    cache_hits.labels(cache_name).set(self.hits)
+                    cache_evicted.labels(cache_name).set(self.evicted_size)
+                    cache_total.labels(cache_name).set(self.hits + self.misses)
+            except Exception as e:
+                logger.warn("Error calculating metrics for %s: %s", cache_name, e)
+                raise
 
             yield GaugeMetricFamily("__unused", "")
 

From 8834396b8ac96f76263d80dea9b559e2597215a6 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 21 Feb 2018 21:09:33 +0000
Subject: [PATCH 724/830] Actuall set cache factors in workers

---
 synapse/app/synctl.py | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/synapse/app/synctl.py b/synapse/app/synctl.py
index d658f967ba87..356e5cb6a725 100755
--- a/synapse/app/synctl.py
+++ b/synapse/app/synctl.py
@@ -111,7 +111,7 @@ def stop(pidfile, app):
 
 
 Worker = collections.namedtuple("Worker", [
-    "app", "configfile", "pidfile", "cache_factor"
+    "app", "configfile", "pidfile", "cache_factor", "cache_factors",
 ])
 
 
@@ -218,6 +218,10 @@ def main():
                 or pidfile
             )
             worker_cache_factor = worker_config.get("synctl_cache_factor") or cache_factor
+            worker_cache_factors = (
+                worker_config.get("synctl_cache_factors")
+                or cache_factors
+            )
             daemonize = worker_config.get("daemonize") or config.get("daemonize")
             assert daemonize, "Main process must have daemonize set to true"
 
@@ -233,8 +237,10 @@ def main():
             assert worker_daemonize, "In config %r: expected '%s' to be True" % (
                 worker_configfile, "worker_daemonize")
             worker_cache_factor = worker_config.get("synctl_cache_factor")
+            worker_cache_factors = worker_config.get("synctl_cache_factors", {})
         workers.append(Worker(
             worker_app, worker_configfile, worker_pidfile, worker_cache_factor,
+            worker_cache_factors,
         ))
 
     action = options.action
@@ -269,15 +275,19 @@ def main():
             start(configfile)
 
         for worker in workers:
+            env = os.environ.copy()
+
             if worker.cache_factor:
                 os.environ["SYNAPSE_CACHE_FACTOR"] = str(worker.cache_factor)
 
+            for cache_name, factor in worker.cache_factors.iteritems():
+                os.environ["SYNAPSE_CACHE_FACTOR_" + cache_name.upper()] = str(factor)
+
             start_worker(worker.app, configfile, worker.configfile)
 
-            if cache_factor:
-                os.environ["SYNAPSE_CACHE_FACTOR"] = str(cache_factor)
-            else:
-                os.environ.pop("SYNAPSE_CACHE_FACTOR", None)
+            # Reset env back to the original
+            os.environ.clear()
+            os.environ.update(env)
 
 
 if __name__ == "__main__":

From 8afddf7afec8c6ee790a10e24fe817d663bfb4ee Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 25 Sep 2018 16:34:40 +0100
Subject: [PATCH 725/830] Fix error handling for missing auth_event

When we were authorizing an event, if there was no `m.room.create` in its
auth_events, we would raise a SynapseError with a cryptic message, which then
meant that we would bail out of processing any incoming events, rather than
storing a rejection for the faulty event and moving on.

We should treat the absent event the same as any other auth failure, by
raising an AuthError, so that the event is marked as rejected.
---
 synapse/event_auth.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/synapse/event_auth.py b/synapse/event_auth.py
index 6baeccca38ae..02fa46ef7f82 100644
--- a/synapse/event_auth.py
+++ b/synapse/event_auth.py
@@ -98,9 +98,9 @@ def check(event, auth_events, do_sig_check=True, do_size_check=True):
     creation_event = auth_events.get((EventTypes.Create, ""), None)
 
     if not creation_event:
-        raise SynapseError(
+        raise AuthError(
             403,
-            "Room %r does not exist" % (event.room_id,)
+            "No create event in auth events",
         )
 
     creating_domain = get_domain_from_id(event.room_id)

From a5e70b31a12c3c75d5e71a98f8ad60621086d47b Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 26 Sep 2018 14:41:14 +0100
Subject: [PATCH 726/830] changelog

---
 changelog.d/3960.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3960.bugfix

diff --git a/changelog.d/3960.bugfix b/changelog.d/3960.bugfix
new file mode 100644
index 000000000000..a01dc60df6e5
--- /dev/null
+++ b/changelog.d/3960.bugfix
@@ -0,0 +1 @@
+Fix error message for events with m.room.create missing from auth_events
\ No newline at end of file

From d4e0861ff94a809ad36c6b951c1e903457c44def Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Thu, 27 Sep 2018 00:23:21 +1000
Subject: [PATCH 727/830] Reduce the load on our CI (#3957)

* changelog

* reduce circleci config

* plus a handy script

* fix regex
---
 .circleci/config.yml              | 28 ++++++++++++++++++++--------
 changelog.d/3957.misc             |  1 +
 scripts-dev/next_github_number.sh |  9 +++++++++
 3 files changed, 30 insertions(+), 8 deletions(-)
 create mode 100644 changelog.d/3957.misc
 create mode 100755 scripts-dev/next_github_number.sh

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 605430fb3f09..521aca22ef97 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -99,23 +99,35 @@ workflows:
   version: 2
   build:
     jobs:
-      - sytestpy2
-      - sytestpy2postgres
-      - sytestpy3
-      - sytestpy3postgres
+      - sytestpy2:
+          filters:
+            branches:
+              only: /develop|master|release-.*/
+      - sytestpy2postgres:
+          filters:
+            branches:
+              only: /develop|master|release-.*/
+      - sytestpy3:
+          filters:
+            branches:
+              only: /develop|master|release-.*/
+      - sytestpy3postgres:
+          filters:
+            branches:
+              only: /develop|master|release-.*/
       - sytestpy2merged:
           filters:
             branches:
-              ignore: /develop|master/
+              ignore: /develop|master|release-.*/
       - sytestpy2postgresmerged:
           filters:
             branches:
-              ignore: /develop|master/
+              ignore: /develop|master|release-.*/
       - sytestpy3merged:
           filters:
             branches:
-              ignore: /develop|master/
+              ignore: /develop|master|release-.*/
       - sytestpy3postgresmerged:
           filters:
             branches:
-              ignore: /develop|master/
+              ignore: /develop|master|release-.*/
diff --git a/changelog.d/3957.misc b/changelog.d/3957.misc
new file mode 100644
index 000000000000..69d647f119cf
--- /dev/null
+++ b/changelog.d/3957.misc
@@ -0,0 +1 @@
+CircleCI now only runs merged jobs on PRs, and commit jobs on develop, master, and release branches.
diff --git a/scripts-dev/next_github_number.sh b/scripts-dev/next_github_number.sh
new file mode 100755
index 000000000000..376280025a39
--- /dev/null
+++ b/scripts-dev/next_github_number.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -e
+
+# Fetch the current GitHub issue number, add one to it -- presto! The likely
+# next PR number.
+CURRENT_NUMBER=`curl -s "https://api.github.com/repos/matrix-org/synapse/issues?state=all&per_page=1" | jq -r ".[0].number"`
+CURRENT_NUMBER=$((CURRENT_NUMBER+1))
+echo $CURRENT_NUMBER
\ No newline at end of file

From 28781b65e7cb8b0d1fc409d0d78253662b49fe06 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 26 Sep 2018 16:16:41 +0100
Subject: [PATCH 728/830] fix #3854

---
 synapse/storage/monthly_active_users.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py
index 59580949f190..0fe8c8e24c2e 100644
--- a/synapse/storage/monthly_active_users.py
+++ b/synapse/storage/monthly_active_users.py
@@ -172,6 +172,10 @@ def upsert_monthly_active_user(self, user_id):
             Deferred[bool]: True if a new entry was created, False if an
                 existing one was updated.
         """
+        # Am consciously deciding to lock the table on the basis that is ought
+        # never be a big table and alternative approaches (batching multiple
+        # upserts into a single txn) introduced a lot of extra complexity.
+        # See https://github.com/matrix-org/synapse/issues/3854 for more
         is_insert = yield self._simple_upsert(
             desc="upsert_monthly_active_user",
             table="monthly_active_users",
@@ -181,7 +185,6 @@ def upsert_monthly_active_user(self, user_id):
             values={
                 "timestamp": int(self._clock.time_msec()),
             },
-            lock=False,
         )
         if is_insert:
             self.user_last_seen_monthly_active.invalidate((user_id,))

From d514608b5cc80cfe1f93a9953eb8264162c4a47b Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@matrix.org>
Date: Wed, 26 Sep 2018 16:19:53 +0100
Subject: [PATCH 729/830] towncrier

---
 changelog.d/3961.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3961.bugfix

diff --git a/changelog.d/3961.bugfix b/changelog.d/3961.bugfix
new file mode 100644
index 000000000000..e46b5834aad0
--- /dev/null
+++ b/changelog.d/3961.bugfix
@@ -0,0 +1 @@
+Fix errors due to concurrent monthly_active_user upserts 

From 607eec0456dde3b7e5b33c3e9297af234659caae Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 25 Sep 2018 16:53:55 +0100
Subject: [PATCH 730/830] fix docstring for FederationClient.get_state_for_room

trivial fixes for docstring
---
 synapse/federation/federation_client.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index 5a92428f56c2..8bf1ad0c1fc5 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -289,8 +289,7 @@ def get_pdu(self, destinations, event_id, outlier=False, timeout=None):
     @defer.inlineCallbacks
     @log_function
     def get_state_for_room(self, destination, room_id, event_id):
-        """Requests all of the `current` state PDUs for a given room from
-        a remote home server.
+        """Requests all of the room state at a given event from a remote home server.
 
         Args:
             destination (str): The remote homeserver to query for the state.
@@ -298,9 +297,10 @@ def get_state_for_room(self, destination, room_id, event_id):
             event_id (str): The id of the event we want the state at.
 
         Returns:
-            Deferred: Results in a list of PDUs.
+            Deferred[Tuple[List[EventBase], List[EventBase]]]:
+                A list of events in the state, and a list of events in the auth chain
+                for the given event.
         """
-
         try:
             # First we try and ask for just the IDs, as thats far quicker if
             # we have most of the state and auth_chain already.

From ab59f3d8dab9b0a1c25c8f136ced9f28861a86ec Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 26 Sep 2018 16:51:28 +0100
Subject: [PATCH 731/830] changelog

---
 changelog.d/3963.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3963.misc

diff --git a/changelog.d/3963.misc b/changelog.d/3963.misc
new file mode 100644
index 000000000000..f1e0eaf18e48
--- /dev/null
+++ b/changelog.d/3963.misc
@@ -0,0 +1 @@
+fix docstring for FederationClient.get_state_for_room

From 9453c65948bce69f259c1ba1005565d86b1d8730 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 25 Sep 2018 17:38:54 +0100
Subject: [PATCH 732/830] remove spurious federation checks on localhost

There's really no point in checking for destinations called "localhost" because
there is nothing stopping people creating other DNS entries which point to
127.0.0.1. The right fix for this is
https://github.com/matrix-org/synapse/issues/3953.

Blocking localhost, on the other hand, means that you get a surprise when
trying to connect a test server on localhost to an existing server (with a
'normal' server_name).
---
 synapse/federation/transaction_queue.py | 37 ++++---------------------
 1 file changed, 6 insertions(+), 31 deletions(-)

diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py
index 8cbf8c4f7fbf..ae47aaae0b10 100644
--- a/synapse/federation/transaction_queue.py
+++ b/synapse/federation/transaction_queue.py
@@ -137,26 +137,6 @@ def __init__(self, hs):
 
         self._processing_pending_presence = False
 
-    def can_send_to(self, destination):
-        """Can we send messages to the given server?
-
-        We can't send messages to ourselves. If we are running on localhost
-        then we can only federation with other servers running on localhost.
-        Otherwise we only federate with servers on a public domain.
-
-        Args:
-            destination(str): The server we are possibly trying to send to.
-        Returns:
-            bool: True if we can send to the server.
-        """
-
-        if destination == self.server_name:
-            return False
-        if self.server_name.startswith("localhost"):
-            return destination.startswith("localhost")
-        else:
-            return not destination.startswith("localhost")
-
     def notify_new_events(self, current_id):
         """This gets called when we have some new events we might want to
         send out to other servers.
@@ -279,10 +259,7 @@ def _send_pdu(self, pdu, destinations):
         self._order += 1
 
         destinations = set(destinations)
-        destinations = set(
-            dest for dest in destinations if self.can_send_to(dest)
-        )
-
+        destinations.discard(self.server_name)
         logger.debug("Sending to: %s", str(destinations))
 
         if not destinations:
@@ -358,7 +335,7 @@ def _process_presence_inner(self, states):
 
         for destinations, states in hosts_and_states:
             for destination in destinations:
-                if not self.can_send_to(destination):
+                if destination == self.server_name:
                     continue
 
                 self.pending_presence_by_dest.setdefault(
@@ -377,7 +354,8 @@ def send_edu(self, destination, edu_type, content, key=None):
             content=content,
         )
 
-        if not self.can_send_to(destination):
+        if not destination == self.server_name:
+            logger.info("Not sending EDU to ourselves")
             return
 
         sent_edus_counter.inc()
@@ -392,11 +370,8 @@ def send_edu(self, destination, edu_type, content, key=None):
         self._attempt_new_transaction(destination)
 
     def send_device_messages(self, destination):
-        if destination == self.server_name or destination == "localhost":
-            return
-
-        if not self.can_send_to(destination):
-            return
+        if destination == self.server_name:
+            logger.info("Not sending device update to ourselves")
 
         self._attempt_new_transaction(destination)
 

From 0c4a99ea2d6046c815c8040d8d060795ab787b5f Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 26 Sep 2018 16:54:54 +0100
Subject: [PATCH 733/830] changelog

---
 changelog.d/3964.feature | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3964.feature

diff --git a/changelog.d/3964.feature b/changelog.d/3964.feature
new file mode 100644
index 000000000000..599222eb589e
--- /dev/null
+++ b/changelog.d/3964.feature
@@ -0,0 +1 @@
+Remove spurious check which made 'localhost' servers not work

From a87d419a85e1c966da5bb4fc08f3309eb78ecb7a Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 26 Sep 2018 07:48:35 +0100
Subject: [PATCH 734/830] Run notify_app_services as a bg process

This ensures that its resource usage metrics get recorded somewhere rather than
getting lost.

(It also fixes an error when called from a nested logging context which
completes before the bg process)
---
 synapse/notifier.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/synapse/notifier.py b/synapse/notifier.py
index f1d92c1395d2..340b16ce25ae 100644
--- a/synapse/notifier.py
+++ b/synapse/notifier.py
@@ -24,9 +24,10 @@
 from synapse.api.errors import AuthError
 from synapse.handlers.presence import format_user_presence_state
 from synapse.metrics import LaterGauge
+from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.types import StreamToken
 from synapse.util.async_helpers import ObservableDeferred, timeout_deferred
-from synapse.util.logcontext import PreserveLoggingContext, run_in_background
+from synapse.util.logcontext import PreserveLoggingContext
 from synapse.util.logutils import log_function
 from synapse.util.metrics import Measure
 from synapse.visibility import filter_events_for_client
@@ -248,7 +249,10 @@ def _notify_pending_new_room_events(self, max_room_stream_id):
     def _on_new_room_event(self, event, room_stream_id, extra_users=[]):
         """Notify any user streams that are interested in this room event"""
         # poke any interested application service.
-        run_in_background(self._notify_app_services, room_stream_id)
+        run_as_background_process(
+            "notify_app_services",
+            self._notify_app_services, room_stream_id,
+        )
 
         if self.federation_sender:
             self.federation_sender.notify_new_events(room_stream_id)

From 3c37c7e45b1befc6728fc0b26724bdea2e0845bd Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 26 Sep 2018 17:01:30 +0100
Subject: [PATCH 735/830] changelog

---
 changelog.d/3965.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3965.misc

diff --git a/changelog.d/3965.misc b/changelog.d/3965.misc
new file mode 100644
index 000000000000..e7e4a9c5a8ef
--- /dev/null
+++ b/changelog.d/3965.misc
@@ -0,0 +1 @@
+Run notify_app_services as a bg process

From e70b4ce06920102e2460dfb65bc357e2d7e8b794 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 26 Sep 2018 07:56:06 +0100
Subject: [PATCH 736/830] Logging improvements

Some logging tweaks to help with debugging incoming federation transactions
---
 changelog.d/3966.misc          |  1 +
 synapse/handlers/federation.py |  4 ++++
 synapse/state/v1.py            | 14 ++++++++++++--
 3 files changed, 17 insertions(+), 2 deletions(-)
 create mode 100644 changelog.d/3966.misc

diff --git a/changelog.d/3966.misc b/changelog.d/3966.misc
new file mode 100644
index 000000000000..1e3c8e1706ca
--- /dev/null
+++ b/changelog.d/3966.misc
@@ -0,0 +1 @@
+Improve the logging when handling a federation transaction
\ No newline at end of file
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 2ccdc3bfa71a..a70ae8c830de 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -572,6 +572,10 @@ def _process_received_pdu(self, origin, event, state, auth_chain):
                     })
                     seen_ids.add(e.event_id)
 
+                logger.info(
+                    "[%s %s] persisting newly-received auth/state events %s",
+                    room_id, event_id, [e["event"].event_id for e in event_infos]
+                )
                 yield self._handle_new_events(origin, event_infos)
 
             try:
diff --git a/synapse/state/v1.py b/synapse/state/v1.py
index c95477d31890..7a7157b352d4 100644
--- a/synapse/state/v1.py
+++ b/synapse/state/v1.py
@@ -65,10 +65,15 @@ def resolve_events_with_factory(state_sets, event_map, state_map_factory):
         for event_ids in itervalues(conflicted_state)
         for event_id in event_ids
     )
+    needed_event_count = len(needed_events)
     if event_map is not None:
         needed_events -= set(iterkeys(event_map))
 
-    logger.info("Asking for %d conflicted events", len(needed_events))
+    logger.info(
+        "Asking for %d/%d conflicted events",
+        len(needed_events),
+        needed_event_count,
+    )
 
     # dict[str, FrozenEvent]: a map from state event id to event. Only includes
     # the state events which are in conflict (and those in event_map)
@@ -85,11 +90,16 @@ def resolve_events_with_factory(state_sets, event_map, state_map_factory):
     )
 
     new_needed_events = set(itervalues(auth_events))
+    new_needed_event_count = len(new_needed_events)
     new_needed_events -= needed_events
     if event_map is not None:
         new_needed_events -= set(iterkeys(event_map))
 
-    logger.info("Asking for %d auth events", len(new_needed_events))
+    logger.info(
+        "Asking for %d/%d auth events",
+        len(new_needed_events),
+        new_needed_event_count,
+    )
 
     state_map_new = yield state_map_factory(new_needed_events)
     state_map.update(state_map_new)

From ae6ad4cf4190dbd2dadd72fc8131e4c139f8eb4a Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Thu, 27 Sep 2018 11:22:25 +0100
Subject: [PATCH 737/830] docstrings and unittests for storage.state (#3958)

I spent ages trying to figure out how I was going mad...
---
 changelog.d/3958.misc       |  1 +
 synapse/storage/state.py    | 30 ++++++++++++++++++++--------
 tests/storage/test_state.py | 39 +++++++++++++++++++++++++++++++++++++
 3 files changed, 62 insertions(+), 8 deletions(-)
 create mode 100644 changelog.d/3958.misc

diff --git a/changelog.d/3958.misc b/changelog.d/3958.misc
new file mode 100644
index 000000000000..5931d06dcff0
--- /dev/null
+++ b/changelog.d/3958.misc
@@ -0,0 +1 @@
+Fix docstrings and add tests for state store methods
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index 4b971efdbaf8..3f4cbd61c453 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -255,7 +255,17 @@ def _get_state_group_delta_txn(txn):
         )
 
     @defer.inlineCallbacks
-    def get_state_groups_ids(self, room_id, event_ids):
+    def get_state_groups_ids(self, _room_id, event_ids):
+        """Get the event IDs of all the state for the state groups for the given events
+
+        Args:
+            _room_id (str): id of the room for these events
+            event_ids (iterable[str]): ids of the events
+
+        Returns:
+            Deferred[dict[int, dict[tuple[str, str], str]]]:
+                dict of state_group_id -> (dict of (type, state_key) -> event id)
+        """
         if not event_ids:
             defer.returnValue({})
 
@@ -270,7 +280,7 @@ def get_state_groups_ids(self, room_id, event_ids):
 
     @defer.inlineCallbacks
     def get_state_ids_for_group(self, state_group):
-        """Get the state IDs for the given state group
+        """Get the event IDs of all the state in the given state group
 
         Args:
             state_group (int)
@@ -286,7 +296,9 @@ def get_state_ids_for_group(self, state_group):
     def get_state_groups(self, room_id, event_ids):
         """ Get the state groups for the given list of event_ids
 
-        The return value is a dict mapping group names to lists of events.
+        Returns:
+            Deferred[dict[int, list[EventBase]]]:
+                dict of state_group_id -> list of state events.
         """
         if not event_ids:
             defer.returnValue({})
@@ -324,7 +336,9 @@ def _get_state_groups_from_groups(self, groups, types, members=None):
                 member events (if True), or to exclude member events (if False)
 
         Returns:
-            dictionary state_group -> (dict of (type, state_key) -> event id)
+        Returns:
+            Deferred[dict[int, dict[tuple[str, str], str]]]:
+                dict of state_group_id -> (dict of (type, state_key) -> event id)
         """
         results = {}
 
@@ -732,8 +746,8 @@ def _get_state_for_groups(self, groups, types=None, filtered_types=None):
                 If None, `types` filtering is applied to all events.
 
         Returns:
-            Deferred[dict[int, dict[(type, state_key), EventBase]]]
-                a dictionary mapping from state group to state dictionary.
+            Deferred[dict[int, dict[tuple[str, str], str]]]:
+                dict of state_group_id -> (dict of (type, state_key) -> event id)
         """
         if types is not None:
             non_member_types = [t for t in types if t[0] != EventTypes.Member]
@@ -788,8 +802,8 @@ def _get_state_for_groups_using_cache(
                 If None, `types` filtering is applied to all events.
 
         Returns:
-            Deferred[dict[int, dict[(type, state_key), EventBase]]]
-                a dictionary mapping from state group to state dictionary.
+            Deferred[dict[int, dict[tuple[str, str], str]]]:
+                dict of state_group_id -> (dict of (type, state_key) -> event id)
         """
         if types:
             types = frozenset(types)
diff --git a/tests/storage/test_state.py b/tests/storage/test_state.py
index b91096593216..b9c5b39d5982 100644
--- a/tests/storage/test_state.py
+++ b/tests/storage/test_state.py
@@ -74,6 +74,45 @@ def assertStateMapEqual(self, s1, s2):
             self.assertEqual(s1[t].event_id, s2[t].event_id)
         self.assertEqual(len(s1), len(s2))
 
+    @defer.inlineCallbacks
+    def test_get_state_groups_ids(self):
+        e1 = yield self.inject_state_event(
+            self.room, self.u_alice, EventTypes.Create, '', {}
+        )
+        e2 = yield self.inject_state_event(
+            self.room, self.u_alice, EventTypes.Name, '', {"name": "test room"}
+        )
+
+        state_group_map = yield self.store.get_state_groups_ids(self.room, [e2.event_id])
+        self.assertEqual(len(state_group_map), 1)
+        state_map = list(state_group_map.values())[0]
+        self.assertDictEqual(
+            state_map,
+            {
+                (EventTypes.Create, ''): e1.event_id,
+                (EventTypes.Name, ''): e2.event_id,
+            },
+        )
+
+    @defer.inlineCallbacks
+    def test_get_state_groups(self):
+        e1 = yield self.inject_state_event(
+            self.room, self.u_alice, EventTypes.Create, '', {}
+        )
+        e2 = yield self.inject_state_event(
+            self.room, self.u_alice, EventTypes.Name, '', {"name": "test room"}
+        )
+
+        state_group_map = yield self.store.get_state_groups(
+            self.room, [e2.event_id])
+        self.assertEqual(len(state_group_map), 1)
+        state_list = list(state_group_map.values())[0]
+
+        self.assertEqual(
+            {ev.event_id for ev in state_list},
+            {e1.event_id, e2.event_id},
+        )
+
     @defer.inlineCallbacks
     def test_get_state_for_event(self):
 

From 4a15a3e4d539dcea9a4a57e7cd800a926f2a17c3 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Thu, 27 Sep 2018 11:25:34 +0100
Subject: [PATCH 738/830] Include eventid in log lines when processing incoming
 federation transactions (#3959)

when processing incoming transactions, it can be hard to see what's going on,
because we process a bunch of stuff in parallel, and because we may end up
recursively working our way through a chain of three or four events.

This commit creates a way to use logcontexts to add the relevant event ids to
the log lines.
---
 changelog.d/3959.feature                |  1 +
 synapse/federation/federation_server.py | 32 ++++++------
 synapse/handlers/federation.py          | 65 +++++++++++++++----------
 synapse/util/logcontext.py              | 41 ++++++++++++++--
 tests/test_federation.py                | 28 ++++++-----
 tests/util/test_logcontext.py           |  5 ++
 6 files changed, 115 insertions(+), 57 deletions(-)
 create mode 100644 changelog.d/3959.feature

diff --git a/changelog.d/3959.feature b/changelog.d/3959.feature
new file mode 100644
index 000000000000..b3a4f37a8da1
--- /dev/null
+++ b/changelog.d/3959.feature
@@ -0,0 +1 @@
+Include eventid in log lines when processing incoming federation transactions
\ No newline at end of file
diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index 9a571e4fc7d8..819e8f73315e 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -46,6 +46,7 @@
 from synapse.types import get_domain_from_id
 from synapse.util.async_helpers import Linearizer, concurrently_execute
 from synapse.util.caches.response_cache import ResponseCache
+from synapse.util.logcontext import nested_logging_context
 from synapse.util.logutils import log_function
 
 # when processing incoming transactions, we try to handle multiple rooms in
@@ -187,21 +188,22 @@ def process_pdus_for_room(room_id):
 
             for pdu in pdus_by_room[room_id]:
                 event_id = pdu.event_id
-                try:
-                    yield self._handle_received_pdu(
-                        origin, pdu
-                    )
-                    pdu_results[event_id] = {}
-                except FederationError as e:
-                    logger.warn("Error handling PDU %s: %s", event_id, e)
-                    pdu_results[event_id] = {"error": str(e)}
-                except Exception as e:
-                    f = failure.Failure()
-                    pdu_results[event_id] = {"error": str(e)}
-                    logger.error(
-                        "Failed to handle PDU %s: %s",
-                        event_id, f.getTraceback().rstrip(),
-                    )
+                with nested_logging_context(event_id):
+                    try:
+                        yield self._handle_received_pdu(
+                            origin, pdu
+                        )
+                        pdu_results[event_id] = {}
+                    except FederationError as e:
+                        logger.warn("Error handling PDU %s: %s", event_id, e)
+                        pdu_results[event_id] = {"error": str(e)}
+                    except Exception as e:
+                        f = failure.Failure()
+                        pdu_results[event_id] = {"error": str(e)}
+                        logger.error(
+                            "Failed to handle PDU %s: %s",
+                            event_id, f.getTraceback().rstrip(),
+                        )
 
         yield concurrently_execute(
             process_pdus_for_room, pdus_by_room.keys(),
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 2ccdc3bfa71a..993546387c0b 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -339,14 +339,18 @@ def on_receive_pdu(
                             "[%s %s] Requesting state at missing prev_event %s",
                             room_id, event_id, p,
                         )
-                        state, got_auth_chain = (
-                            yield self.federation_client.get_state_for_room(
-                                origin, room_id, p,
+
+                        with logcontext.nested_logging_context(p):
+                            state, got_auth_chain = (
+                                yield self.federation_client.get_state_for_room(
+                                    origin, room_id, p,
+                                )
                             )
-                        )
-                        auth_chains.update(got_auth_chain)
-                        state_group = {(x.type, x.state_key): x.event_id for x in state}
-                        state_groups.append(state_group)
+                            auth_chains.update(got_auth_chain)
+                            state_group = {
+                                (x.type, x.state_key): x.event_id for x in state
+                            }
+                            state_groups.append(state_group)
 
                     # Resolve any conflicting state
                     def fetch(ev_ids):
@@ -483,20 +487,21 @@ def _get_missing_events_for_pdu(self, origin, pdu, prevs, min_depth):
                 "[%s %s] Handling received prev_event %s",
                 room_id, event_id, ev.event_id,
             )
-            try:
-                yield self.on_receive_pdu(
-                    origin,
-                    ev,
-                    sent_to_us_directly=False,
-                )
-            except FederationError as e:
-                if e.code == 403:
-                    logger.warn(
-                        "[%s %s] Received prev_event %s failed history check.",
-                        room_id, event_id, ev.event_id,
+            with logcontext.nested_logging_context(ev.event_id):
+                try:
+                    yield self.on_receive_pdu(
+                        origin,
+                        ev,
+                        sent_to_us_directly=False,
                     )
-                else:
-                    raise
+                except FederationError as e:
+                    if e.code == 403:
+                        logger.warn(
+                            "[%s %s] Received prev_event %s failed history check.",
+                            room_id, event_id, ev.event_id,
+                        )
+                    else:
+                        raise
 
     @defer.inlineCallbacks
     def _process_received_pdu(self, origin, event, state, auth_chain):
@@ -1135,7 +1140,8 @@ def _handle_queued_pdus(self, room_queue):
             try:
                 logger.info("Processing queued PDU %s which was received "
                             "while we were joining %s", p.event_id, p.room_id)
-                yield self.on_receive_pdu(origin, p, sent_to_us_directly=True)
+                with logcontext.nested_logging_context(p.event_id):
+                    yield self.on_receive_pdu(origin, p, sent_to_us_directly=True)
             except Exception as e:
                 logger.warn(
                     "Error handling queued PDU %s from %s: %s",
@@ -1581,15 +1587,22 @@ def _handle_new_events(self, origin, event_infos, backfilled=False):
 
         Notifies about the events where appropriate.
         """
-        contexts = yield logcontext.make_deferred_yieldable(defer.gatherResults(
-            [
-                logcontext.run_in_background(
-                    self._prep_event,
+
+        @defer.inlineCallbacks
+        def prep(ev_info):
+            event = ev_info["event"]
+            with logcontext.nested_logging_context(suffix=event.event_id):
+                res = yield self._prep_event(
                     origin,
-                    ev_info["event"],
+                    event,
                     state=ev_info.get("state"),
                     auth_events=ev_info.get("auth_events"),
                 )
+            defer.returnValue(res)
+
+        contexts = yield logcontext.make_deferred_yieldable(defer.gatherResults(
+            [
+                logcontext.run_in_background(prep, ev_info)
                 for ev_info in event_infos
             ], consumeErrors=True,
         ))
diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py
index a0c2d37610ea..89224b26cc41 100644
--- a/synapse/util/logcontext.py
+++ b/synapse/util/logcontext.py
@@ -200,7 +200,7 @@ def __nonzero__(self):
 
     sentinel = Sentinel()
 
-    def __init__(self, name=None, parent_context=None):
+    def __init__(self, name=None, parent_context=None, request=None):
         self.previous_context = LoggingContext.current_context()
         self.name = name
 
@@ -218,6 +218,13 @@ def __init__(self, name=None, parent_context=None):
 
         self.parent_context = parent_context
 
+        if self.parent_context is not None:
+            self.parent_context.copy_to(self)
+
+        if request is not None:
+            # the request param overrides the request from the parent context
+            self.request = request
+
     def __str__(self):
         return "%s@%x" % (self.name, id(self))
 
@@ -256,9 +263,6 @@ def __enter__(self):
             )
         self.alive = True
 
-        if self.parent_context is not None:
-            self.parent_context.copy_to(self)
-
         return self
 
     def __exit__(self, type, value, traceback):
@@ -439,6 +443,35 @@ def __exit__(self, type, value, traceback):
                 )
 
 
+def nested_logging_context(suffix, parent_context=None):
+    """Creates a new logging context as a child of another.
+
+    The nested logging context will have a 'request' made up of the parent context's
+    request, plus the given suffix.
+
+    CPU/db usage stats will be added to the parent context's on exit.
+
+    Normal usage looks like:
+
+        with nested_logging_context(suffix):
+            # ... do stuff
+
+    Args:
+        suffix (str): suffix to add to the parent context's 'request'.
+        parent_context (LoggingContext|None): parent context. Will use the current context
+            if None.
+
+    Returns:
+        LoggingContext: new logging context.
+    """
+    if parent_context is None:
+        parent_context = LoggingContext.current_context()
+    return LoggingContext(
+        parent_context=parent_context,
+        request=parent_context.request + "-" + suffix,
+    )
+
+
 def preserve_fn(f):
     """Function decorator which wraps the function with run_in_background"""
     def g(*args, **kwargs):
diff --git a/tests/test_federation.py b/tests/test_federation.py
index 2540604fcc91..ff55c7a62740 100644
--- a/tests/test_federation.py
+++ b/tests/test_federation.py
@@ -6,6 +6,7 @@
 from synapse.events import FrozenEvent
 from synapse.types import Requester, UserID
 from synapse.util import Clock
+from synapse.util.logcontext import LoggingContext
 
 from tests import unittest
 from tests.server import ThreadedMemoryReactorClock, setup_test_homeserver
@@ -117,9 +118,10 @@ def post_json(destination, path, data, headers=None, timeout=0):
             }
         )
 
-        d = self.handler.on_receive_pdu(
-            "test.serv", lying_event, sent_to_us_directly=True
-        )
+        with LoggingContext(request="lying_event"):
+            d = self.handler.on_receive_pdu(
+                "test.serv", lying_event, sent_to_us_directly=True
+            )
 
         # Step the reactor, so the database fetches come back
         self.reactor.advance(1)
@@ -209,11 +211,12 @@ def get_json(destination, path, args, headers=None):
             }
         )
 
-        d = self.handler.on_receive_pdu(
-            "test.serv", good_event, sent_to_us_directly=True
-        )
-        self.reactor.advance(1)
-        self.assertEqual(self.successResultOf(d), None)
+        with LoggingContext(request="good_event"):
+            d = self.handler.on_receive_pdu(
+                "test.serv", good_event, sent_to_us_directly=True
+            )
+            self.reactor.advance(1)
+            self.assertEqual(self.successResultOf(d), None)
 
         bad_event = FrozenEvent(
             {
@@ -230,10 +233,11 @@ def get_json(destination, path, args, headers=None):
             }
         )
 
-        d = self.handler.on_receive_pdu(
-            "test.serv", bad_event, sent_to_us_directly=True
-        )
-        self.reactor.advance(1)
+        with LoggingContext(request="bad_event"):
+            d = self.handler.on_receive_pdu(
+                "test.serv", bad_event, sent_to_us_directly=True
+            )
+            self.reactor.advance(1)
 
         extrem = maybeDeferred(
             self.homeserver.datastore.get_latest_event_ids_in_room, self.room_id
diff --git a/tests/util/test_logcontext.py b/tests/util/test_logcontext.py
index 4633db77b3fa..8adaee3c8d3d 100644
--- a/tests/util/test_logcontext.py
+++ b/tests/util/test_logcontext.py
@@ -159,6 +159,11 @@ def test_make_deferred_yieldable_on_non_deferred(self):
             self.assertEqual(r, "bum")
             self._check_test_key("one")
 
+    def test_nested_logging_context(self):
+        with LoggingContext(request="foo"):
+            nested_context = logcontext.nested_logging_context(suffix="bar")
+            self.assertEqual(nested_context.request, "foo-bar")
+
 
 # a function which returns a deferred which has been "called", but
 # which had a function which returned another incomplete deferred on

From e3c159863d72ba1628394497bba45dd96b9cc1ac Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 26 Sep 2018 08:09:07 +0100
Subject: [PATCH 739/830] Clarifications in FederationHandler

* add some comments on things that look a bit bogus
* rename this `state` variable to avoid confusion with the `state` used
  elsewhere in this function. (There was no actual conflict, but it was
  a confusing bit of spaghetti.)
---
 synapse/handlers/federation.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 128926e71951..6793f9b6c648 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -341,14 +341,23 @@ def on_receive_pdu(
                         )
 
                         with logcontext.nested_logging_context(p):
-                            state, got_auth_chain = (
+                            # XXX if any of the missing prevs share missing state or auth
+                            # events, we'll end up requesting those missing events for
+                            # *each* missing prev, contributing to the hammering of /event
+                            # as per https://github.com/matrix-org/synapse/issues/2164.
+                            remote_state, got_auth_chain = (
                                 yield self.federation_client.get_state_for_room(
                                     origin, room_id, p,
                                 )
                             )
+
+                            # XXX hrm I'm not convinced that duplicate events will compare
+                            # for equality, so I'm not sure this does what the author
+                            # hoped.
                             auth_chains.update(got_auth_chain)
+
                             state_group = {
-                                (x.type, x.state_key): x.event_id for x in state
+                                (x.type, x.state_key): x.event_id for x in remote_state
                             }
                             state_groups.append(state_group)
 

From ad8e137062deb70ab39bcade147e943abbeb6839 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 26 Sep 2018 17:43:30 +0100
Subject: [PATCH 740/830] changelog

---
 changelog.d/3967.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3967.misc

diff --git a/changelog.d/3967.misc b/changelog.d/3967.misc
new file mode 100644
index 000000000000..dc808aec731a
--- /dev/null
+++ b/changelog.d/3967.misc
@@ -0,0 +1 @@
+Clarifications in FederationHandler

From 28223841e05a77a44ec2c0b29d1e930c68974913 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 26 Sep 2018 19:17:36 +0100
Subject: [PATCH 741/830] more comments

---
 synapse/federation/federation_client.py | 2 --
 synapse/handlers/federation.py          | 7 +++----
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index 8bf1ad0c1fc5..d05ed91d64b7 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -209,8 +209,6 @@ def get_pdu(self, destinations, event_id, outlier=False, timeout=None):
         Will attempt to get the PDU from each destination in the list until
         one succeeds.
 
-        This will persist the PDU locally upon receipt.
-
         Args:
             destinations (list): Which home servers to query
             event_id (str): event to fetch
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 6793f9b6c648..38bebbf5989c 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -341,10 +341,9 @@ def on_receive_pdu(
                         )
 
                         with logcontext.nested_logging_context(p):
-                            # XXX if any of the missing prevs share missing state or auth
-                            # events, we'll end up requesting those missing events for
-                            # *each* missing prev, contributing to the hammering of /event
-                            # as per https://github.com/matrix-org/synapse/issues/2164.
+                            # note that if any of the missing prevs share missing state or
+                            # auth events, the requests to fetch those events are deduped
+                            # by the get_pdu_cache in federation_client.
                             remote_state, got_auth_chain = (
                                 yield self.federation_client.get_state_for_room(
                                     origin, room_id, p,

From a215b698c43f4cfe8a2fdb9c160c8ecd1c1297c5 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 26 Sep 2018 09:52:56 +0100
Subject: [PATCH 742/830] Fix "unhashable type: 'list'" exception in federation
 handling

get_state_groups returns a map from state_group_id to a list of FrozenEvents,
so was very much the wrong thing to be putting as one of the entries in the
list passed to resolve_events_with_factory (which expects maps from
(event_type, state_key) to event id).

We actually want get_state_groups_ids().values() rather than
get_state_groups().

This fixes the main problem in #3923, but there are other problems with this
bit of code which get discovered once you do so.
---
 synapse/handlers/federation.py | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 38bebbf5989c..2d6b8edec425 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -106,7 +106,7 @@ def __init__(self, hs):
 
         self.hs = hs
 
-        self.store = hs.get_datastore()
+        self.store = hs.get_datastore()  # type: synapse.storage.DataStore
         self.federation_client = hs.get_federation_client()
         self.state_handler = hs.get_state_handler()
         self.server_name = hs.hostname
@@ -325,12 +325,17 @@ def on_receive_pdu(
 
                 # Calculate the state of the previous events, and
                 # de-conflict them to find the current state.
-                state_groups = []
                 auth_chains = set()
                 try:
                     # Get the state of the events we know about
-                    ours = yield self.store.get_state_groups(room_id, list(seen))
-                    state_groups.append(ours)
+                    ours = yield self.store.get_state_groups_ids(room_id, seen)
+
+                    # state_maps is a list of mappings from (type, state_key) to event_id
+                    # type: list[dict[tuple[str, str], str]]
+                    state_maps = list(ours.values())
+
+                    # we don't need this any more, let's delete it.
+                    del ours
 
                     # Ask the remote server for the states we don't
                     # know about
@@ -355,10 +360,10 @@ def on_receive_pdu(
                             # hoped.
                             auth_chains.update(got_auth_chain)
 
-                            state_group = {
+                            remote_state_map = {
                                 (x.type, x.state_key): x.event_id for x in remote_state
                             }
-                            state_groups.append(state_group)
+                            state_maps.append(remote_state_map)
 
                     # Resolve any conflicting state
                     def fetch(ev_ids):
@@ -368,7 +373,7 @@ def fetch(ev_ids):
 
                     room_version = yield self.store.get_room_version(room_id)
                     state_map = yield resolve_events_with_factory(
-                        room_version, state_groups, {event_id: pdu}, fetch
+                        room_version, state_maps, {event_id: pdu}, fetch,
                     )
 
                     state = (yield self.store.get_events(state_map.values())).values()

From bd61c82bdf97460a33080bcb6b2c836616a3b415 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 26 Sep 2018 12:16:13 +0100
Subject: [PATCH 743/830] Include state from remote servers in pdu handling

If we've fetched state events from remote servers in order to resolve the state
for a new event, we need to actually pass those events into
resolve_events_with_factory (so that it can do the state res) and then persist
the ones we need - otherwise other bits of the codebase get confused about why
we have state groups pointing to non-existent events.
---
 synapse/handlers/federation.py | 23 +++++++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 2d6b8edec425..cdad565d048a 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -326,6 +326,9 @@ def on_receive_pdu(
                 # Calculate the state of the previous events, and
                 # de-conflict them to find the current state.
                 auth_chains = set()
+                event_map = {
+                    event_id: pdu,
+                }
                 try:
                     # Get the state of the events we know about
                     ours = yield self.store.get_state_groups_ids(room_id, seen)
@@ -365,18 +368,30 @@ def on_receive_pdu(
                             }
                             state_maps.append(remote_state_map)
 
+                            for x in remote_state:
+                                event_map[x.event_id] = x
+
                     # Resolve any conflicting state
+                    @defer.inlineCallbacks
                     def fetch(ev_ids):
-                        return self.store.get_events(
-                            ev_ids, get_prev_content=False, check_redacted=False
+                        fetched = yield self.store.get_events(
+                            ev_ids, get_prev_content=False, check_redacted=False,
                         )
+                        # add any events we fetch here to the `event_map` so that we
+                        # can use them to build the state event list below.
+                        event_map.update(fetched)
+                        defer.returnValue(fetched)
 
                     room_version = yield self.store.get_room_version(room_id)
                     state_map = yield resolve_events_with_factory(
-                        room_version, state_maps, {event_id: pdu}, fetch,
+                        room_version, state_maps, event_map, fetch,
                     )
 
-                    state = (yield self.store.get_events(state_map.values())).values()
+                    # we need to give _process_received_pdu the actual state events
+                    # rather than event ids, so generate that now.
+                    state = [
+                        event_map[e] for e in six.itervalues(state_map)
+                    ]
                     auth_chain = list(auth_chains)
                 except Exception:
                     logger.warn(

From 333bee27f53916bf5354a39a79aa468967730326 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 26 Sep 2018 19:49:59 +0100
Subject: [PATCH 744/830] Include event when resolving state for missing prevs

If we have a forward extremity for a room as `E`, and you receive `A`, `B`,
s.t. `A -> B -> E`, and `B` also points to an unknown event `X`, then we need
to do state res between `X` and `E`.

When that happens, we need to make sure we include `X` in the state that goes
into the state res alg.

Fixes #3934.
---
 synapse/handlers/federation.py | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index cdad565d048a..d05b63673f45 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -323,8 +323,8 @@ def on_receive_pdu(
                         affected=pdu.event_id,
                     )
 
-                # Calculate the state of the previous events, and
-                # de-conflict them to find the current state.
+                # Calculate the state after each of the previous events, and
+                # resolve them to find the correct state at the current event.
                 auth_chains = set()
                 event_map = {
                     event_id: pdu,
@@ -358,6 +358,20 @@ def on_receive_pdu(
                                 )
                             )
 
+                            # we want the state *after* p; get_state_for_room returns the
+                            # state *before* p.
+                            remote_event = yield self.federation_client.get_pdu(
+                                [origin], p, outlier=True,
+                            )
+
+                            if remote_event is None:
+                                raise Exception(
+                                    "Unable to get missing prev_event %s" % (p, )
+                                )
+
+                            if remote_event.is_state():
+                                remote_state.append(remote_event)
+
                             # XXX hrm I'm not convinced that duplicate events will compare
                             # for equality, so I'm not sure this does what the author
                             # hoped.

From 948a6d877606deb4ccc8e42fd25f50e2edcc068d Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 26 Sep 2018 19:56:51 +0100
Subject: [PATCH 745/830] changelog

---
 changelog.d/3968.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3968.bugfix

diff --git a/changelog.d/3968.bugfix b/changelog.d/3968.bugfix
new file mode 100644
index 000000000000..18d43cd64e05
--- /dev/null
+++ b/changelog.d/3968.bugfix
@@ -0,0 +1 @@
+Fix exceptions when processing incoming events over federation
\ No newline at end of file

From 0d36fe35634cc44212073f2f6fab764ee7d8c0a9 Mon Sep 17 00:00:00 2001
From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com>
Date: Tue, 25 Sep 2018 09:43:21 +0100
Subject: [PATCH 746/830] Build and push docker image to hub

---
 .circleci/config.yml | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 521aca22ef97..ff9d2a0c2d2d 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -1,5 +1,12 @@
 version: 2
 jobs:
+  dockerhubupload:
+    machine: true
+    steps:
+      - checkout
+      - run: docker build -f docker/Dockerfile -t matrixdotorg/synapse:$CIRCLE_TAG .
+      - run: docker login --username matrixdotorg --password $DOCKER_HUB_PASSWORD
+      - run: docker push
   sytestpy2:
     machine: true
     steps:
@@ -131,3 +138,7 @@ workflows:
           filters:
             branches:
               ignore: /develop|master|release-.*/
+      - dockerhub:
+          filters:
+            tags:
+              only: /v[0-9].[0-9]+.[0-9]+(.[0-9]+)?/

From 3852c2bc3900813affe7adf761238694fb6a1fd4 Mon Sep 17 00:00:00 2001
From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com>
Date: Thu, 27 Sep 2018 12:00:01 +0100
Subject: [PATCH 747/830] Fix typo

---
 .circleci/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index ff9d2a0c2d2d..e404818a5eea 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -138,7 +138,7 @@ workflows:
           filters:
             branches:
               ignore: /develop|master|release-.*/
-      - dockerhub:
+      - dockerhubupload:
           filters:
             tags:
               only: /v[0-9].[0-9]+.[0-9]+(.[0-9]+)?/

From 73a089f461dce236ce84054e591eff873200dbbb Mon Sep 17 00:00:00 2001
From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com>
Date: Tue, 25 Sep 2018 10:29:38 +0100
Subject: [PATCH 748/830] Make username configurable in the UI too

---
 .circleci/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index e404818a5eea..9a4347718584 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -5,7 +5,7 @@ jobs:
     steps:
       - checkout
       - run: docker build -f docker/Dockerfile -t matrixdotorg/synapse:$CIRCLE_TAG .
-      - run: docker login --username matrixdotorg --password $DOCKER_HUB_PASSWORD
+      - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
       - run: docker push
   sytestpy2:
     machine: true

From a3c11f73204857a363f78a26412f5aa46adf8650 Mon Sep 17 00:00:00 2001
From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com>
Date: Tue, 25 Sep 2018 10:37:08 +0100
Subject: [PATCH 749/830] Also, don't run this job on any branches

---
 .circleci/config.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 9a4347718584..cb329aa24146 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -141,4 +141,6 @@ workflows:
       - dockerhubupload:
           filters:
             tags:
-              only: /v[0-9].[0-9]+.[0-9]+(.[0-9]+)?/
+              only: /^v[0-9].[0-9]+.[0-9]+(.[0-9]+)?/
+            branches:
+              ignore: /.*/

From 8607397ea763cedaafe51124063504436a268fa9 Mon Sep 17 00:00:00 2001
From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com>
Date: Thu, 27 Sep 2018 11:44:21 +0100
Subject: [PATCH 750/830] Make a ":latest" tag, and a SHA1 commit ID one too.

Latest is horrible and makes debugging what has happened anywhere a
nightmare. We push a latest because of demand for it, but we'll also
push a SHA1 commit id so those wanting to know what they're running
(and be able to roll back if required) can use those instead.

Note that latest here is defined as "most recent master commit" not
"most recent released version", as the actual semantics of making latest
correct while still being able to build bugfixed releases of previous
versions is just ARGH. So we define it as "master" not "latest release".
---
 .circleci/config.yml | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index cb329aa24146..3cb14793fcd9 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -1,12 +1,21 @@
 version: 2
 jobs:
-  dockerhubupload:
+  dockerhubuploadrelease:
     machine: true
     steps:
       - checkout
       - run: docker build -f docker/Dockerfile -t matrixdotorg/synapse:$CIRCLE_TAG .
       - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
-      - run: docker push
+      - run: docker push matrixdotorg/synapse:$CIRCLE_TAG
+  dockerhubuploadlatest:
+    machine: true
+    steps:
+      - checkout
+      - run: docker build -f docker/Dockerfile -t matrixdotorg/synapse:$CIRCLE_SHA1 .
+      - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
+      - run: docker tag matrixdotorg/synapse:$CIRCLE_SHA1 matrixdotorg/synapse:latest
+      - run: docker push matrixdotorg/synapse:$CIRCLE_SHA1
+      - run: docker push matrixdotorg/synapse:latest
   sytestpy2:
     machine: true
     steps:
@@ -138,9 +147,13 @@ workflows:
           filters:
             branches:
               ignore: /develop|master|release-.*/
-      - dockerhubupload:
+      - dockerhubuploadrelease:
           filters:
             tags:
               only: /^v[0-9].[0-9]+.[0-9]+(.[0-9]+)?/
             branches:
               ignore: /.*/
+      - dockerhubuploadlatest:
+          filters:
+            branches:
+              only: master

From 74bbdd0412b8e35994c82cbbccad4af2122039fb Mon Sep 17 00:00:00 2001
From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com>
Date: Thu, 27 Sep 2018 11:56:18 +0100
Subject: [PATCH 751/830] Do the changelog.d dance

---
 changelog.d/3946.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3946.misc

diff --git a/changelog.d/3946.misc b/changelog.d/3946.misc
new file mode 100644
index 000000000000..803857a297f1
--- /dev/null
+++ b/changelog.d/3946.misc
@@ -0,0 +1 @@
+Automate pushes to docker hub

From dc5db01ff25dac6ec74ceea9b4d815a8c43cd7dd Mon Sep 17 00:00:00 2001
From: Schnuffle <schnuffle@gotec-cafe.de>
Date: Thu, 27 Sep 2018 13:38:50 +0200
Subject: [PATCH 752/830] Replaced all occurences of e.message with str(e)
 Signed-off-by: Schnuffle  <schnuffle@github.com>

---
 scripts-dev/dump_macaroon.py     | 2 +-
 synapse/api/filtering.py         | 2 +-
 synapse/app/__init__.py          | 2 +-
 synapse/app/appservice.py        | 2 +-
 synapse/app/client_reader.py     | 2 +-
 synapse/app/event_creator.py     | 2 +-
 synapse/app/federation_reader.py | 2 +-
 synapse/app/federation_sender.py | 2 +-
 synapse/app/frontend_proxy.py    | 2 +-
 synapse/app/homeserver.py        | 4 ++--
 synapse/app/media_repository.py  | 2 +-
 synapse/app/pusher.py            | 2 +-
 synapse/app/synchrotron.py       | 2 +-
 synapse/app/user_dir.py          | 2 +-
 synapse/config/__main__.py       | 2 +-
 synapse/handlers/e2e_keys.py     | 2 +-
 synapse/handlers/profile.py      | 2 +-
 tests/unittest.py                | 2 +-
 18 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/scripts-dev/dump_macaroon.py b/scripts-dev/dump_macaroon.py
index 6e45be75d6db..fcc5568835e9 100755
--- a/scripts-dev/dump_macaroon.py
+++ b/scripts-dev/dump_macaroon.py
@@ -21,4 +21,4 @@
     verifier.verify(macaroon, key)
     print "Signature is correct"
 except Exception as e:
-    print e.message
+    print str(e)
diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py
index a31a9a17e0b7..eed8c67e6a7b 100644
--- a/synapse/api/filtering.py
+++ b/synapse/api/filtering.py
@@ -226,7 +226,7 @@ def check_valid_filter(self, user_filter_json):
             jsonschema.validate(user_filter_json, USER_FILTER_SCHEMA,
                                 format_checker=FormatChecker())
         except jsonschema.ValidationError as e:
-            raise SynapseError(400, e.message)
+            raise SynapseError(400, str(e))
 
 
 class FilterCollection(object):
diff --git a/synapse/app/__init__.py b/synapse/app/__init__.py
index 3b6b9368b89d..c3afcc573b4b 100644
--- a/synapse/app/__init__.py
+++ b/synapse/app/__init__.py
@@ -24,7 +24,7 @@
     python_dependencies.check_requirements()
 except python_dependencies.MissingRequirementError as e:
     message = "\n".join([
-        "Missing Requirement: %s" % (e.message,),
+        "Missing Requirement: %s" % (str(e),),
         "To install run:",
         "    pip install --upgrade --force \"%s\"" % (e.dependency,),
         "",
diff --git a/synapse/app/appservice.py b/synapse/app/appservice.py
index 02039f7e793c..8559e141af47 100644
--- a/synapse/app/appservice.py
+++ b/synapse/app/appservice.py
@@ -136,7 +136,7 @@ def start(config_options):
             "Synapse appservice", config_options
         )
     except ConfigError as e:
-        sys.stderr.write("\n" + e.message + "\n")
+        sys.stderr.write("\n" + str(e) + "\n")
         sys.exit(1)
 
     assert config.worker_app == "synapse.app.appservice"
diff --git a/synapse/app/client_reader.py b/synapse/app/client_reader.py
index 4c73c637bb71..76aed8c60af3 100644
--- a/synapse/app/client_reader.py
+++ b/synapse/app/client_reader.py
@@ -153,7 +153,7 @@ def start(config_options):
             "Synapse client reader", config_options
         )
     except ConfigError as e:
-        sys.stderr.write("\n" + e.message + "\n")
+        sys.stderr.write("\n" + str(e) + "\n")
         sys.exit(1)
 
     assert config.worker_app == "synapse.app.client_reader"
diff --git a/synapse/app/event_creator.py b/synapse/app/event_creator.py
index bc82197b2a83..9060ab14f688 100644
--- a/synapse/app/event_creator.py
+++ b/synapse/app/event_creator.py
@@ -169,7 +169,7 @@ def start(config_options):
             "Synapse event creator", config_options
         )
     except ConfigError as e:
-        sys.stderr.write("\n" + e.message + "\n")
+        sys.stderr.write("\n" + str(e) + "\n")
         sys.exit(1)
 
     assert config.worker_app == "synapse.app.event_creator"
diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py
index 18ca71ef99e8..228a297fb853 100644
--- a/synapse/app/federation_reader.py
+++ b/synapse/app/federation_reader.py
@@ -140,7 +140,7 @@ def start(config_options):
             "Synapse federation reader", config_options
         )
     except ConfigError as e:
-        sys.stderr.write("\n" + e.message + "\n")
+        sys.stderr.write("\n" + str(e) + "\n")
         sys.exit(1)
 
     assert config.worker_app == "synapse.app.federation_reader"
diff --git a/synapse/app/federation_sender.py b/synapse/app/federation_sender.py
index 6501c5779230..e9a99d76e129 100644
--- a/synapse/app/federation_sender.py
+++ b/synapse/app/federation_sender.py
@@ -160,7 +160,7 @@ def start(config_options):
             "Synapse federation sender", config_options
         )
     except ConfigError as e:
-        sys.stderr.write("\n" + e.message + "\n")
+        sys.stderr.write("\n" + str(e) + "\n")
         sys.exit(1)
 
     assert config.worker_app == "synapse.app.federation_sender"
diff --git a/synapse/app/frontend_proxy.py b/synapse/app/frontend_proxy.py
index b076fbe522b0..fc4b25de1c38 100644
--- a/synapse/app/frontend_proxy.py
+++ b/synapse/app/frontend_proxy.py
@@ -228,7 +228,7 @@ def start(config_options):
             "Synapse frontend proxy", config_options
         )
     except ConfigError as e:
-        sys.stderr.write("\n" + e.message + "\n")
+        sys.stderr.write("\n" + str(e) + "\n")
         sys.exit(1)
 
     assert config.worker_app == "synapse.app.frontend_proxy"
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 8c5d858b0b65..a98fdbd210ef 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -301,7 +301,7 @@ def run_startup_checks(self, db_conn, database_engine):
         try:
             database_engine.check_database(db_conn.cursor())
         except IncorrectDatabaseSetup as e:
-            quit_with_error(e.message)
+            quit_with_error(str(e))
 
 
 # Gauges to expose monthly active user control metrics
@@ -328,7 +328,7 @@ def setup(config_options):
             config_options,
         )
     except ConfigError as e:
-        sys.stderr.write("\n" + e.message + "\n")
+        sys.stderr.write("\n" + str(e) + "\n")
         sys.exit(1)
 
     if not config:
diff --git a/synapse/app/media_repository.py b/synapse/app/media_repository.py
index 992d182dba42..acc0487adcde 100644
--- a/synapse/app/media_repository.py
+++ b/synapse/app/media_repository.py
@@ -133,7 +133,7 @@ def start(config_options):
             "Synapse media repository", config_options
         )
     except ConfigError as e:
-        sys.stderr.write("\n" + e.message + "\n")
+        sys.stderr.write("\n" + str(e) + "\n")
         sys.exit(1)
 
     assert config.worker_app == "synapse.app.media_repository"
diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py
index 2ec4c7defb98..630dcda478fe 100644
--- a/synapse/app/pusher.py
+++ b/synapse/app/pusher.py
@@ -191,7 +191,7 @@ def start(config_options):
             "Synapse pusher", config_options
         )
     except ConfigError as e:
-        sys.stderr.write("\n" + e.message + "\n")
+        sys.stderr.write("\n" + str(e) + "\n")
         sys.exit(1)
 
     assert config.worker_app == "synapse.app.pusher"
diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py
index df81b7bcbe11..9a7fc6ee9d8c 100644
--- a/synapse/app/synchrotron.py
+++ b/synapse/app/synchrotron.py
@@ -410,7 +410,7 @@ def start(config_options):
             "Synapse synchrotron", config_options
         )
     except ConfigError as e:
-        sys.stderr.write("\n" + e.message + "\n")
+        sys.stderr.write("\n" + str(e) + "\n")
         sys.exit(1)
 
     assert config.worker_app == "synapse.app.synchrotron"
diff --git a/synapse/app/user_dir.py b/synapse/app/user_dir.py
index b383e79c1ccd..0a5f62b50984 100644
--- a/synapse/app/user_dir.py
+++ b/synapse/app/user_dir.py
@@ -188,7 +188,7 @@ def start(config_options):
             "Synapse user directory", config_options
         )
     except ConfigError as e:
-        sys.stderr.write("\n" + e.message + "\n")
+        sys.stderr.write("\n" + str(e) + "\n")
         sys.exit(1)
 
     assert config.worker_app == "synapse.app.user_dir"
diff --git a/synapse/config/__main__.py b/synapse/config/__main__.py
index 58c97a70afd6..8fccf573eeac 100644
--- a/synapse/config/__main__.py
+++ b/synapse/config/__main__.py
@@ -25,7 +25,7 @@
         try:
             config = HomeServerConfig.load_config("", sys.argv[3:])
         except ConfigError as e:
-            sys.stderr.write("\n" + e.message + "\n")
+            sys.stderr.write("\n" + str(e) + "\n")
             sys.exit(1)
 
         print (getattr(config, key))
diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py
index 578e9250fba8..9dc46aa15f75 100644
--- a/synapse/handlers/e2e_keys.py
+++ b/synapse/handlers/e2e_keys.py
@@ -341,7 +341,7 @@ def _upload_one_time_keys_for_user(self, user_id, device_id, time_now,
 def _exception_to_failure(e):
     if isinstance(e, CodeMessageException):
         return {
-            "status": e.code, "message": e.message,
+            "status": e.code, "message": str(e),
         }
 
     if isinstance(e, NotRetryingDestination):
diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py
index 75b8b7ce6a76..f284d5a38560 100644
--- a/synapse/handlers/profile.py
+++ b/synapse/handlers/profile.py
@@ -278,7 +278,7 @@ def _update_join_states(self, requester, target_user):
             except Exception as e:
                 logger.warn(
                     "Failed to update join event for room %s - %s",
-                    room_id, str(e.message)
+                    room_id, str(e)
                 )
 
 
diff --git a/tests/unittest.py b/tests/unittest.py
index ef905e6389de..043710afaff3 100644
--- a/tests/unittest.py
+++ b/tests/unittest.py
@@ -121,7 +121,7 @@ def assertObjectHasAttributes(self, attrs, obj):
             try:
                 self.assertEquals(attrs[key], getattr(obj, key))
             except AssertionError as e:
-                raise (type(e))(e.message + " for '.%s'" % key)
+                raise (type(e))(str(e) + " for '.%s'" % key)
 
     def assert_dict(self, required, actual):
         """Does a partial assert of a dict.

From a873896096bd05b608d7db80c9f0c12090877644 Mon Sep 17 00:00:00 2001
From: Schnuffle <schnuffle@gotec-cafe.de>
Date: Thu, 27 Sep 2018 14:01:44 +0200
Subject: [PATCH 753/830] Added changelog fragment

---
 changelog.d/3970.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3970.bugfix

diff --git a/changelog.d/3970.bugfix b/changelog.d/3970.bugfix
new file mode 100644
index 000000000000..c4caa31edee5
--- /dev/null
+++ b/changelog.d/3970.bugfix
@@ -0,0 +1 @@
+Replaced all occurences of e.message with str(e)Replaced all occurences of e.message with str(e). Contributed by Schnuffle

From 484a9b8c81a2315a05a00c7094b7f5fb4a9d5794 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 27 Sep 2018 12:48:06 +0100
Subject: [PATCH 754/830] Remove redundant, failing, test

This test didn't do what it claimed to do, and what it claimed to do was the
same as test_cant_hide_direct_ancestors anyway.

This stuff is tested by sytest anyway.
---
 tests/test_federation.py | 106 ---------------------------------------
 1 file changed, 106 deletions(-)

diff --git a/tests/test_federation.py b/tests/test_federation.py
index ff55c7a62740..952a0a7b51fb 100644
--- a/tests/test_federation.py
+++ b/tests/test_federation.py
@@ -141,109 +141,3 @@ def post_json(destination, path, data, headers=None, timeout=0):
             self.homeserver.datastore.get_latest_event_ids_in_room, self.room_id
         )
         self.assertEqual(self.successResultOf(extrem)[0], "$join:test.serv")
-
-    def test_cant_hide_past_history(self):
-        """
-        If you send a message, you must be able to provide the direct
-        prev_events that said event references.
-        """
-
-        def post_json(destination, path, data, headers=None, timeout=0):
-            if path.startswith("/_matrix/federation/v1/get_missing_events/"):
-                return {
-                    "events": [
-                        {
-                            "room_id": self.room_id,
-                            "sender": "@baduser:test.serv",
-                            "event_id": "three:test.serv",
-                            "depth": 1000,
-                            "origin_server_ts": 1,
-                            "type": "m.room.message",
-                            "origin": "test.serv",
-                            "content": "hewwo?",
-                            "auth_events": [],
-                            "prev_events": [("four:test.serv", {})],
-                        }
-                    ]
-                }
-
-        self.http_client.post_json = post_json
-
-        def get_json(destination, path, args, headers=None):
-            if path.startswith("/_matrix/federation/v1/state_ids/"):
-                d = self.successResultOf(
-                    self.homeserver.datastore.get_state_ids_for_event("one:test.serv")
-                )
-
-                return succeed(
-                    {
-                        "pdu_ids": [
-                            y
-                            for x, y in d.items()
-                            if x == ("m.room.member", "@us:test")
-                        ],
-                        "auth_chain_ids": list(d.values()),
-                    }
-                )
-
-        self.http_client.get_json = get_json
-
-        # Figure out what the most recent event is
-        most_recent = self.successResultOf(
-            maybeDeferred(
-                self.homeserver.datastore.get_latest_event_ids_in_room, self.room_id
-            )
-        )[0]
-
-        # Make a good event
-        good_event = FrozenEvent(
-            {
-                "room_id": self.room_id,
-                "sender": "@baduser:test.serv",
-                "event_id": "one:test.serv",
-                "depth": 1000,
-                "origin_server_ts": 1,
-                "type": "m.room.message",
-                "origin": "test.serv",
-                "content": "hewwo?",
-                "auth_events": [],
-                "prev_events": [(most_recent, {})],
-            }
-        )
-
-        with LoggingContext(request="good_event"):
-            d = self.handler.on_receive_pdu(
-                "test.serv", good_event, sent_to_us_directly=True
-            )
-            self.reactor.advance(1)
-            self.assertEqual(self.successResultOf(d), None)
-
-        bad_event = FrozenEvent(
-            {
-                "room_id": self.room_id,
-                "sender": "@baduser:test.serv",
-                "event_id": "two:test.serv",
-                "depth": 1000,
-                "origin_server_ts": 1,
-                "type": "m.room.message",
-                "origin": "test.serv",
-                "content": "hewwo?",
-                "auth_events": [],
-                "prev_events": [("one:test.serv", {}), ("three:test.serv", {})],
-            }
-        )
-
-        with LoggingContext(request="bad_event"):
-            d = self.handler.on_receive_pdu(
-                "test.serv", bad_event, sent_to_us_directly=True
-            )
-            self.reactor.advance(1)
-
-        extrem = maybeDeferred(
-            self.homeserver.datastore.get_latest_event_ids_in_room, self.room_id
-        )
-        self.assertEqual(self.successResultOf(extrem)[0], "two:test.serv")
-
-        state = self.homeserver.get_state_handler().get_current_state_ids(self.room_id)
-        self.reactor.advance(1)
-        self.assertIn(("m.room.member", "@us:test"), self.successResultOf(state).keys())

From 861c063ebc8989e6b25d2e00e7aedf0ecc780f41 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Thu, 27 Sep 2018 22:47:01 +1000
Subject: [PATCH 755/830] Update 3970.bugfix

---
 changelog.d/3970.bugfix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/changelog.d/3970.bugfix b/changelog.d/3970.bugfix
index c4caa31edee5..562531549788 100644
--- a/changelog.d/3970.bugfix
+++ b/changelog.d/3970.bugfix
@@ -1 +1 @@
-Replaced all occurences of e.message with str(e)Replaced all occurences of e.message with str(e). Contributed by Schnuffle
+Replaced all occurences of e.message with str(e). Contributed by Schnuffle

From b3064532d005e08a3f5ec0c2a2decf688208f5e6 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Thu, 27 Sep 2018 23:21:54 +1000
Subject: [PATCH 756/830] Run our oldest supported configuration in CI (#3952)

---
 .travis.yml                    |  3 +++
 changelog.d/3952.misc          |  1 +
 synapse/python_dependencies.py | 27 ++++++++++++++-------------
 tox.ini                        | 20 ++++++++++++++++++++
 4 files changed, 38 insertions(+), 13 deletions(-)
 create mode 100644 changelog.d/3952.misc

diff --git a/.travis.yml b/.travis.yml
index b6faca4b9258..2077f6af72ef 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -20,6 +20,9 @@ matrix:
   - python: 2.7
     env: TOX_ENV=py27
 
+  - python: 2.7
+    env: TOX_ENV=py27-old
+
   - python: 2.7
     env: TOX_ENV=py27-postgres TRIAL_FLAGS="-j 4"
     services:
diff --git a/changelog.d/3952.misc b/changelog.d/3952.misc
new file mode 100644
index 000000000000..015e4a43e682
--- /dev/null
+++ b/changelog.d/3952.misc
@@ -0,0 +1 @@
+Run the test suite on the oldest supported versions of our dependencies in CI.
\ No newline at end of file
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index c779f69fa07f..0f339a03203e 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -33,31 +33,32 @@
 # [2] https://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-dependencies
 REQUIREMENTS = {
     "jsonschema>=2.5.1": ["jsonschema>=2.5.1"],
-    "frozendict>=0.4": ["frozendict"],
+    "frozendict>=1": ["frozendict"],
     "unpaddedbase64>=1.1.0": ["unpaddedbase64>=1.1.0"],
     "canonicaljson>=1.1.3": ["canonicaljson>=1.1.3"],
     "signedjson>=1.0.0": ["signedjson>=1.0.0"],
     "pynacl>=1.2.1": ["nacl>=1.2.1", "nacl.bindings"],
-    "service_identity>=1.0.0": ["service_identity>=1.0.0"],
+    "service_identity>=16.0.0": ["service_identity>=16.0.0"],
     "Twisted>=17.1.0": ["twisted>=17.1.0"],
     "treq>=15.1": ["treq>=15.1"],
 
     # Twisted has required pyopenssl 16.0 since about Twisted 16.6.
     "pyopenssl>=16.0.0": ["OpenSSL>=16.0.0"],
 
-    "pyyaml": ["yaml"],
-    "pyasn1": ["pyasn1"],
-    "daemonize": ["daemonize"],
-    "bcrypt": ["bcrypt>=3.1.0"],
-    "pillow": ["PIL"],
-    "pydenticon": ["pydenticon"],
-    "sortedcontainers": ["sortedcontainers"],
-    "pysaml2>=3.0.0": ["saml2>=3.0.0"],
-    "pymacaroons-pynacl": ["pymacaroons"],
+    "pyyaml>=3.11": ["yaml"],
+    "pyasn1>=0.1.9": ["pyasn1"],
+    "pyasn1-modules>=0.0.7": ["pyasn1_modules"],
+    "daemonize>=2.3.1": ["daemonize"],
+    "bcrypt>=3.1.0": ["bcrypt>=3.1.0"],
+    "pillow>=3.1.2": ["PIL"],
+    "pydenticon>=0.2": ["pydenticon"],
+    "sortedcontainers>=1.4.4": ["sortedcontainers"],
+    "pysaml2>=3.0.0": ["saml2"],
+    "pymacaroons-pynacl>=0.9.3": ["pymacaroons"],
     "msgpack-python>=0.3.0": ["msgpack"],
     "phonenumbers>=8.2.0": ["phonenumbers"],
-    "six": ["six"],
-    "prometheus_client": ["prometheus_client"],
+    "six>=1.10": ["six"],
+    "prometheus_client>=0.0.18": ["prometheus_client"],
 
     # we use attr.s(slots), which arrived in 16.0.0
     "attrs>=16.0.0": ["attr>=16.0.0"],
diff --git a/tox.ini b/tox.ini
index e4db563b4bfd..87b5e4782dcc 100644
--- a/tox.ini
+++ b/tox.ini
@@ -64,6 +64,26 @@ setenv =
     {[base]setenv}
     SYNAPSE_POSTGRES = 1
 
+# A test suite for the oldest supported versions of Python libraries, to catch
+# any uses of APIs not available in them.
+[testenv:py27-old]
+skip_install=True
+deps =
+    # Old automat version for Twisted
+    Automat == 0.3.0
+
+    mock
+    lxml
+commands =
+    /usr/bin/find "{toxinidir}" -name '*.pyc' -delete
+    # Make all greater-thans equals so we test the oldest version of our direct
+    # dependencies, but make the pyopenssl 17.0, which can work against an
+    # OpenSSL 1.1 compiled cryptography (as older ones don't compile on Travis).
+    /bin/sh -c 'python -m synapse.python_dependencies | sed -e "s/>=/==/g" -e "s/psycopg2==2.6//" -e "s/pyopenssl==16.0.0/pyopenssl==17.0.0/" | xargs pip install'
+    # Install Synapse itself. This won't update any libraries.
+    pip install -e .
+    {envbindir}/trial {env:TRIAL_FLAGS:} {posargs:tests} {env:TOXSUFFIX:}
+
 [testenv:py35]
 usedevelop=true
 

From 19475cf3371819170737515c414e3cc7c2a32d43 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 28 Sep 2018 11:55:57 +0100
Subject: [PATCH 757/830] Remove redundant call to start_get_pdu_cache

I think this got forgotten in #3932. We were getting away with it because it
was the last call in this function.
---
 changelog.d/3980.bugfix   | 1 +
 synapse/app/homeserver.py | 1 -
 2 files changed, 1 insertion(+), 1 deletion(-)
 create mode 100644 changelog.d/3980.bugfix

diff --git a/changelog.d/3980.bugfix b/changelog.d/3980.bugfix
new file mode 100644
index 000000000000..7578414ede72
--- /dev/null
+++ b/changelog.d/3980.bugfix
@@ -0,0 +1 @@
+Fix some instances of ExpiringCache not expiring cache items
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index a98fdbd210ef..e3f0d99a3ff2 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -386,7 +386,6 @@ def start():
         hs.get_pusherpool().start()
         hs.get_datastore().start_profiling()
         hs.get_datastore().start_doing_background_updates()
-        hs.get_federation_client().start_get_pdu_cache()
 
     reactor.callWhenRunning(start)
 

From 5304449738b0193012a613edf74437e4af2ff416 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Christian=20Gr=C3=BCnhage?=
 <jan.christian@gruenhage.xyz>
Date: Fri, 28 Sep 2018 13:36:35 +0200
Subject: [PATCH 758/830] build python 3 docker images on circle CI (#3976)

---
 .circleci/config.yml  | 16 +++++++++++-----
 changelog.d/3976.misc |  1 +
 2 files changed, 12 insertions(+), 5 deletions(-)
 create mode 100644 changelog.d/3976.misc

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 3cb14793fcd9..6ae3a42235a7 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -4,18 +4,24 @@ jobs:
     machine: true
     steps:
       - checkout
-      - run: docker build -f docker/Dockerfile -t matrixdotorg/synapse:$CIRCLE_TAG .
+      - run: docker build -f docker/Dockerfile -t matrixdotorg/synapse:${CIRCLE_TAG} .
+      - run: docker build -f docker/Dockerfile -t matrixdotorg/synapse:${CIRCLE_TAG}-py3 --build-arg PYTHON_VERSION=3.6 .
       - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
-      - run: docker push matrixdotorg/synapse:$CIRCLE_TAG
+      - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}
+      - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}-py3
   dockerhubuploadlatest:
     machine: true
     steps:
       - checkout
-      - run: docker build -f docker/Dockerfile -t matrixdotorg/synapse:$CIRCLE_SHA1 .
+      - run: docker build -f docker/Dockerfile -t matrixdotorg/synapse:${CIRCLE_SHA1} .
+      - run: docker build -f docker/Dockerfile -t matrixdotorg/synapse:${CIRCLE_SHA1}-py3 --build-arg PYTHON_VERSION=3.6 .
       - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
-      - run: docker tag matrixdotorg/synapse:$CIRCLE_SHA1 matrixdotorg/synapse:latest
-      - run: docker push matrixdotorg/synapse:$CIRCLE_SHA1
+      - run: docker tag matrixdotorg/synapse:${CIRCLE_SHA1} matrixdotorg/synapse:latest
+      - run: docker tag matrixdotorg/synapse:${CIRCLE_SHA1}-py3 matrixdotorg/synapse:latest-py3
+      - run: docker push matrixdotorg/synapse:${CIRCLE_SHA1}
+      - run: docker push matrixdotorg/synapse:${CIRCLE_SHA1}-py3
       - run: docker push matrixdotorg/synapse:latest
+      - run: docker push matrixdotorg/synapse:latest-py3
   sytestpy2:
     machine: true
     steps:
diff --git a/changelog.d/3976.misc b/changelog.d/3976.misc
new file mode 100644
index 000000000000..282148c986cd
--- /dev/null
+++ b/changelog.d/3976.misc
@@ -0,0 +1 @@
+Build py3 docker images for docker hub too

From 965154d60af59b69eac01f7cfcf821a757ae93fa Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 28 Sep 2018 12:45:54 +0100
Subject: [PATCH 759/830] Fix complete fail to do the right thing

---
 synapse/federation/transaction_queue.py | 3 ++-
 synapse/handlers/typing.py              | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py
index ae47aaae0b10..98b5950800f3 100644
--- a/synapse/federation/transaction_queue.py
+++ b/synapse/federation/transaction_queue.py
@@ -354,7 +354,7 @@ def send_edu(self, destination, edu_type, content, key=None):
             content=content,
         )
 
-        if not destination == self.server_name:
+        if destination == self.server_name:
             logger.info("Not sending EDU to ourselves")
             return
 
@@ -372,6 +372,7 @@ def send_edu(self, destination, edu_type, content, key=None):
     def send_device_messages(self, destination):
         if destination == self.server_name:
             logger.info("Not sending device update to ourselves")
+            return
 
         self._attempt_new_transaction(destination)
 
diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py
index 2d2d3d5a0d91..bf82b3f86481 100644
--- a/synapse/handlers/typing.py
+++ b/synapse/handlers/typing.py
@@ -218,6 +218,7 @@ def _push_remote(self, member, typing):
 
             for domain in set(get_domain_from_id(u) for u in users):
                 if domain != self.server_name:
+                    logger.debug("sending typing update to %s", domain)
                     self.federation.send_edu(
                         destination=domain,
                         edu_type="m.typing",

From 9a8bbc9a59e4f5534225d0d07c86652242c88efe Mon Sep 17 00:00:00 2001
From: Bruno Windels <bruno@windels.cloud>
Date: Fri, 28 Sep 2018 13:36:56 +0100
Subject: [PATCH 760/830] add --no-admin flag to registration script (#3836)

---
 changelog.d/3836.bugfix          |  1 +
 scripts/register_new_matrix_user | 18 ++++++++++++++----
 2 files changed, 15 insertions(+), 4 deletions(-)
 create mode 100644 changelog.d/3836.bugfix

diff --git a/changelog.d/3836.bugfix b/changelog.d/3836.bugfix
new file mode 100644
index 000000000000..add49fbec0d5
--- /dev/null
+++ b/changelog.d/3836.bugfix
@@ -0,0 +1 @@
+support registering regular users non-interactively with register_new_matrix_user script
\ No newline at end of file
diff --git a/scripts/register_new_matrix_user b/scripts/register_new_matrix_user
index 8c3d429351c4..91bdb3a25bd8 100755
--- a/scripts/register_new_matrix_user
+++ b/scripts/register_new_matrix_user
@@ -133,7 +133,7 @@ def register_new_user(user, password, server_location, shared_secret, admin):
             print "Passwords do not match"
             sys.exit(1)
 
-    if not admin:
+    if admin is None:
         admin = raw_input("Make admin [no]: ")
         if admin in ("y", "yes", "true"):
             admin = True
@@ -160,10 +160,16 @@ if __name__ == "__main__":
         default=None,
         help="New password for user. Will prompt if omitted.",
     )
-    parser.add_argument(
+    admin_group = parser.add_mutually_exclusive_group()
+    admin_group.add_argument(
         "-a", "--admin",
         action="store_true",
-        help="Register new user as an admin. Will prompt if omitted.",
+        help="Register new user as an admin. Will prompt if --no-admin is not set either.",
+    )
+    admin_group.add_argument(
+        "--no-admin",
+        action="store_true",
+        help="Register new user as a regular user. Will prompt if --admin is not set either.",
     )
 
     group = parser.add_mutually_exclusive_group(required=True)
@@ -197,4 +203,8 @@ if __name__ == "__main__":
     else:
         secret = args.shared_secret
 
-    register_new_user(args.user, args.password, args.server_url, secret, args.admin)
+    admin = None
+    if args.admin or args.no_admin:
+        admin = args.admin
+
+    register_new_user(args.user, args.password, args.server_url, secret, admin)

From 8ea887856c1bcb97c90d88fe20f7f82ed7f48967 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 1 Oct 2018 11:59:52 +0100
Subject: [PATCH 761/830] Don't update eviction metrics on explicit removal

---
 synapse/util/caches/expiringcache.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/synapse/util/caches/expiringcache.py b/synapse/util/caches/expiringcache.py
index 8ac56d85ff36..f2f55ba6c99d 100644
--- a/synapse/util/caches/expiringcache.py
+++ b/synapse/util/caches/expiringcache.py
@@ -103,11 +103,6 @@ def pop(self, key, default=None):
         if value is SENTINEL:
             return default
 
-        if self.iterable:
-            self.metrics.inc_evictions(len(value.value))
-        else:
-            self.metrics.inc_evictions()
-
         return value
 
     def __contains__(self, key):

From 4f3e3ac192f2c5ff37198f50af99bc8fbd57beca Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 1 Oct 2018 12:25:27 +0100
Subject: [PATCH 762/830] Correctly match 'dict.pop' api

---
 synapse/util/caches/expiringcache.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/synapse/util/caches/expiringcache.py b/synapse/util/caches/expiringcache.py
index f2f55ba6c99d..f36978027748 100644
--- a/synapse/util/caches/expiringcache.py
+++ b/synapse/util/caches/expiringcache.py
@@ -98,10 +98,18 @@ def __getitem__(self, key):
 
         return entry.value
 
-    def pop(self, key, default=None):
-        value = self._cache.pop(key, SENTINEL)
+    def pop(self, key, default=SENTINEL):
+        """Removes and returns the value with the given key from the cache.
+
+        If the key isn't in the cache then `default` will be returned if
+        specified, otherwise `KeyError` will get raised.
+
+        Identical functionality to `dict.pop(..)`.
+        """
+
+        value = self._cache.pop(key, default)
         if value is SENTINEL:
-            return default
+            raise KeyError(key)
 
         return value
 

From 53c5fa4e6c179c6ed855169dcb99b69699db75be Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Mon, 1 Oct 2018 12:29:17 +0100
Subject: [PATCH 763/830] Further reduce the size of the docker image (#3972)

Rewrite the dockerfile as a multistage build: this means we can get rid of a whole load of cruft which we don't need.
---
 changelog.d/3972.misc |  1 +
 docker/Dockerfile     | 71 ++++++++++++++++++++++++++++---------------
 2 files changed, 47 insertions(+), 25 deletions(-)
 create mode 100644 changelog.d/3972.misc

diff --git a/changelog.d/3972.misc b/changelog.d/3972.misc
new file mode 100644
index 000000000000..e56299ee78b1
--- /dev/null
+++ b/changelog.d/3972.misc
@@ -0,0 +1 @@
+Further reduce the docker image size
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 1d00defc2dcd..db44c02a9234 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -1,9 +1,13 @@
 ARG PYTHON_VERSION=2
-FROM docker.io/python:${PYTHON_VERSION}-alpine3.8
 
-COPY . /synapse
+###
+### Stage 0: builder
+###
+FROM docker.io/python:${PYTHON_VERSION}-alpine3.8 as builder
 
-RUN apk add --no-cache --virtual .build_deps \
+# install the OS build deps
+
+RUN apk add \
         build-base \
         libffi-dev \
         libjpeg-turbo-dev \
@@ -11,30 +15,47 @@ RUN apk add --no-cache --virtual .build_deps \
         libxslt-dev \
         linux-headers \
         postgresql-dev \
-        zlib-dev \
- && cd /synapse \
- && apk add --no-cache --virtual .runtime_deps \
- 	libffi \
-        libjpeg-turbo \
-	libressl \
-	libxslt \
-	libpq \
-	zlib \
-	su-exec \
- && pip install --upgrade \
+        zlib-dev
+
+# build things which have slow build steps, before we copy synapse, so that
+# the layer can be cached.
+#
+# (we really just care about caching a wheel here, as the "pip install" below
+# will install them again.)
+
+RUN pip install --prefix="/install" --no-warn-script-location \
+        cryptography \
+        msgpack-python \
+        pillow \
+        pynacl
+
+# now install synapse and all of the python deps to /install.
+
+COPY . /synapse
+RUN pip install --prefix="/install" --no-warn-script-location \
         lxml \
-        pip \
         psycopg2 \
-        setuptools \
- && mkdir -p /synapse/cache \
- && pip install -f /synapse/cache --upgrade --process-dependency-links . \
- && mv /synapse/docker/start.py /synapse/docker/conf / \
- && rm -rf \
-        setup.cfg \
-        setup.py \
-        synapse \
- && apk del .build_deps
- 
+        /synapse
+
+###
+### Stage 1: runtime
+###
+
+FROM docker.io/python:${PYTHON_VERSION}-alpine3.8
+
+RUN apk add --no-cache --virtual .runtime_deps \
+        libffi \
+        libjpeg-turbo \
+        libressl \
+        libxslt \
+        libpq \
+        zlib \
+        su-exec
+
+COPY --from=builder /install /usr/local
+COPY ./docker/start.py /start.py
+COPY ./docker/conf /conf
+
 VOLUME ["/data"]
 
 EXPOSE 8008/tcp 8448/tcp

From 82f922b4af8d41e15484e1913775d234c548d9f2 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 1 Oct 2018 14:19:36 +0100
Subject: [PATCH 764/830] Fix lazy loaded sync with rejected state events

In particular, we assume that the name and canonical alias events in
the state have not been rejected. In practice this may not be the case
(though we should probably think about fixing that) so lets ensure that
we gracefully handle that case, rather than 404'ing the sync request
like we do now.
---
 synapse/handlers/sync.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index c7d69d9d805c..67b8ca28c744 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -567,13 +567,13 @@ def compute_summary(self, room_id, sync_config, batch, state, now_token):
         # be a valid name or canonical_alias - i.e. we're checking that they
         # haven't been "deleted" by blatting {} over the top.
         if name_id:
-            name = yield self.store.get_event(name_id, allow_none=False)
+            name = yield self.store.get_event(name_id, allow_none=True)
             if name and name.content:
                 defer.returnValue(summary)
 
         if canonical_alias_id:
             canonical_alias = yield self.store.get_event(
-                canonical_alias_id, allow_none=False,
+                canonical_alias_id, allow_none=True,
             )
             if canonical_alias and canonical_alias.content:
                 defer.returnValue(summary)

From 5fa27eac7827acdd5409d46eab22fe682689d30b Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 1 Oct 2018 14:24:41 +0100
Subject: [PATCH 765/830] Newsfile

---
 changelog.d/3986.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3986.bugfix

diff --git a/changelog.d/3986.bugfix b/changelog.d/3986.bugfix
new file mode 100644
index 000000000000..ce74345365ed
--- /dev/null
+++ b/changelog.d/3986.bugfix
@@ -0,0 +1 @@
+Fix lazy loaded sync in the presence of rejected state events

From 6e05fd032c670080f9e9f99f2e2e8b8eccf24c7d Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Tue, 2 Oct 2018 00:11:58 +1000
Subject: [PATCH 766/830] Fix userconsent on Python 3 (#3938)

---
 changelog.d/3938.bugfix              |   1 +
 synapse/api/urls.py                  |   2 +-
 tests/server_notices/test_consent.py | 100 +++++++++++++++++++++++++
 tests/storage/test_client_ips.py     |  48 +-----------
 tests/unittest.py                    |  80 +++++++++++++++++++-
 tests/utils.py                       | 105 +++++++++++++++------------
 6 files changed, 240 insertions(+), 96 deletions(-)
 create mode 100644 changelog.d/3938.bugfix
 create mode 100644 tests/server_notices/test_consent.py

diff --git a/changelog.d/3938.bugfix b/changelog.d/3938.bugfix
new file mode 100644
index 000000000000..01ccca21a739
--- /dev/null
+++ b/changelog.d/3938.bugfix
@@ -0,0 +1 @@
+Sending server notices regarding user consent now works on Python 3.
diff --git a/synapse/api/urls.py b/synapse/api/urls.py
index 71347912f12c..6d9f1ca0ef46 100644
--- a/synapse/api/urls.py
+++ b/synapse/api/urls.py
@@ -64,7 +64,7 @@ def build_user_consent_uri(self, user_id):
         """
         mac = hmac.new(
             key=self._hmac_secret,
-            msg=user_id,
+            msg=user_id.encode('ascii'),
             digestmod=sha256,
         ).hexdigest()
         consent_uri = "%s_matrix/consent?%s" % (
diff --git a/tests/server_notices/test_consent.py b/tests/server_notices/test_consent.py
new file mode 100644
index 000000000000..95badc985e35
--- /dev/null
+++ b/tests/server_notices/test_consent.py
@@ -0,0 +1,100 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from synapse.rest.client.v1 import admin, login, room
+from synapse.rest.client.v2_alpha import sync
+
+from tests import unittest
+
+
+class ConsentNoticesTests(unittest.HomeserverTestCase):
+
+    servlets = [
+        sync.register_servlets,
+        admin.register_servlets,
+        login.register_servlets,
+        room.register_servlets,
+    ]
+
+    def make_homeserver(self, reactor, clock):
+
+        self.consent_notice_message = "consent %(consent_uri)s"
+        config = self.default_config()
+        config.user_consent_version = "1"
+        config.user_consent_server_notice_content = {
+            "msgtype": "m.text",
+            "body": self.consent_notice_message,
+        }
+        config.public_baseurl = "https://example.com/"
+        config.form_secret = "123abc"
+
+        config.server_notices_mxid = "@notices:test"
+        config.server_notices_mxid_display_name = "test display name"
+        config.server_notices_mxid_avatar_url = None
+        config.server_notices_room_name = "Server Notices"
+
+        hs = self.setup_test_homeserver(config=config)
+
+        return hs
+
+    def prepare(self, reactor, clock, hs):
+        self.user_id = self.register_user("bob", "abc123")
+        self.access_token = self.login("bob", "abc123")
+
+    def test_get_sync_message(self):
+        """
+        When user consent server notices are enabled, a sync will cause a notice
+        to fire (in a room which the user is invited to). The notice contains
+        the notice URL + an authentication code.
+        """
+        # Initial sync, to get the user consent room invite
+        request, channel = self.make_request(
+            "GET", "/_matrix/client/r0/sync", access_token=self.access_token
+        )
+        self.render(request)
+        self.assertEqual(channel.code, 200)
+
+        # Get the Room ID to join
+        room_id = list(channel.json_body["rooms"]["invite"].keys())[0]
+
+        # Join the room
+        request, channel = self.make_request(
+            "POST",
+            "/_matrix/client/r0/rooms/" + room_id + "/join",
+            access_token=self.access_token,
+        )
+        self.render(request)
+        self.assertEqual(channel.code, 200)
+
+        # Sync again, to get the message in the room
+        request, channel = self.make_request(
+            "GET", "/_matrix/client/r0/sync", access_token=self.access_token
+        )
+        self.render(request)
+        self.assertEqual(channel.code, 200)
+
+        # Get the message
+        room = channel.json_body["rooms"]["join"][room_id]
+        messages = [
+            x for x in room["timeline"]["events"] if x["type"] == "m.room.message"
+        ]
+
+        # One message, with the consent URL
+        self.assertEqual(len(messages), 1)
+        self.assertTrue(
+            messages[0]["content"]["body"].startswith(
+                "consent https://example.com/_matrix/consent"
+            )
+        )
diff --git a/tests/storage/test_client_ips.py b/tests/storage/test_client_ips.py
index 2ffbb9f14f0f..4577e9422b53 100644
--- a/tests/storage/test_client_ips.py
+++ b/tests/storage/test_client_ips.py
@@ -14,10 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import hashlib
-import hmac
-import json
-
 from mock import Mock
 
 from twisted.internet import defer
@@ -145,34 +141,8 @@ def make_homeserver(self, reactor, clock):
         return hs
 
     def prepare(self, hs, reactor, clock):
-        self.hs.config.registration_shared_secret = u"shared"
         self.store = self.hs.get_datastore()
-
-        # Create the user
-        request, channel = self.make_request("GET", "/_matrix/client/r0/admin/register")
-        self.render(request)
-        nonce = channel.json_body["nonce"]
-
-        want_mac = hmac.new(key=b"shared", digestmod=hashlib.sha1)
-        want_mac.update(nonce.encode('ascii') + b"\x00bob\x00abc123\x00admin")
-        want_mac = want_mac.hexdigest()
-
-        body = json.dumps(
-            {
-                "nonce": nonce,
-                "username": "bob",
-                "password": "abc123",
-                "admin": True,
-                "mac": want_mac,
-            }
-        )
-        request, channel = self.make_request(
-            "POST", "/_matrix/client/r0/admin/register", body.encode('utf8')
-        )
-        self.render(request)
-
-        self.assertEqual(channel.code, 200)
-        self.user_id = channel.json_body["user_id"]
+        self.user_id = self.register_user("bob", "abc123", True)
 
     def test_request_with_xforwarded(self):
         """
@@ -194,20 +164,7 @@ def test_request_from_getPeer(self):
     def _runtest(self, headers, expected_ip, make_request_args):
         device_id = "bleb"
 
-        body = json.dumps(
-            {
-                "type": "m.login.password",
-                "user": "bob",
-                "password": "abc123",
-                "device_id": device_id,
-            }
-        )
-        request, channel = self.make_request(
-            "POST", "/_matrix/client/r0/login", body.encode('utf8'), **make_request_args
-        )
-        self.render(request)
-        self.assertEqual(channel.code, 200)
-        access_token = channel.json_body["access_token"].encode('ascii')
+        access_token = self.login("bob", "abc123", device_id=device_id)
 
         # Advance to a known time
         self.reactor.advance(123456 - self.reactor.seconds())
@@ -215,7 +172,6 @@ def _runtest(self, headers, expected_ip, make_request_args):
         request, channel = self.make_request(
             "GET",
             "/_matrix/client/r0/admin/users/" + self.user_id,
-            body.encode('utf8'),
             access_token=access_token,
             **make_request_args
         )
diff --git a/tests/unittest.py b/tests/unittest.py
index 043710afaff3..a59291cc6010 100644
--- a/tests/unittest.py
+++ b/tests/unittest.py
@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import hashlib
+import hmac
 import logging
 
 from mock import Mock
@@ -32,6 +34,7 @@
 from synapse.util.logcontext import LoggingContextFilter
 
 from tests.server import get_clock, make_request, render, setup_test_homeserver
+from tests.utils import default_config
 
 # Set up putting Synapse's logs into Trial's.
 rootLogger = logging.getLogger()
@@ -121,7 +124,7 @@ def assertObjectHasAttributes(self, attrs, obj):
             try:
                 self.assertEquals(attrs[key], getattr(obj, key))
             except AssertionError as e:
-                raise (type(e))(str(e) + " for '.%s'" % key)
+                raise (type(e))(e.message + " for '.%s'" % key)
 
     def assert_dict(self, required, actual):
         """Does a partial assert of a dict.
@@ -223,6 +226,15 @@ def make_homeserver(self, reactor, clock):
         hs = self.setup_test_homeserver()
         return hs
 
+    def default_config(self, name="test"):
+        """
+        Get a default HomeServer config object.
+
+        Args:
+            name (str): The homeserver name/domain.
+        """
+        return default_config(name)
+
     def prepare(self, reactor, clock, homeserver):
         """
         Prepare for the test.  This involves things like mocking out parts of
@@ -297,3 +309,69 @@ def get_success(self, d):
             return d
         self.pump()
         return self.successResultOf(d)
+
+    def register_user(self, username, password, admin=False):
+        """
+        Register a user. Requires the Admin API be registered.
+
+        Args:
+            username (bytes/unicode): The user part of the new user.
+            password (bytes/unicode): The password of the new user.
+            admin (bool): Whether the user should be created as an admin
+            or not.
+
+        Returns:
+            The MXID of the new user (unicode).
+        """
+        self.hs.config.registration_shared_secret = u"shared"
+
+        # Create the user
+        request, channel = self.make_request("GET", "/_matrix/client/r0/admin/register")
+        self.render(request)
+        nonce = channel.json_body["nonce"]
+
+        want_mac = hmac.new(key=b"shared", digestmod=hashlib.sha1)
+        nonce_str = b"\x00".join([username.encode('utf8'), password.encode('utf8')])
+        if admin:
+            nonce_str += b"\x00admin"
+        else:
+            nonce_str += b"\x00notadmin"
+        want_mac.update(nonce.encode('ascii') + b"\x00" + nonce_str)
+        want_mac = want_mac.hexdigest()
+
+        body = json.dumps(
+            {
+                "nonce": nonce,
+                "username": username,
+                "password": password,
+                "admin": admin,
+                "mac": want_mac,
+            }
+        )
+        request, channel = self.make_request(
+            "POST", "/_matrix/client/r0/admin/register", body.encode('utf8')
+        )
+        self.render(request)
+        self.assertEqual(channel.code, 200)
+
+        user_id = channel.json_body["user_id"]
+        return user_id
+
+    def login(self, username, password, device_id=None):
+        """
+        Log in a user, and get an access token. Requires the Login API be
+        registered.
+
+        """
+        body = {"type": "m.login.password", "user": username, "password": password}
+        if device_id:
+            body["device_id"] = device_id
+
+        request, channel = self.make_request(
+            "POST", "/_matrix/client/r0/login", json.dumps(body).encode('utf8')
+        )
+        self.render(request)
+        self.assertEqual(channel.code, 200)
+
+        access_token = channel.json_body["access_token"].encode('ascii')
+        return access_token
diff --git a/tests/utils.py b/tests/utils.py
index aaed1149c31e..1ef80e7b79c6 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -96,6 +96,62 @@ def _cleanup():
         atexit.register(_cleanup)
 
 
+def default_config(name):
+    """
+    Create a reasonable test config.
+    """
+    config = Mock()
+    config.signing_key = [MockKey()]
+    config.event_cache_size = 1
+    config.enable_registration = True
+    config.macaroon_secret_key = "not even a little secret"
+    config.expire_access_token = False
+    config.server_name = name
+    config.trusted_third_party_id_servers = []
+    config.room_invite_state_types = []
+    config.password_providers = []
+    config.worker_replication_url = ""
+    config.worker_app = None
+    config.email_enable_notifs = False
+    config.block_non_admin_invites = False
+    config.federation_domain_whitelist = None
+    config.federation_rc_reject_limit = 10
+    config.federation_rc_sleep_limit = 10
+    config.federation_rc_sleep_delay = 100
+    config.federation_rc_concurrent = 10
+    config.filter_timeline_limit = 5000
+    config.user_directory_search_all_users = False
+    config.user_consent_server_notice_content = None
+    config.block_events_without_consent_error = None
+    config.media_storage_providers = []
+    config.auto_join_rooms = []
+    config.limit_usage_by_mau = False
+    config.hs_disabled = False
+    config.hs_disabled_message = ""
+    config.hs_disabled_limit_type = ""
+    config.max_mau_value = 50
+    config.mau_trial_days = 0
+    config.mau_limits_reserved_threepids = []
+    config.admin_contact = None
+    config.rc_messages_per_second = 10000
+    config.rc_message_burst_count = 10000
+
+    # we need a sane default_room_version, otherwise attempts to create rooms will
+    # fail.
+    config.default_room_version = "1"
+
+    # disable user directory updates, because they get done in the
+    # background, which upsets the test runner.
+    config.update_user_directory = False
+
+    def is_threepid_reserved(threepid):
+        return ServerConfig.is_threepid_reserved(config, threepid)
+
+    config.is_threepid_reserved.side_effect = is_threepid_reserved
+
+    return config
+
+
 class TestHomeServer(HomeServer):
     DATASTORE_CLASS = DataStore
 
@@ -124,54 +180,7 @@ def setup_test_homeserver(
         from twisted.internet import reactor
 
     if config is None:
-        config = Mock()
-        config.signing_key = [MockKey()]
-        config.event_cache_size = 1
-        config.enable_registration = True
-        config.macaroon_secret_key = "not even a little secret"
-        config.expire_access_token = False
-        config.server_name = name
-        config.trusted_third_party_id_servers = []
-        config.room_invite_state_types = []
-        config.password_providers = []
-        config.worker_replication_url = ""
-        config.worker_app = None
-        config.email_enable_notifs = False
-        config.block_non_admin_invites = False
-        config.federation_domain_whitelist = None
-        config.federation_rc_reject_limit = 10
-        config.federation_rc_sleep_limit = 10
-        config.federation_rc_sleep_delay = 100
-        config.federation_rc_concurrent = 10
-        config.filter_timeline_limit = 5000
-        config.user_directory_search_all_users = False
-        config.user_consent_server_notice_content = None
-        config.block_events_without_consent_error = None
-        config.media_storage_providers = []
-        config.auto_join_rooms = []
-        config.limit_usage_by_mau = False
-        config.hs_disabled = False
-        config.hs_disabled_message = ""
-        config.hs_disabled_limit_type = ""
-        config.max_mau_value = 50
-        config.mau_trial_days = 0
-        config.mau_limits_reserved_threepids = []
-        config.admin_contact = None
-        config.rc_messages_per_second = 10000
-        config.rc_message_burst_count = 10000
-
-        # we need a sane default_room_version, otherwise attempts to create rooms will
-        # fail.
-        config.default_room_version = "1"
-
-        # disable user directory updates, because they get done in the
-        # background, which upsets the test runner.
-        config.update_user_directory = False
-
-        def is_threepid_reserved(threepid):
-            return ServerConfig.is_threepid_reserved(config, threepid)
-
-        config.is_threepid_reserved.side_effect = is_threepid_reserved
+        config = default_config(name)
 
     config.use_frozen_dicts = True
     config.ldap_enabled = False

From abdc141c2bc872e7db6761f652551eb1bbf0859d Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Tue, 2 Oct 2018 01:08:38 +1000
Subject: [PATCH 767/830] Update instructions to point to pip install (#3985)

---
 .circleci/merge_base_branch.sh | 13 ++++---
 README.rst                     | 70 +++++++++-------------------------
 UPGRADE.rst                    |  6 +--
 changelog.d/3985.misc          |  1 +
 4 files changed, 29 insertions(+), 61 deletions(-)
 create mode 100644 changelog.d/3985.misc

diff --git a/.circleci/merge_base_branch.sh b/.circleci/merge_base_branch.sh
index 9614eb91b6fb..6b0bf3aa48df 100755
--- a/.circleci/merge_base_branch.sh
+++ b/.circleci/merge_base_branch.sh
@@ -9,13 +9,16 @@ source $BASH_ENV
 
 if [[ -z "${CIRCLE_PR_NUMBER}" ]]
 then
-    echo "Can't figure out what the PR number is!"
-    exit 1
+    echo "Can't figure out what the PR number is! Assuming merge target is develop."
+
+    # It probably hasn't had a PR opened yet. Since all PRs land on develop, we
+    # can probably assume it's based on it and will be merged into it.
+    GITBASE="develop"
+else
+    # Get the reference, using the GitHub API
+    GITBASE=`curl -q https://api.github.com/repos/matrix-org/synapse/pulls/${CIRCLE_PR_NUMBER} | jq -r '.base.ref'`
 fi
 
-# Get the reference, using the GitHub API
-GITBASE=`curl -q https://api.github.com/repos/matrix-org/synapse/pulls/${CIRCLE_PR_NUMBER} | jq -r '.base.ref'`
-
 # Show what we are before
 git show -s
 
diff --git a/README.rst b/README.rst
index 5547f617ba23..e1ea351f847a 100644
--- a/README.rst
+++ b/README.rst
@@ -81,7 +81,7 @@ Thanks for using Matrix!
 Synapse Installation
 ====================
 
-Synapse is the reference python/twisted Matrix homeserver implementation.
+Synapse is the reference Python/Twisted Matrix homeserver implementation.
 
 System requirements:
 
@@ -91,12 +91,13 @@ System requirements:
 
 Installing from source
 ----------------------
+
 (Prebuilt packages are available for some platforms - see `Platform-Specific
 Instructions`_.)
 
-Synapse is written in python but some of the libraries it uses are written in
-C. So before we can install synapse itself we need a working C compiler and the
-header files for python C extensions.
+Synapse is written in Python but some of the libraries it uses are written in
+C. So before we can install Synapse itself we need a working C compiler and the
+header files for Python C extensions.
 
 Installing prerequisites on Ubuntu or Debian::
 
@@ -143,18 +144,24 @@ Installing prerequisites on OpenBSD::
     doas pkg_add python libffi py-pip py-setuptools sqlite3 py-virtualenv \
                  libxslt
 
-To install the synapse homeserver run::
+To install the Synapse homeserver run::
 
     virtualenv -p python2.7 ~/.synapse
     source ~/.synapse/bin/activate
     pip install --upgrade pip
     pip install --upgrade setuptools
-    pip install https://github.com/matrix-org/synapse/tarball/master
+    pip install matrix-synapse
 
-This installs synapse, along with the libraries it uses, into a virtual
+This installs Synapse, along with the libraries it uses, into a virtual
 environment under ``~/.synapse``.  Feel free to pick a different directory
 if you prefer.
 
+This Synapse installation can then be later upgraded by using pip again with the
+update flag::
+
+    source ~/.synapse/bin/activate
+    pip install -U matrix-synapse
+
 In case of problems, please see the _`Troubleshooting` section below.
 
 There is an offical synapse image available at
@@ -167,7 +174,7 @@ Alternatively, Andreas Peters (previously Silvio Fricke) has contributed a
 Dockerfile to automate a synapse server in a single Docker image, at
 https://hub.docker.com/r/avhost/docker-matrix/tags/
 
-Configuring synapse
+Configuring Synapse
 -------------------
 
 Before you can start Synapse, you will need to generate a configuration
@@ -249,26 +256,6 @@ Setting up a TURN server
 For reliable VoIP calls to be routed via this homeserver, you MUST configure
 a TURN server.  See `<docs/turn-howto.rst>`_ for details.
 
-IPv6
-----
-
-As of Synapse 0.19 we finally support IPv6, many thanks to @kyrias and @glyph
-for providing PR #1696.
-
-However, for federation to work on hosts with IPv6 DNS servers you **must**
-be running Twisted 17.1.0 or later - see https://github.com/matrix-org/synapse/issues/1002
-for details.  We can't make Synapse depend on Twisted 17.1 by default
-yet as it will break most older distributions (see https://github.com/matrix-org/synapse/pull/1909)
-so if you are using operating system dependencies you'll have to install your
-own Twisted 17.1 package via pip or backports etc.
-
-If you're running in a virtualenv then pip should have installed the newest
-Twisted automatically, but if your virtualenv is old you will need to manually
-upgrade to a newer Twisted dependency via:
-
-    pip install Twisted>=17.1.0
-
-
 Running Synapse
 ===============
 
@@ -444,8 +431,7 @@ settings require a slightly more difficult installation process.
    using the ``.`` command, rather than ``bash``'s ``source``.
 5) Optionally, use ``pip`` to install ``lxml``, which Synapse needs to parse
    webpages for their titles.
-6) Use ``pip`` to install this repository: ``pip install
-   https://github.com/matrix-org/synapse/tarball/master``
+6) Use ``pip`` to install this repository: ``pip install matrix-synapse``
 7) Optionally, change ``_synapse``'s shell to ``/bin/false`` to reduce the
    chance of a compromised Synapse server being used to take over your box.
 
@@ -473,7 +459,7 @@ Troubleshooting
 Troubleshooting Installation
 ----------------------------
 
-Synapse requires pip 1.7 or later, so if your OS provides too old a version you
+Synapse requires pip 8 or later, so if your OS provides too old a version you
 may need to manually upgrade it::
 
     sudo pip install --upgrade pip
@@ -508,28 +494,6 @@ failing, e.g.::
 
     pip install twisted
 
-On OS X, if you encounter clang: error: unknown argument: '-mno-fused-madd' you
-will need to export CFLAGS=-Qunused-arguments.
-
-Troubleshooting Running
------------------------
-
-If synapse fails with ``missing "sodium.h"`` crypto errors, you may need
-to manually upgrade PyNaCL, as synapse uses NaCl (https://nacl.cr.yp.to/) for
-encryption and digital signatures.
-Unfortunately PyNACL currently has a few issues
-(https://github.com/pyca/pynacl/issues/53) and
-(https://github.com/pyca/pynacl/issues/79) that mean it may not install
-correctly, causing all tests to fail with errors about missing "sodium.h". To
-fix try re-installing from PyPI or directly from
-(https://github.com/pyca/pynacl)::
-
-    # Install from PyPI
-    pip install --user --upgrade --force pynacl
-
-    # Install from github
-    pip install --user https://github.com/pyca/pynacl/tarball/master
-
 Running out of File Handles
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/UPGRADE.rst b/UPGRADE.rst
index f6bb1070b103..6cf3169f7575 100644
--- a/UPGRADE.rst
+++ b/UPGRADE.rst
@@ -18,7 +18,7 @@ instructions that may be required are listed later in this document.
 
    .. code:: bash
 
-       pip install --upgrade --process-dependency-links https://github.com/matrix-org/synapse/tarball/master
+       pip install --upgrade --process-dependency-links matrix-synapse
 
        # restart synapse
        synctl restart
@@ -48,11 +48,11 @@ returned by the Client-Server API:
     # configured on port 443.
     curl -kv https://<host.name>/_matrix/client/versions 2>&1 | grep "Server:"
 
-Upgrading to $NEXT_VERSION
+Upgrading to v0.27.3
 ====================
 
 This release expands the anonymous usage stats sent if the opt-in
-``report_stats`` configuration is set to ``true``. We now capture RSS memory 
+``report_stats`` configuration is set to ``true``. We now capture RSS memory
 and cpu use at a very coarse level. This requires administrators to install
 the optional ``psutil`` python module.
 
diff --git a/changelog.d/3985.misc b/changelog.d/3985.misc
new file mode 100644
index 000000000000..ba935caf3a40
--- /dev/null
+++ b/changelog.d/3985.misc
@@ -0,0 +1 @@
+Updated the installation instructions to point to the matrix-synapse package on PyPI.

From 8174c6725b5271923930432d1927dd39cff3547c Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 1 Oct 2018 18:48:51 +0100
Subject: [PATCH 768/830] Avoid reraise, to improve stacktraces

---
 changelog.d/3989.misc          |  1 +
 synapse/handlers/federation.py | 20 ++++++++++----------
 synapse/handlers/message.py    | 25 +++++++++++++------------
 3 files changed, 24 insertions(+), 22 deletions(-)
 create mode 100644 changelog.d/3989.misc

diff --git a/changelog.d/3989.misc b/changelog.d/3989.misc
new file mode 100644
index 000000000000..26700d168f01
--- /dev/null
+++ b/changelog.d/3989.misc
@@ -0,0 +1 @@
+Improve stacktraces in certain exceptions in the logs
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index d05b63673f45..45d955e6f545 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -18,7 +18,6 @@
 
 import itertools
 import logging
-import sys
 
 import six
 from six import iteritems, itervalues
@@ -1602,6 +1601,9 @@ def _handle_new_event(self, origin, event, state=None, auth_events=None,
             auth_events=auth_events,
         )
 
+        # reraise does not allow inlineCallbacks to preserve the stacktrace, so we
+        # hack around with a try/finally instead.
+        success = False
         try:
             if not event.internal_metadata.is_outlier() and not backfilled:
                 yield self.action_generator.handle_push_actions_for_event(
@@ -1612,15 +1614,13 @@ def _handle_new_event(self, origin, event, state=None, auth_events=None,
                 [(event, context)],
                 backfilled=backfilled,
             )
-        except:  # noqa: E722, as we reraise the exception this is fine.
-            tp, value, tb = sys.exc_info()
-
-            logcontext.run_in_background(
-                self.store.remove_push_actions_from_staging,
-                event.event_id,
-            )
-
-            six.reraise(tp, value, tb)
+            success = True
+        finally:
+            if not success:
+                logcontext.run_in_background(
+                    self.store.remove_push_actions_from_staging,
+                    event.event_id,
+                )
 
         defer.returnValue(context)
 
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index e484061cc0f5..4954b23a0d6c 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -14,9 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-import sys
 
-import six
 from six import iteritems, itervalues, string_types
 
 from canonicaljson import encode_canonical_json, json
@@ -624,6 +622,9 @@ def handle_new_client_event(
             event, context
         )
 
+        # reraise does not allow inlineCallbacks to preserve the stacktrace, so we
+        # hack around with a try/finally instead.
+        success = False
         try:
             # If we're a worker we need to hit out to the master.
             if self.config.worker_app:
@@ -636,6 +637,7 @@ def handle_new_client_event(
                     ratelimit=ratelimit,
                     extra_users=extra_users,
                 )
+                success = True
                 return
 
             yield self.persist_and_notify_client_event(
@@ -645,17 +647,16 @@ def handle_new_client_event(
                 ratelimit=ratelimit,
                 extra_users=extra_users,
             )
-        except:  # noqa: E722, as we reraise the exception this is fine.
-            # Ensure that we actually remove the entries in the push actions
-            # staging area, if we calculated them.
-            tp, value, tb = sys.exc_info()
-
-            run_in_background(
-                self.store.remove_push_actions_from_staging,
-                event.event_id,
-            )
 
-            six.reraise(tp, value, tb)
+            success = True
+        finally:
+            if not success:
+                # Ensure that we actually remove the entries in the push actions
+                # staging area, if we calculated them.
+                run_in_background(
+                    self.store.remove_push_actions_from_staging,
+                    event.event_id,
+                )
 
     @defer.inlineCallbacks
     def persist_and_notify_client_event(

From 334e075dd85a21e859eb89948b606c2710b932ad Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 2 Oct 2018 11:49:39 +0100
Subject: [PATCH 769/830] Fix error when logging incomplete requests

If a connection is lost before a request is read from Request, Twisted
sets `method` (and `uri`) attributes to dummy values. These dummy values
have incorrect types (i.e. they're not bytes), and so things like
`__repr__` would raise an exception.

To fix this we had a helper method to return the method with a
consistent type.
---
 synapse/http/site.py | 27 +++++++++++++++++++++------
 1 file changed, 21 insertions(+), 6 deletions(-)

diff --git a/synapse/http/site.py b/synapse/http/site.py
index 50be2de3bbc7..e508c0bd4f2d 100644
--- a/synapse/http/site.py
+++ b/synapse/http/site.py
@@ -75,14 +75,14 @@ def __repr__(self):
         return '<%s at 0x%x method=%r uri=%r clientproto=%r site=%r>' % (
             self.__class__.__name__,
             id(self),
-            self.method.decode('ascii', errors='replace'),
+            self.get_method(),
             self.get_redacted_uri(),
             self.clientproto.decode('ascii', errors='replace'),
             self.site.site_tag,
         )
 
     def get_request_id(self):
-        return "%s-%i" % (self.method.decode('ascii'), self.request_seq)
+        return "%s-%i" % (self.get_method(), self.request_seq)
 
     def get_redacted_uri(self):
         uri = self.uri
@@ -90,6 +90,21 @@ def get_redacted_uri(self):
             uri = self.uri.decode('ascii')
         return redact_uri(uri)
 
+    def get_method(self):
+        """Gets the method associated with the request (or placeholder if not
+        method has yet been received).
+
+        Note: This is necessary as the placeholder value in twisted is str
+        rather than bytes, so we need to sanitise `self.method`.
+
+        Returns:
+            str
+        """
+        method = self.method
+        if isinstance(method, bytes):
+            method = self.method.decode('ascii')
+        return method
+
     def get_user_agent(self):
         return self.requestHeaders.getRawHeaders(b"User-Agent", [None])[-1]
 
@@ -119,7 +134,7 @@ def render(self, resrc):
             # dispatching to the handler, so that the handler
             # can update the servlet name in the request
             # metrics
-            requests_counter.labels(self.method.decode('ascii'),
+            requests_counter.labels(self.get_method(),
                                     self.request_metrics.name).inc()
 
     @contextlib.contextmanager
@@ -207,14 +222,14 @@ def _started_processing(self, servlet_name):
         self.start_time = time.time()
         self.request_metrics = RequestMetrics()
         self.request_metrics.start(
-            self.start_time, name=servlet_name, method=self.method.decode('ascii'),
+            self.start_time, name=servlet_name, method=self.get_method(),
         )
 
         self.site.access_logger.info(
             "%s - %s - Received request: %s %s",
             self.getClientIP(),
             self.site.site_tag,
-            self.method.decode('ascii'),
+            self.get_method(),
             self.get_redacted_uri()
         )
 
@@ -280,7 +295,7 @@ def _finished_processing(self):
             int(usage.db_txn_count),
             self.sentLength,
             code,
-            self.method.decode('ascii'),
+            self.get_method(),
             self.get_redacted_uri(),
             self.clientproto.decode('ascii', errors='replace'),
             user_agent,

From eeb755d17b5a38a3fb9f4648c6c56962efd58500 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 2 Oct 2018 12:01:51 +0100
Subject: [PATCH 770/830] Newsfile

---
 changelog.d/3990.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3990.bugfix

diff --git a/changelog.d/3990.bugfix b/changelog.d/3990.bugfix
new file mode 100644
index 000000000000..51ea105a64f3
--- /dev/null
+++ b/changelog.d/3990.bugfix
@@ -0,0 +1 @@
+Fix error when logging incomplete HTTP requests

From 7232917f12460efa2e3579331b6a97f046afe90b Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Tue, 2 Oct 2018 22:53:47 +1000
Subject: [PATCH 771/830] Disable frozen dicts by default (#3987)

---
 changelog.d/3987.misc                          |  1 +
 synapse/events/__init__.py                     | 13 ++++++++++---
 tests/replication/slave/storage/test_events.py |  6 +++++-
 tests/utils.py                                 |  3 ++-
 4 files changed, 18 insertions(+), 5 deletions(-)
 create mode 100644 changelog.d/3987.misc

diff --git a/changelog.d/3987.misc b/changelog.d/3987.misc
new file mode 100644
index 000000000000..d6b501621103
--- /dev/null
+++ b/changelog.d/3987.misc
@@ -0,0 +1 @@
+Disable USE_FROZEN_DICTS for unittests by default.
diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py
index b782af630807..12f1eb0a3ec4 100644
--- a/synapse/events/__init__.py
+++ b/synapse/events/__init__.py
@@ -13,15 +13,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
+from distutils.util import strtobool
+
 import six
 
 from synapse.util.caches import intern_dict
 from synapse.util.frozenutils import freeze
 
 # Whether we should use frozen_dict in FrozenEvent. Using frozen_dicts prevents
-# bugs where we accidentally share e.g. signature dicts. However, converting
-# a dict to frozen_dicts is expensive.
-USE_FROZEN_DICTS = True
+# bugs where we accidentally share e.g. signature dicts. However, converting a
+# dict to frozen_dicts is expensive.
+#
+# NOTE: This is overridden by the configuration by the Synapse worker apps, but
+# for the sake of tests, it is set here while it cannot be configured on the
+# homeserver object itself.
+USE_FROZEN_DICTS = strtobool(os.environ.get("SYNAPSE_USE_FROZEN_DICTS", "0"))
 
 
 class _EventInternalMetadata(object):
diff --git a/tests/replication/slave/storage/test_events.py b/tests/replication/slave/storage/test_events.py
index db44d33c68cb..41be5d5a1a1d 100644
--- a/tests/replication/slave/storage/test_events.py
+++ b/tests/replication/slave/storage/test_events.py
@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from canonicaljson import encode_canonical_json
+
 from synapse.events import FrozenEvent, _EventInternalMetadata
 from synapse.events.snapshot import EventContext
 from synapse.replication.slave.storage.events import SlavedEventStore
@@ -26,7 +28,9 @@
 
 
 def dict_equals(self, other):
-    return self.__dict__ == other.__dict__
+    me = encode_canonical_json(self._event_dict)
+    them = encode_canonical_json(other._event_dict)
+    return me == them
 
 
 def patch__eq__(cls):
diff --git a/tests/utils.py b/tests/utils.py
index 1ef80e7b79c6..dd347a0c5986 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -136,6 +136,8 @@ def default_config(name):
     config.rc_messages_per_second = 10000
     config.rc_message_burst_count = 10000
 
+    config.use_frozen_dicts = False
+
     # we need a sane default_room_version, otherwise attempts to create rooms will
     # fail.
     config.default_room_version = "1"
@@ -182,7 +184,6 @@ def setup_test_homeserver(
     if config is None:
         config = default_config(name)
 
-    config.use_frozen_dicts = True
     config.ldap_enabled = False
 
     if "clock" not in kargs:

From 7258d081a5523040dddf5a99ff745acf4fe238bf Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 2 Oct 2018 15:47:57 +0100
Subject: [PATCH 772/830] Fix bug when invalidating destination retry timings

---
 synapse/storage/transactions.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py
index ab54977a7592..a3032cdce914 100644
--- a/synapse/storage/transactions.py
+++ b/synapse/storage/transactions.py
@@ -216,7 +216,7 @@ def set_destination_retry_timings(self, destination,
             retry_interval (int) - how long until next retry in ms
         """
 
-        self._destination_retry_cache.pop(destination)
+        self._destination_retry_cache.pop(destination, None)
         return self.runInteraction(
             "set_destination_retry_timings",
             self._set_destination_retry_timings,

From 8f614f842d3b3e1de868c3e28b9932ed21f48f77 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 2 Oct 2018 15:49:39 +0100
Subject: [PATCH 773/830] Newsfile

---
 changelog.d/3991.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3991.bugfix

diff --git a/changelog.d/3991.bugfix b/changelog.d/3991.bugfix
new file mode 100644
index 000000000000..6545871f55eb
--- /dev/null
+++ b/changelog.d/3991.bugfix
@@ -0,0 +1 @@
+Add a cache to get_destination_retry_timings

From 058a4c665e55a8654d253bf9cafaf814b465fbd9 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Wed, 3 Oct 2018 00:59:11 +1000
Subject: [PATCH 774/830] Remove Jenkins & other old dev junk (#3988)

---
 MANIFEST.in                         |  7 ++---
 MAP.rst                             | 35 -----------------------
 changelog.d/3988.misc               |  1 +
 jenkins-dendron-haproxy-postgres.sh | 23 ---------------
 jenkins-dendron-postgres.sh         | 20 -------------
 jenkins-flake8.sh                   | 22 ---------------
 jenkins-postgres.sh                 | 18 ------------
 jenkins-sqlite.sh                   | 16 -----------
 jenkins-unittests.sh                | 30 --------------------
 jenkins/clone.sh                    | 44 -----------------------------
 scripts-dev/copyrighter-sql.pl      | 33 ----------------------
 scripts-dev/copyrighter.pl          | 33 ----------------------
 12 files changed, 4 insertions(+), 278 deletions(-)
 delete mode 100644 MAP.rst
 create mode 100644 changelog.d/3988.misc
 delete mode 100755 jenkins-dendron-haproxy-postgres.sh
 delete mode 100755 jenkins-dendron-postgres.sh
 delete mode 100755 jenkins-flake8.sh
 delete mode 100755 jenkins-postgres.sh
 delete mode 100755 jenkins-sqlite.sh
 delete mode 100755 jenkins-unittests.sh
 delete mode 100755 jenkins/clone.sh
 delete mode 100755 scripts-dev/copyrighter-sql.pl
 delete mode 100755 scripts-dev/copyrighter.pl

diff --git a/MANIFEST.in b/MANIFEST.in
index 47ae5a77b9d2..c6a37ac685a7 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -23,13 +23,9 @@ recursive-include synapse/static *.gif
 recursive-include synapse/static *.html
 recursive-include synapse/static *.js
 
-exclude jenkins.sh
-exclude jenkins*.sh
-exclude jenkins*
 exclude Dockerfile
 exclude .dockerignore
 exclude test_postgresql.sh
-recursive-exclude jenkins *.sh
 
 include pyproject.toml
 recursive-include changelog.d *
@@ -38,3 +34,6 @@ prune .github
 prune demo/etc
 prune docker
 prune .circleci
+
+exclude jenkins*
+recursive-exclude jenkins *.sh
diff --git a/MAP.rst b/MAP.rst
deleted file mode 100644
index 0f8e9818a88d..000000000000
--- a/MAP.rst
+++ /dev/null
@@ -1,35 +0,0 @@
-Directory Structure
-===================
-
-Warning: this may be a bit stale...
-
-::
-
-    .
-    ├── cmdclient           Basic CLI python Matrix client
-    ├── demo                Scripts for running standalone Matrix demos
-    ├── docs                All doc, including the draft Matrix API spec
-    │   ├── client-server       The client-server Matrix API spec
-    │   ├── model               Domain-specific elements of the Matrix API spec
-    │   ├── server-server       The server-server model of the Matrix API spec
-    │   └── sphinx              The internal API doc of the Synapse homeserver
-    ├── experiments         Early experiments of using Synapse's internal APIs
-    ├── graph               Visualisation of Matrix's distributed message store 
-    ├── synapse             The reference Matrix homeserver implementation
-    │   ├── api                 Common building blocks for the APIs
-    │   │   ├── events              Definition of state representation Events 
-    │   │   └── streams             Definition of streamable Event objects
-    │   ├── app                 The __main__ entry point for the homeserver
-    │   ├── crypto              The PKI client/server used for secure federation
-    │   │   └── resource            PKI helper objects (e.g. keys)
-    │   ├── federation          Server-server state replication logic
-    │   ├── handlers            The main business logic of the homeserver
-    │   ├── http                Wrappers around Twisted's HTTP server & client
-    │   ├── rest                Servlet-style RESTful API
-    │   ├── storage             Persistence subsystem (currently only sqlite3)
-    │   │   └── schema              sqlite persistence schema
-    │   └── util                Synapse-specific utilities
-    ├── tests               Unit tests for the Synapse homeserver
-    └── webclient           Basic AngularJS Matrix web client
-
-
diff --git a/changelog.d/3988.misc b/changelog.d/3988.misc
new file mode 100644
index 000000000000..10379d183bea
--- /dev/null
+++ b/changelog.d/3988.misc
@@ -0,0 +1 @@
+Remove unused Jenkins and development related files from the repo.
diff --git a/jenkins-dendron-haproxy-postgres.sh b/jenkins-dendron-haproxy-postgres.sh
deleted file mode 100755
index 07979bf8b825..000000000000
--- a/jenkins-dendron-haproxy-postgres.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/bin/bash
-
-set -eux
-
-: ${WORKSPACE:="$(pwd)"}
-
-export WORKSPACE
-export PYTHONDONTWRITEBYTECODE=yep
-export SYNAPSE_CACHE_FACTOR=1
-
-export HAPROXY_BIN=/home/haproxy/haproxy-1.6.11/haproxy
-
-./jenkins/prepare_synapse.sh
-./jenkins/clone.sh sytest https://github.com/matrix-org/sytest.git
-./jenkins/clone.sh dendron https://github.com/matrix-org/dendron.git
-./dendron/jenkins/build_dendron.sh
-./sytest/jenkins/prep_sytest_for_postgres.sh
-
-./sytest/jenkins/install_and_run.sh \
-    --python $WORKSPACE/.tox/py27/bin/python \
-    --synapse-directory $WORKSPACE \
-    --dendron $WORKSPACE/dendron/bin/dendron \
-    --haproxy \
diff --git a/jenkins-dendron-postgres.sh b/jenkins-dendron-postgres.sh
deleted file mode 100755
index 3b932fe34023..000000000000
--- a/jenkins-dendron-postgres.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/bash
-
-set -eux
-
-: ${WORKSPACE:="$(pwd)"}
-
-export WORKSPACE
-export PYTHONDONTWRITEBYTECODE=yep
-export SYNAPSE_CACHE_FACTOR=1
-
-./jenkins/prepare_synapse.sh
-./jenkins/clone.sh sytest https://github.com/matrix-org/sytest.git
-./jenkins/clone.sh dendron https://github.com/matrix-org/dendron.git
-./dendron/jenkins/build_dendron.sh
-./sytest/jenkins/prep_sytest_for_postgres.sh
-
-./sytest/jenkins/install_and_run.sh \
-    --python $WORKSPACE/.tox/py27/bin/python \
-    --synapse-directory $WORKSPACE \
-    --dendron $WORKSPACE/dendron/bin/dendron \
diff --git a/jenkins-flake8.sh b/jenkins-flake8.sh
deleted file mode 100755
index 11f1cab6c863..000000000000
--- a/jenkins-flake8.sh
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/bin/bash
-
-set -eux
-
-: ${WORKSPACE:="$(pwd)"}
-
-export PYTHONDONTWRITEBYTECODE=yep
-export SYNAPSE_CACHE_FACTOR=1
-
-# Output test results as junit xml
-export TRIAL_FLAGS="--reporter=subunit"
-export TOXSUFFIX="| subunit-1to2 | subunit2junitxml --no-passthrough --output-to=results.xml"
-# Write coverage reports to a separate file for each process
-export COVERAGE_OPTS="-p"
-export DUMP_COVERAGE_COMMAND="coverage help"
-
-# Output flake8 violations to violations.flake8.log
-export PEP8SUFFIX="--output-file=violations.flake8.log"
-
-rm .coverage* || echo "No coverage files to remove"
-
-tox -e packaging -e pep8
diff --git a/jenkins-postgres.sh b/jenkins-postgres.sh
deleted file mode 100755
index 1afb736394e4..000000000000
--- a/jenkins-postgres.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/bin/bash
-
-set -eux
-
-: ${WORKSPACE:="$(pwd)"}
-
-export WORKSPACE
-export PYTHONDONTWRITEBYTECODE=yep
-export SYNAPSE_CACHE_FACTOR=1
-
-./jenkins/prepare_synapse.sh
-./jenkins/clone.sh sytest https://github.com/matrix-org/sytest.git
-
-./sytest/jenkins/prep_sytest_for_postgres.sh
-
-./sytest/jenkins/install_and_run.sh \
-    --python $WORKSPACE/.tox/py27/bin/python \
-    --synapse-directory $WORKSPACE \
diff --git a/jenkins-sqlite.sh b/jenkins-sqlite.sh
deleted file mode 100755
index baf4713a01d6..000000000000
--- a/jenkins-sqlite.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/bash
-
-set -eux
-
-: ${WORKSPACE:="$(pwd)"}
-
-export WORKSPACE
-export PYTHONDONTWRITEBYTECODE=yep
-export SYNAPSE_CACHE_FACTOR=1
-
-./jenkins/prepare_synapse.sh
-./jenkins/clone.sh sytest https://github.com/matrix-org/sytest.git
-
-./sytest/jenkins/install_and_run.sh \
-    --python $WORKSPACE/.tox/py27/bin/python \
-    --synapse-directory $WORKSPACE \
diff --git a/jenkins-unittests.sh b/jenkins-unittests.sh
deleted file mode 100755
index 4c2f103e80a1..000000000000
--- a/jenkins-unittests.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-set -eux
-
-: ${WORKSPACE:="$(pwd)"}
-
-export PYTHONDONTWRITEBYTECODE=yep
-export SYNAPSE_CACHE_FACTOR=1
-
-# Output test results as junit xml
-export TRIAL_FLAGS="--reporter=subunit"
-export TOXSUFFIX="| subunit-1to2 | subunit2junitxml --no-passthrough --output-to=results.xml"
-# Write coverage reports to a separate file for each process
-export COVERAGE_OPTS="-p"
-export DUMP_COVERAGE_COMMAND="coverage help"
-
-# Output flake8 violations to violations.flake8.log
-# Don't exit with non-0 status code on Jenkins,
-# so that the build steps continue and a later step can decided whether to
-# UNSTABLE or FAILURE this build.
-export PEP8SUFFIX="--output-file=violations.flake8.log || echo flake8 finished with status code \$?"
-
-rm .coverage* || echo "No coverage files to remove"
-
-tox --notest -e py27
-TOX_BIN=$WORKSPACE/.tox/py27/bin
-python synapse/python_dependencies.py | xargs -n1 $TOX_BIN/pip install
-$TOX_BIN/pip install lxml
-
-tox -e py27
diff --git a/jenkins/clone.sh b/jenkins/clone.sh
deleted file mode 100755
index ab30ac778237..000000000000
--- a/jenkins/clone.sh
+++ /dev/null
@@ -1,44 +0,0 @@
-#! /bin/bash
-
-# This clones a project from github into a named subdirectory
-# If the project has a branch with the same name as this branch
-# then it will checkout that branch after cloning.
-# Otherwise it will checkout "origin/develop."
-# The first argument is the name of the directory to checkout
-# the branch into.
-# The second argument is the URL of the remote repository to checkout.
-# Usually something like https://github.com/matrix-org/sytest.git
-
-set -eux
-
-NAME=$1
-PROJECT=$2
-BASE=".$NAME-base"
-
-# Update our mirror.
-if [ ! -d ".$NAME-base" ]; then
-  # Create a local mirror of the source repository.
-  # This saves us from having to download the entire repository
-  # when this script is next run.
-  git clone "$PROJECT" "$BASE" --mirror
-else
-  # Fetch any updates from the source repository.
-  (cd "$BASE"; git fetch -p)
-fi
-
-# Remove the existing repository so that we have a clean copy
-rm -rf "$NAME"
-# Cloning with --shared means that we will share portions of the
-# .git directory with our local mirror.
-git clone "$BASE" "$NAME" --shared
-
-# Jenkins may have supplied us with the name of the branch in the
-# environment. Otherwise we will have to guess based on the current
-# commit.
-: ${GIT_BRANCH:="origin/$(git rev-parse --abbrev-ref HEAD)"}
-cd "$NAME"
-# check out the relevant branch
-git checkout "${GIT_BRANCH}" || (
-    echo >&2 "No ref ${GIT_BRANCH} found, falling back to develop"
-    git checkout "origin/develop"
-)
diff --git a/scripts-dev/copyrighter-sql.pl b/scripts-dev/copyrighter-sql.pl
deleted file mode 100755
index 13e630fc1195..000000000000
--- a/scripts-dev/copyrighter-sql.pl
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/usr/bin/perl -pi
-# Copyright 2014-2016 OpenMarket Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-$copyright = <<EOT;
-/* Copyright 2016 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-EOT
-
-s/^(# -\*- coding: utf-8 -\*-\n)?/$1$copyright/ if ($. == 1);
diff --git a/scripts-dev/copyrighter.pl b/scripts-dev/copyrighter.pl
deleted file mode 100755
index 03656f697a9c..000000000000
--- a/scripts-dev/copyrighter.pl
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/usr/bin/perl -pi
-# Copyright 2014-2016 OpenMarket Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-$copyright = <<EOT;
-# Copyright 2016 OpenMarket Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-EOT
-
-s/^(# -\*- coding: utf-8 -\*-\n)?/$1$copyright/ if ($. == 1);

From 9c834b8ee9e9755aef6eed1a169b07c4b281d912 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 2 Oct 2018 16:22:39 +0100
Subject: [PATCH 775/830] Add tests

---
 tests/storage/test_transactions.py | 45 ++++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)
 create mode 100644 tests/storage/test_transactions.py

diff --git a/tests/storage/test_transactions.py b/tests/storage/test_transactions.py
new file mode 100644
index 000000000000..14169afa96c1
--- /dev/null
+++ b/tests/storage/test_transactions.py
@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from tests.unittest import HomeserverTestCase
+
+
+class TransactionStoreTestCase(HomeserverTestCase):
+    def prepare(self, reactor, clock, homeserver):
+        self.store = homeserver.get_datastore()
+
+    def test_get_set_transactions(self):
+        """Tests that we can successfully get a non-existent entry for
+        destination retries, as well as testing tht we can set and get
+        correctly.
+        """
+        d = self.store.get_destination_retry_timings("example.com")
+        r = self.get_success(d)
+        self.assertIsNone(r)
+
+        d = self.store.set_destination_retry_timings("example.com", 50, 100)
+        self.get_success(d)
+
+        d = self.store.get_destination_retry_timings("example.com")
+        r = self.get_success(d)
+
+        self.assert_dict({"retry_last_ts": 50, "retry_interval": 100}, r)
+
+    def test_initial_set_transactions(self):
+        """Tests that we can successfully set the destination retries (there
+        was a bug around invalidating the cache that broke this)
+        """
+        d = self.store.set_destination_retry_timings("example.com", 50, 100)
+        self.get_success(d)

From da8f82008d73407896c3dd902740af0d85d38d0a Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Wed, 3 Oct 2018 02:15:48 +1000
Subject: [PATCH 776/830] version

---
 synapse/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/__init__.py b/synapse/__init__.py
index b1f7a89fba5e..312e4e2fa61e 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -27,4 +27,4 @@
 except ImportError:
     pass
 
-__version__ = "0.33.5.1"
+__version__ = "0.33.6rc1"

From ccce6f26a651005e9d56fadafebcb90231213f89 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Wed, 3 Oct 2018 02:16:21 +1000
Subject: [PATCH 777/830] changelog

---
 CHANGES.md               | 75 ++++++++++++++++++++++++++++++++++++++++
 changelog.d/3578.bugfix  |  1 -
 changelog.d/3699.misc    |  2 --
 changelog.d/3794.misc    |  1 -
 changelog.d/3836.bugfix  |  1 -
 changelog.d/3868.bugfix  |  1 -
 changelog.d/3873.misc    |  2 --
 changelog.d/3879.bugfix  |  1 -
 changelog.d/3883.feature |  1 -
 changelog.d/3889.bugfix  |  1 -
 changelog.d/3892.bugfix  |  1 -
 changelog.d/3894.feature |  1 -
 changelog.d/3895.bugfix  |  1 -
 changelog.d/3897.misc    |  1 -
 changelog.d/3899.bugfix  |  1 -
 changelog.d/3903.misc    |  1 -
 changelog.d/3904.misc    |  1 -
 changelog.d/3906.misc    |  1 -
 changelog.d/3907.bugfix  |  1 -
 changelog.d/3908.bugfix  |  1 -
 changelog.d/3909.misc    |  1 -
 changelog.d/3910.bugfix  |  1 -
 changelog.d/3911.misc    |  1 -
 changelog.d/3912.misc    |  1 -
 changelog.d/3914.bugfix  |  1 -
 changelog.d/3916.feature |  1 -
 changelog.d/3924.misc    |  1 -
 changelog.d/3925.misc    |  1 -
 changelog.d/3927.misc    |  1 -
 changelog.d/3932.bugfix  |  1 -
 changelog.d/3933.misc    |  1 -
 changelog.d/3936.bugfix  |  1 -
 changelog.d/3938.bugfix  |  1 -
 changelog.d/3946.misc    |  1 -
 changelog.d/3947.misc    |  1 -
 changelog.d/3948.misc    |  1 -
 changelog.d/3952.misc    |  1 -
 changelog.d/3956.bugfix  |  1 -
 changelog.d/3957.misc    |  1 -
 changelog.d/3958.misc    |  1 -
 changelog.d/3959.feature |  1 -
 changelog.d/3960.bugfix  |  1 -
 changelog.d/3961.bugfix  |  1 -
 changelog.d/3963.misc    |  1 -
 changelog.d/3964.feature |  1 -
 changelog.d/3965.misc    |  1 -
 changelog.d/3966.misc    |  1 -
 changelog.d/3967.misc    |  1 -
 changelog.d/3968.bugfix  |  1 -
 changelog.d/3970.bugfix  |  1 -
 changelog.d/3972.misc    |  1 -
 changelog.d/3976.misc    |  1 -
 changelog.d/3980.bugfix  |  1 -
 changelog.d/3985.misc    |  1 -
 changelog.d/3986.bugfix  |  1 -
 changelog.d/3987.misc    |  1 -
 changelog.d/3988.misc    |  1 -
 changelog.d/3989.misc    |  1 -
 changelog.d/3990.bugfix  |  1 -
 changelog.d/3991.bugfix  |  1 -
 60 files changed, 75 insertions(+), 61 deletions(-)
 delete mode 100644 changelog.d/3578.bugfix
 delete mode 100644 changelog.d/3699.misc
 delete mode 100644 changelog.d/3794.misc
 delete mode 100644 changelog.d/3836.bugfix
 delete mode 100644 changelog.d/3868.bugfix
 delete mode 100644 changelog.d/3873.misc
 delete mode 100644 changelog.d/3879.bugfix
 delete mode 100644 changelog.d/3883.feature
 delete mode 100644 changelog.d/3889.bugfix
 delete mode 100644 changelog.d/3892.bugfix
 delete mode 100644 changelog.d/3894.feature
 delete mode 100644 changelog.d/3895.bugfix
 delete mode 100644 changelog.d/3897.misc
 delete mode 100644 changelog.d/3899.bugfix
 delete mode 100644 changelog.d/3903.misc
 delete mode 100644 changelog.d/3904.misc
 delete mode 100644 changelog.d/3906.misc
 delete mode 100644 changelog.d/3907.bugfix
 delete mode 100644 changelog.d/3908.bugfix
 delete mode 100644 changelog.d/3909.misc
 delete mode 100644 changelog.d/3910.bugfix
 delete mode 100644 changelog.d/3911.misc
 delete mode 100644 changelog.d/3912.misc
 delete mode 100644 changelog.d/3914.bugfix
 delete mode 100644 changelog.d/3916.feature
 delete mode 100644 changelog.d/3924.misc
 delete mode 100644 changelog.d/3925.misc
 delete mode 100644 changelog.d/3927.misc
 delete mode 100644 changelog.d/3932.bugfix
 delete mode 100644 changelog.d/3933.misc
 delete mode 100644 changelog.d/3936.bugfix
 delete mode 100644 changelog.d/3938.bugfix
 delete mode 100644 changelog.d/3946.misc
 delete mode 100644 changelog.d/3947.misc
 delete mode 100644 changelog.d/3948.misc
 delete mode 100644 changelog.d/3952.misc
 delete mode 100644 changelog.d/3956.bugfix
 delete mode 100644 changelog.d/3957.misc
 delete mode 100644 changelog.d/3958.misc
 delete mode 100644 changelog.d/3959.feature
 delete mode 100644 changelog.d/3960.bugfix
 delete mode 100644 changelog.d/3961.bugfix
 delete mode 100644 changelog.d/3963.misc
 delete mode 100644 changelog.d/3964.feature
 delete mode 100644 changelog.d/3965.misc
 delete mode 100644 changelog.d/3966.misc
 delete mode 100644 changelog.d/3967.misc
 delete mode 100644 changelog.d/3968.bugfix
 delete mode 100644 changelog.d/3970.bugfix
 delete mode 100644 changelog.d/3972.misc
 delete mode 100644 changelog.d/3976.misc
 delete mode 100644 changelog.d/3980.bugfix
 delete mode 100644 changelog.d/3985.misc
 delete mode 100644 changelog.d/3986.bugfix
 delete mode 100644 changelog.d/3987.misc
 delete mode 100644 changelog.d/3988.misc
 delete mode 100644 changelog.d/3989.misc
 delete mode 100644 changelog.d/3990.bugfix
 delete mode 100644 changelog.d/3991.bugfix

diff --git a/CHANGES.md b/CHANGES.md
index 45d3cdb1314b..e9b491559e29 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,78 @@
+Synapse 0.33.6rc1 (2018-10-03)
+==============================
+
+Features
+--------
+
+- Adding the ability to change MAX_UPLOAD_SIZE for the docker container variables. ([\#3883](https://github.com/matrix-org/synapse/issues/3883))
+- Report "python_version" in the phone home stats ([\#3894](https://github.com/matrix-org/synapse/issues/3894))
+- Always LL ourselves if we're in a room ([\#3916](https://github.com/matrix-org/synapse/issues/3916))
+- Include eventid in log lines when processing incoming federation transactions ([\#3959](https://github.com/matrix-org/synapse/issues/3959))
+- Remove spurious check which made 'localhost' servers not work ([\#3964](https://github.com/matrix-org/synapse/issues/3964))
+
+
+Bugfixes
+--------
+
+- Fix problem when playing media from Chrome using direct URL (thanks @remjey!) ([\#3578](https://github.com/matrix-org/synapse/issues/3578))
+- support registering regular users non-interactively with register_new_matrix_user script ([\#3836](https://github.com/matrix-org/synapse/issues/3836))
+- Fix broken invite email links for self hosted riots ([\#3868](https://github.com/matrix-org/synapse/issues/3868))
+- Don't ratelimit autojoins ([\#3879](https://github.com/matrix-org/synapse/issues/3879))
+- Fix 500 error when deleting unknown room alias ([\#3889](https://github.com/matrix-org/synapse/issues/3889))
+- Fix some b'abcd' noise in logs and metrics ([\#3892](https://github.com/matrix-org/synapse/issues/3892), [\#3895](https://github.com/matrix-org/synapse/issues/3895))
+- When we join a room, always try the server we used for the alias lookup first, to avoid unresponsive and out-of-date servers. ([\#3899](https://github.com/matrix-org/synapse/issues/3899))
+- Fix incorrect server-name indication for outgoing federation requests ([\#3907](https://github.com/matrix-org/synapse/issues/3907))
+- Fix adding client IPs to the database failing on Python 3. ([\#3908](https://github.com/matrix-org/synapse/issues/3908))
+- Fix bug where things occaisonally were not being timed out correctly. ([\#3910](https://github.com/matrix-org/synapse/issues/3910))
+- Fix bug where outbound federation would stop talking to some servers when using workers ([\#3914](https://github.com/matrix-org/synapse/issues/3914))
+- Fix some instances of ExpiringCache not expiring cache items ([\#3932](https://github.com/matrix-org/synapse/issues/3932), [\#3980](https://github.com/matrix-org/synapse/issues/3980))
+- Fix out-of-bounds error when LLing yourself ([\#3936](https://github.com/matrix-org/synapse/issues/3936))
+- Sending server notices regarding user consent now works on Python 3. ([\#3938](https://github.com/matrix-org/synapse/issues/3938))
+- Fix exceptions from metrics handler ([\#3956](https://github.com/matrix-org/synapse/issues/3956))
+- Fix error message for events with m.room.create missing from auth_events ([\#3960](https://github.com/matrix-org/synapse/issues/3960))
+- Fix errors due to concurrent monthly_active_user upserts ([\#3961](https://github.com/matrix-org/synapse/issues/3961))
+- Fix exceptions when processing incoming events over federation ([\#3968](https://github.com/matrix-org/synapse/issues/3968))
+- Replaced all occurences of e.message with str(e). Contributed by Schnuffle ([\#3970](https://github.com/matrix-org/synapse/issues/3970))
+- Fix lazy loaded sync in the presence of rejected state events ([\#3986](https://github.com/matrix-org/synapse/issues/3986))
+- Fix error when logging incomplete HTTP requests ([\#3990](https://github.com/matrix-org/synapse/issues/3990))
+- Add a cache to get_destination_retry_timings ([\#3991](https://github.com/matrix-org/synapse/issues/3991))
+
+
+Internal Changes
+----------------
+
+- Unit tests can now be run under PostgreSQL in Docker using 
+  ``test_postgresql.sh``. ([\#3699](https://github.com/matrix-org/synapse/issues/3699))
+- Speed up calculation of typing updates for replication ([\#3794](https://github.com/matrix-org/synapse/issues/3794))
+- Remove documentation regarding installation on Cygwin, the use of WSL is 
+  recommended instead. ([\#3873](https://github.com/matrix-org/synapse/issues/3873))
+- Fix typo in README, synaspse -> synapse ([\#3897](https://github.com/matrix-org/synapse/issues/3897))
+- Increase the timeout when filling missing events in federation requests ([\#3903](https://github.com/matrix-org/synapse/issues/3903))
+- Improve the logging when handling a federation transaction ([\#3904](https://github.com/matrix-org/synapse/issues/3904), [\#3966](https://github.com/matrix-org/synapse/issues/3966))
+- Improve logging of outbound federation requests ([\#3906](https://github.com/matrix-org/synapse/issues/3906), [\#3909](https://github.com/matrix-org/synapse/issues/3909))
+- Fix the docker image building on python 3 ([\#3911](https://github.com/matrix-org/synapse/issues/3911))
+- Add a regression test for logging failed HTTP requests on Python 3. ([\#3912](https://github.com/matrix-org/synapse/issues/3912))
+- Comments and interface cleanup for on_receive_pdu ([\#3924](https://github.com/matrix-org/synapse/issues/3924))
+- Fix spurious exceptions when remote http client closes conncetion ([\#3925](https://github.com/matrix-org/synapse/issues/3925))
+- Log exceptions thrown by background tasks ([\#3927](https://github.com/matrix-org/synapse/issues/3927))
+- Add a cache to get_destination_retry_timings ([\#3933](https://github.com/matrix-org/synapse/issues/3933))
+- Automate pushes to docker hub ([\#3946](https://github.com/matrix-org/synapse/issues/3946))
+- Require attrs 16.0.0 or later ([\#3947](https://github.com/matrix-org/synapse/issues/3947))
+- Fix incompatibility with python3 on alpine ([\#3948](https://github.com/matrix-org/synapse/issues/3948))
+- Run the test suite on the oldest supported versions of our dependencies in CI. ([\#3952](https://github.com/matrix-org/synapse/issues/3952))
+- CircleCI now only runs merged jobs on PRs, and commit jobs on develop, master, and release branches. ([\#3957](https://github.com/matrix-org/synapse/issues/3957))
+- Fix docstrings and add tests for state store methods ([\#3958](https://github.com/matrix-org/synapse/issues/3958))
+- fix docstring for FederationClient.get_state_for_room ([\#3963](https://github.com/matrix-org/synapse/issues/3963))
+- Run notify_app_services as a bg process ([\#3965](https://github.com/matrix-org/synapse/issues/3965))
+- Clarifications in FederationHandler ([\#3967](https://github.com/matrix-org/synapse/issues/3967))
+- Further reduce the docker image size ([\#3972](https://github.com/matrix-org/synapse/issues/3972))
+- Build py3 docker images for docker hub too ([\#3976](https://github.com/matrix-org/synapse/issues/3976))
+- Updated the installation instructions to point to the matrix-synapse package on PyPI. ([\#3985](https://github.com/matrix-org/synapse/issues/3985))
+- Disable USE_FROZEN_DICTS for unittests by default. ([\#3987](https://github.com/matrix-org/synapse/issues/3987))
+- Remove unused Jenkins and development related files from the repo. ([\#3988](https://github.com/matrix-org/synapse/issues/3988))
+- Improve stacktraces in certain exceptions in the logs ([\#3989](https://github.com/matrix-org/synapse/issues/3989))
+
+
 Synapse 0.33.5.1 (2018-09-25)
 =============================
 
diff --git a/changelog.d/3578.bugfix b/changelog.d/3578.bugfix
deleted file mode 100644
index 9c52b6fa7e92..000000000000
--- a/changelog.d/3578.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix problem when playing media from Chrome using direct URL (thanks @remjey!)
diff --git a/changelog.d/3699.misc b/changelog.d/3699.misc
deleted file mode 100644
index 437efbd98f13..000000000000
--- a/changelog.d/3699.misc
+++ /dev/null
@@ -1,2 +0,0 @@
-Unit tests can now be run under PostgreSQL in Docker using 
-``test_postgresql.sh``.
diff --git a/changelog.d/3794.misc b/changelog.d/3794.misc
deleted file mode 100644
index 6b98c9609b18..000000000000
--- a/changelog.d/3794.misc
+++ /dev/null
@@ -1 +0,0 @@
-Speed up calculation of typing updates for replication
diff --git a/changelog.d/3836.bugfix b/changelog.d/3836.bugfix
deleted file mode 100644
index add49fbec0d5..000000000000
--- a/changelog.d/3836.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-support registering regular users non-interactively with register_new_matrix_user script
\ No newline at end of file
diff --git a/changelog.d/3868.bugfix b/changelog.d/3868.bugfix
deleted file mode 100644
index 99da8389a5e7..000000000000
--- a/changelog.d/3868.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix broken invite email links for self hosted riots 
diff --git a/changelog.d/3873.misc b/changelog.d/3873.misc
deleted file mode 100644
index 8104b5c085f6..000000000000
--- a/changelog.d/3873.misc
+++ /dev/null
@@ -1,2 +0,0 @@
-Remove documentation regarding installation on Cygwin, the use of WSL is 
-recommended instead.
diff --git a/changelog.d/3879.bugfix b/changelog.d/3879.bugfix
deleted file mode 100644
index 82cb2a8ebc49..000000000000
--- a/changelog.d/3879.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Don't ratelimit autojoins
diff --git a/changelog.d/3883.feature b/changelog.d/3883.feature
deleted file mode 100644
index c11e5c53090e..000000000000
--- a/changelog.d/3883.feature
+++ /dev/null
@@ -1 +0,0 @@
-Adding the ability to change MAX_UPLOAD_SIZE for the docker container variables.
\ No newline at end of file
diff --git a/changelog.d/3889.bugfix b/changelog.d/3889.bugfix
deleted file mode 100644
index e97642598757..000000000000
--- a/changelog.d/3889.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix 500 error when deleting unknown room alias
diff --git a/changelog.d/3892.bugfix b/changelog.d/3892.bugfix
deleted file mode 100644
index 8b30afab040a..000000000000
--- a/changelog.d/3892.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix some b'abcd' noise in logs and metrics
diff --git a/changelog.d/3894.feature b/changelog.d/3894.feature
deleted file mode 100644
index 1ed0cccdb253..000000000000
--- a/changelog.d/3894.feature
+++ /dev/null
@@ -1 +0,0 @@
-Report "python_version" in the phone home stats
diff --git a/changelog.d/3895.bugfix b/changelog.d/3895.bugfix
deleted file mode 100644
index 8b30afab040a..000000000000
--- a/changelog.d/3895.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix some b'abcd' noise in logs and metrics
diff --git a/changelog.d/3897.misc b/changelog.d/3897.misc
deleted file mode 100644
index 87e7ac796e3c..000000000000
--- a/changelog.d/3897.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix typo in README, synaspse -> synapse
\ No newline at end of file
diff --git a/changelog.d/3899.bugfix b/changelog.d/3899.bugfix
deleted file mode 100644
index 5120e3a82309..000000000000
--- a/changelog.d/3899.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-When we join a room, always try the server we used for the alias lookup first, to avoid unresponsive and out-of-date servers.
diff --git a/changelog.d/3903.misc b/changelog.d/3903.misc
deleted file mode 100644
index 49b64bf33355..000000000000
--- a/changelog.d/3903.misc
+++ /dev/null
@@ -1 +0,0 @@
-Increase the timeout when filling missing events in federation requests
\ No newline at end of file
diff --git a/changelog.d/3904.misc b/changelog.d/3904.misc
deleted file mode 100644
index 1e3c8e1706ca..000000000000
--- a/changelog.d/3904.misc
+++ /dev/null
@@ -1 +0,0 @@
-Improve the logging when handling a federation transaction
\ No newline at end of file
diff --git a/changelog.d/3906.misc b/changelog.d/3906.misc
deleted file mode 100644
index 11709186d3e4..000000000000
--- a/changelog.d/3906.misc
+++ /dev/null
@@ -1 +0,0 @@
-Improve logging of outbound federation requests
\ No newline at end of file
diff --git a/changelog.d/3907.bugfix b/changelog.d/3907.bugfix
deleted file mode 100644
index 45e010c0521d..000000000000
--- a/changelog.d/3907.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix incorrect server-name indication for outgoing federation requests
\ No newline at end of file
diff --git a/changelog.d/3908.bugfix b/changelog.d/3908.bugfix
deleted file mode 100644
index 518aee6c4d2e..000000000000
--- a/changelog.d/3908.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix adding client IPs to the database failing on Python 3.
\ No newline at end of file
diff --git a/changelog.d/3909.misc b/changelog.d/3909.misc
deleted file mode 100644
index 11709186d3e4..000000000000
--- a/changelog.d/3909.misc
+++ /dev/null
@@ -1 +0,0 @@
-Improve logging of outbound federation requests
\ No newline at end of file
diff --git a/changelog.d/3910.bugfix b/changelog.d/3910.bugfix
deleted file mode 100644
index 22ec2adc33d5..000000000000
--- a/changelog.d/3910.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix bug where things occaisonally were not being timed out correctly.
diff --git a/changelog.d/3911.misc b/changelog.d/3911.misc
deleted file mode 100644
index e31311d52048..000000000000
--- a/changelog.d/3911.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix the docker image building on python 3
diff --git a/changelog.d/3912.misc b/changelog.d/3912.misc
deleted file mode 100644
index 87d73697ea20..000000000000
--- a/changelog.d/3912.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add a regression test for logging failed HTTP requests on Python 3.
\ No newline at end of file
diff --git a/changelog.d/3914.bugfix b/changelog.d/3914.bugfix
deleted file mode 100644
index 27e6bad59095..000000000000
--- a/changelog.d/3914.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix bug where outbound federation would stop talking to some servers when using workers
diff --git a/changelog.d/3916.feature b/changelog.d/3916.feature
deleted file mode 100644
index 13282d992b9a..000000000000
--- a/changelog.d/3916.feature
+++ /dev/null
@@ -1 +0,0 @@
-Always LL ourselves if we're in a room
diff --git a/changelog.d/3924.misc b/changelog.d/3924.misc
deleted file mode 100644
index 8d010e0bd9c5..000000000000
--- a/changelog.d/3924.misc
+++ /dev/null
@@ -1 +0,0 @@
-Comments and interface cleanup for on_receive_pdu
\ No newline at end of file
diff --git a/changelog.d/3925.misc b/changelog.d/3925.misc
deleted file mode 100644
index 3e41f78ff570..000000000000
--- a/changelog.d/3925.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix spurious exceptions when remote http client closes conncetion
diff --git a/changelog.d/3927.misc b/changelog.d/3927.misc
deleted file mode 100644
index 4bd8e25f67bb..000000000000
--- a/changelog.d/3927.misc
+++ /dev/null
@@ -1 +0,0 @@
-Log exceptions thrown by background tasks
diff --git a/changelog.d/3932.bugfix b/changelog.d/3932.bugfix
deleted file mode 100644
index 7578414ede72..000000000000
--- a/changelog.d/3932.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix some instances of ExpiringCache not expiring cache items
diff --git a/changelog.d/3933.misc b/changelog.d/3933.misc
deleted file mode 100644
index 6545871f55eb..000000000000
--- a/changelog.d/3933.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add a cache to get_destination_retry_timings
diff --git a/changelog.d/3936.bugfix b/changelog.d/3936.bugfix
deleted file mode 100644
index 49b02b9e2781..000000000000
--- a/changelog.d/3936.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix out-of-bounds error when LLing yourself
diff --git a/changelog.d/3938.bugfix b/changelog.d/3938.bugfix
deleted file mode 100644
index 01ccca21a739..000000000000
--- a/changelog.d/3938.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Sending server notices regarding user consent now works on Python 3.
diff --git a/changelog.d/3946.misc b/changelog.d/3946.misc
deleted file mode 100644
index 803857a297f1..000000000000
--- a/changelog.d/3946.misc
+++ /dev/null
@@ -1 +0,0 @@
-Automate pushes to docker hub
diff --git a/changelog.d/3947.misc b/changelog.d/3947.misc
deleted file mode 100644
index 5a9a22bed992..000000000000
--- a/changelog.d/3947.misc
+++ /dev/null
@@ -1 +0,0 @@
-Require attrs 16.0.0 or later
diff --git a/changelog.d/3948.misc b/changelog.d/3948.misc
deleted file mode 100644
index a93edbd1c354..000000000000
--- a/changelog.d/3948.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix incompatibility with python3 on alpine
\ No newline at end of file
diff --git a/changelog.d/3952.misc b/changelog.d/3952.misc
deleted file mode 100644
index 015e4a43e682..000000000000
--- a/changelog.d/3952.misc
+++ /dev/null
@@ -1 +0,0 @@
-Run the test suite on the oldest supported versions of our dependencies in CI.
\ No newline at end of file
diff --git a/changelog.d/3956.bugfix b/changelog.d/3956.bugfix
deleted file mode 100644
index b0828c9fc632..000000000000
--- a/changelog.d/3956.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix exceptions from metrics handler
\ No newline at end of file
diff --git a/changelog.d/3957.misc b/changelog.d/3957.misc
deleted file mode 100644
index 69d647f119cf..000000000000
--- a/changelog.d/3957.misc
+++ /dev/null
@@ -1 +0,0 @@
-CircleCI now only runs merged jobs on PRs, and commit jobs on develop, master, and release branches.
diff --git a/changelog.d/3958.misc b/changelog.d/3958.misc
deleted file mode 100644
index 5931d06dcff0..000000000000
--- a/changelog.d/3958.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix docstrings and add tests for state store methods
diff --git a/changelog.d/3959.feature b/changelog.d/3959.feature
deleted file mode 100644
index b3a4f37a8da1..000000000000
--- a/changelog.d/3959.feature
+++ /dev/null
@@ -1 +0,0 @@
-Include eventid in log lines when processing incoming federation transactions
\ No newline at end of file
diff --git a/changelog.d/3960.bugfix b/changelog.d/3960.bugfix
deleted file mode 100644
index a01dc60df6e5..000000000000
--- a/changelog.d/3960.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix error message for events with m.room.create missing from auth_events
\ No newline at end of file
diff --git a/changelog.d/3961.bugfix b/changelog.d/3961.bugfix
deleted file mode 100644
index e46b5834aad0..000000000000
--- a/changelog.d/3961.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix errors due to concurrent monthly_active_user upserts 
diff --git a/changelog.d/3963.misc b/changelog.d/3963.misc
deleted file mode 100644
index f1e0eaf18e48..000000000000
--- a/changelog.d/3963.misc
+++ /dev/null
@@ -1 +0,0 @@
-fix docstring for FederationClient.get_state_for_room
diff --git a/changelog.d/3964.feature b/changelog.d/3964.feature
deleted file mode 100644
index 599222eb589e..000000000000
--- a/changelog.d/3964.feature
+++ /dev/null
@@ -1 +0,0 @@
-Remove spurious check which made 'localhost' servers not work
diff --git a/changelog.d/3965.misc b/changelog.d/3965.misc
deleted file mode 100644
index e7e4a9c5a8ef..000000000000
--- a/changelog.d/3965.misc
+++ /dev/null
@@ -1 +0,0 @@
-Run notify_app_services as a bg process
diff --git a/changelog.d/3966.misc b/changelog.d/3966.misc
deleted file mode 100644
index 1e3c8e1706ca..000000000000
--- a/changelog.d/3966.misc
+++ /dev/null
@@ -1 +0,0 @@
-Improve the logging when handling a federation transaction
\ No newline at end of file
diff --git a/changelog.d/3967.misc b/changelog.d/3967.misc
deleted file mode 100644
index dc808aec731a..000000000000
--- a/changelog.d/3967.misc
+++ /dev/null
@@ -1 +0,0 @@
-Clarifications in FederationHandler
diff --git a/changelog.d/3968.bugfix b/changelog.d/3968.bugfix
deleted file mode 100644
index 18d43cd64e05..000000000000
--- a/changelog.d/3968.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix exceptions when processing incoming events over federation
\ No newline at end of file
diff --git a/changelog.d/3970.bugfix b/changelog.d/3970.bugfix
deleted file mode 100644
index 562531549788..000000000000
--- a/changelog.d/3970.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Replaced all occurences of e.message with str(e). Contributed by Schnuffle
diff --git a/changelog.d/3972.misc b/changelog.d/3972.misc
deleted file mode 100644
index e56299ee78b1..000000000000
--- a/changelog.d/3972.misc
+++ /dev/null
@@ -1 +0,0 @@
-Further reduce the docker image size
diff --git a/changelog.d/3976.misc b/changelog.d/3976.misc
deleted file mode 100644
index 282148c986cd..000000000000
--- a/changelog.d/3976.misc
+++ /dev/null
@@ -1 +0,0 @@
-Build py3 docker images for docker hub too
diff --git a/changelog.d/3980.bugfix b/changelog.d/3980.bugfix
deleted file mode 100644
index 7578414ede72..000000000000
--- a/changelog.d/3980.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix some instances of ExpiringCache not expiring cache items
diff --git a/changelog.d/3985.misc b/changelog.d/3985.misc
deleted file mode 100644
index ba935caf3a40..000000000000
--- a/changelog.d/3985.misc
+++ /dev/null
@@ -1 +0,0 @@
-Updated the installation instructions to point to the matrix-synapse package on PyPI.
diff --git a/changelog.d/3986.bugfix b/changelog.d/3986.bugfix
deleted file mode 100644
index ce74345365ed..000000000000
--- a/changelog.d/3986.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix lazy loaded sync in the presence of rejected state events
diff --git a/changelog.d/3987.misc b/changelog.d/3987.misc
deleted file mode 100644
index d6b501621103..000000000000
--- a/changelog.d/3987.misc
+++ /dev/null
@@ -1 +0,0 @@
-Disable USE_FROZEN_DICTS for unittests by default.
diff --git a/changelog.d/3988.misc b/changelog.d/3988.misc
deleted file mode 100644
index 10379d183bea..000000000000
--- a/changelog.d/3988.misc
+++ /dev/null
@@ -1 +0,0 @@
-Remove unused Jenkins and development related files from the repo.
diff --git a/changelog.d/3989.misc b/changelog.d/3989.misc
deleted file mode 100644
index 26700d168f01..000000000000
--- a/changelog.d/3989.misc
+++ /dev/null
@@ -1 +0,0 @@
-Improve stacktraces in certain exceptions in the logs
diff --git a/changelog.d/3990.bugfix b/changelog.d/3990.bugfix
deleted file mode 100644
index 51ea105a64f3..000000000000
--- a/changelog.d/3990.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix error when logging incomplete HTTP requests
diff --git a/changelog.d/3991.bugfix b/changelog.d/3991.bugfix
deleted file mode 100644
index 6545871f55eb..000000000000
--- a/changelog.d/3991.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Add a cache to get_destination_retry_timings

From ed763aeba82e1b6fa9f9bb80401e1c7f451a758b Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Wed, 3 Oct 2018 02:27:41 +1000
Subject: [PATCH 778/830] some changelog fixes

---
 CHANGES.md | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index e9b491559e29..03d7412874a8 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -35,17 +35,14 @@ Bugfixes
 - Replaced all occurences of e.message with str(e). Contributed by Schnuffle ([\#3970](https://github.com/matrix-org/synapse/issues/3970))
 - Fix lazy loaded sync in the presence of rejected state events ([\#3986](https://github.com/matrix-org/synapse/issues/3986))
 - Fix error when logging incomplete HTTP requests ([\#3990](https://github.com/matrix-org/synapse/issues/3990))
-- Add a cache to get_destination_retry_timings ([\#3991](https://github.com/matrix-org/synapse/issues/3991))
 
 
 Internal Changes
 ----------------
 
-- Unit tests can now be run under PostgreSQL in Docker using 
-  ``test_postgresql.sh``. ([\#3699](https://github.com/matrix-org/synapse/issues/3699))
+- Unit tests can now be run under PostgreSQL in Docker using ``test_postgresql.sh``. ([\#3699](https://github.com/matrix-org/synapse/issues/3699))
 - Speed up calculation of typing updates for replication ([\#3794](https://github.com/matrix-org/synapse/issues/3794))
-- Remove documentation regarding installation on Cygwin, the use of WSL is 
-  recommended instead. ([\#3873](https://github.com/matrix-org/synapse/issues/3873))
+- Remove documentation regarding installation on Cygwin, the use of WSL is recommended instead. ([\#3873](https://github.com/matrix-org/synapse/issues/3873))
 - Fix typo in README, synaspse -> synapse ([\#3897](https://github.com/matrix-org/synapse/issues/3897))
 - Increase the timeout when filling missing events in federation requests ([\#3903](https://github.com/matrix-org/synapse/issues/3903))
 - Improve the logging when handling a federation transaction ([\#3904](https://github.com/matrix-org/synapse/issues/3904), [\#3966](https://github.com/matrix-org/synapse/issues/3966))
@@ -55,7 +52,7 @@ Internal Changes
 - Comments and interface cleanup for on_receive_pdu ([\#3924](https://github.com/matrix-org/synapse/issues/3924))
 - Fix spurious exceptions when remote http client closes conncetion ([\#3925](https://github.com/matrix-org/synapse/issues/3925))
 - Log exceptions thrown by background tasks ([\#3927](https://github.com/matrix-org/synapse/issues/3927))
-- Add a cache to get_destination_retry_timings ([\#3933](https://github.com/matrix-org/synapse/issues/3933))
+- Add a cache to get_destination_retry_timings ([\#3933](https://github.com/matrix-org/synapse/issues/3933), [\#3991](https://github.com/matrix-org/synapse/issues/3991))
 - Automate pushes to docker hub ([\#3946](https://github.com/matrix-org/synapse/issues/3946))
 - Require attrs 16.0.0 or later ([\#3947](https://github.com/matrix-org/synapse/issues/3947))
 - Fix incompatibility with python3 on alpine ([\#3948](https://github.com/matrix-org/synapse/issues/3948))
@@ -79,8 +76,7 @@ Synapse 0.33.5.1 (2018-09-25)
 Internal Changes
 ----------------
 
-- Fix incompatibility with older Twisted version in tests. Thanks 
-  @OlegGirko! ([\#3940](https://github.com/matrix-org/synapse/issues/3940))
+- Fix incompatibility with older Twisted version in tests. Thanks @OlegGirko! ([\#3940](https://github.com/matrix-org/synapse/issues/3940))
 
 
 Synapse 0.33.5 (2018-09-24)

From ae61ade8919085ad0c90e0b54c4e8338998ee64a Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 2 Oct 2018 23:33:29 +0100
Subject: [PATCH 779/830] Fix bug in forward_extremity update logic

An event does not stop being a forward_extremity just because an outlier or
rejected event refers to it.
---
 changelog.d/3995.bug      |   1 +
 synapse/storage/events.py | 111 ++++++++++++++++++++++++++------------
 2 files changed, 79 insertions(+), 33 deletions(-)
 create mode 100644 changelog.d/3995.bug

diff --git a/changelog.d/3995.bug b/changelog.d/3995.bug
new file mode 100644
index 000000000000..2adc36756b79
--- /dev/null
+++ b/changelog.d/3995.bug
@@ -0,0 +1 @@
+Fix bug in event persistence logic which caused 'NoneType is not iterable'
\ No newline at end of file
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index e7487311ce9b..046174edb35d 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -38,6 +38,7 @@
 from synapse.storage.event_federation import EventFederationStore
 from synapse.storage.events_worker import EventsWorkerStore
 from synapse.types import RoomStreamToken, get_domain_from_id
+from synapse.util import batch_iter
 from synapse.util.async_helpers import ObservableDeferred
 from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
 from synapse.util.frozenutils import frozendict_json_encoder
@@ -386,12 +387,10 @@ def _persist_events(self, events_and_contexts, backfilled=False,
                             )
 
                         for room_id, ev_ctx_rm in iteritems(events_by_room):
-                            # Work out new extremities by recursively adding and removing
-                            # the new events.
                             latest_event_ids = yield self.get_latest_event_ids_in_room(
                                 room_id
                             )
-                            new_latest_event_ids = yield self._calculate_new_extremeties(
+                            new_latest_event_ids = yield self._calculate_new_extremities(
                                 room_id, ev_ctx_rm, latest_event_ids
                             )
 
@@ -400,6 +399,12 @@ def _persist_events(self, events_and_contexts, backfilled=False,
                                 # No change in extremities, so no change in state
                                 continue
 
+                            # there should always be at least one forward extremity.
+                            # (except during the initial persistence of the send_join
+                            # results, in which case there will be no existing
+                            # extremities, so we'll `continue` above and skip this bit.)
+                            assert new_latest_event_ids, "No forward extremities left!"
+
                             new_forward_extremeties[room_id] = new_latest_event_ids
 
                             len_1 = (
@@ -517,44 +522,88 @@ def _persist_events(self, events_and_contexts, backfilled=False,
                     )
 
     @defer.inlineCallbacks
-    def _calculate_new_extremeties(self, room_id, event_contexts, latest_event_ids):
-        """Calculates the new forward extremeties for a room given events to
+    def _calculate_new_extremities(self, room_id, event_contexts, latest_event_ids):
+        """Calculates the new forward extremities for a room given events to
         persist.
 
         Assumes that we are only persisting events for one room at a time.
         """
-        new_latest_event_ids = set(latest_event_ids)
-        # First, add all the new events to the list
-        new_latest_event_ids.update(
-            event.event_id for event, ctx in event_contexts
+
+        # we're only interested in new events which aren't outliers and which aren't
+        # being rejected.
+        new_events = [
+            event for event, ctx in event_contexts
             if not event.internal_metadata.is_outlier() and not ctx.rejected
+        ]
+
+        # start with the existing forward extremities
+        result = set(latest_event_ids)
+
+        # add all the new events to the list
+        result.update(
+            event.event_id for event in new_events
         )
-        # Now remove all events that are referenced by the to-be-added events
-        new_latest_event_ids.difference_update(
+
+        # Now remove all events which are prev_events of any of the new events
+        result.difference_update(
             e_id
-            for event, ctx in event_contexts
+            for event in new_events
             for e_id, _ in event.prev_events
-            if not event.internal_metadata.is_outlier() and not ctx.rejected
         )
 
-        # And finally remove any events that are referenced by previously added
-        # events.
-        rows = yield self._simple_select_many_batch(
-            table="event_edges",
-            column="prev_event_id",
-            iterable=list(new_latest_event_ids),
-            retcols=["prev_event_id"],
-            keyvalues={
-                "is_state": False,
-            },
-            desc="_calculate_new_extremeties",
-        )
+        # Finally, remove any events which are prev_events of any existing events.
+        existing_prevs = yield self._get_events_which_are_prevs(result)
+        result.difference_update(existing_prevs)
 
-        new_latest_event_ids.difference_update(
-            row["prev_event_id"] for row in rows
-        )
+        if not result:
+            logger.warn(
+                "Forward extremity list A+B-C-D is now empty in %s. "
+                "Old extremities (A): %s, new events (B): %s, "
+                "existing events which are reffed by new events (C): %s, "
+                "new events which are reffed by existing events (D): %s",
+                room_id, latest_event_ids, new_events,
+                [e_id for event in new_events for e_id, _ in event.prev_events],
+                existing_prevs,
+            )
+        defer.returnValue(result)
 
-        defer.returnValue(new_latest_event_ids)
+    @defer.inlineCallbacks
+    def _get_events_which_are_prevs(self, event_ids):
+        """Filter the supplied list of event_ids to get those which are prev_events of
+        existing (non-outlier) events.
+
+        Args:
+            event_ids (Iterable[str]): event ids to filter
+
+        Returns:
+            Deferred[List[str]]: filtered event ids
+        """
+        results = []
+
+        def _get_events(txn, batch):
+            sql = """
+            SELECT prev_event_id
+            FROM event_edges
+                INNER JOIN events USING (event_id)
+                LEFT JOIN rejections USING (event_id)
+            WHERE
+                prev_event_id IN (%s)
+                AND rejections.event_id IS NULL
+            """ % (
+                ",".join("?" for _ in batch),
+            )
+
+            txn.execute(sql, batch)
+            results.extend(r[0] for r in txn)
+
+        for chunk in batch_iter(event_ids, 100):
+            yield self.runInteraction(
+                "_get_events_which_are_prevs",
+                _get_events,
+                chunk,
+            )
+
+        defer.returnValue(results)
 
     @defer.inlineCallbacks
     def _get_new_state_after_events(self, room_id, events_context, old_latest_event_ids,
@@ -586,10 +635,6 @@ def _get_new_state_after_events(self, room_id, events_context, old_latest_event_
             the new current state is only returned if we've already calculated
             it.
         """
-
-        if not new_latest_event_ids:
-            return
-
         # map from state_group to ((type, key) -> event_id) state map
         state_groups_map = {}
 

From 3e39783d5d7ca5da7b3619d0328f6aeec48854de Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 2 Oct 2018 23:44:14 +0100
Subject: [PATCH 780/830] remove debugging

---
 synapse/storage/events.py | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 046174edb35d..8822dc7bcbd5 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -555,16 +555,6 @@ def _calculate_new_extremities(self, room_id, event_contexts, latest_event_ids):
         existing_prevs = yield self._get_events_which_are_prevs(result)
         result.difference_update(existing_prevs)
 
-        if not result:
-            logger.warn(
-                "Forward extremity list A+B-C-D is now empty in %s. "
-                "Old extremities (A): %s, new events (B): %s, "
-                "existing events which are reffed by new events (C): %s, "
-                "new events which are reffed by existing events (D): %s",
-                room_id, latest_event_ids, new_events,
-                [e_id for event in new_events for e_id, _ in event.prev_events],
-                existing_prevs,
-            )
         defer.returnValue(result)
 
     @defer.inlineCallbacks

From 2a4ea3baa8e6723f1bdbae2dad0f99c2a73646a5 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 3 Oct 2018 07:09:25 +0100
Subject: [PATCH 781/830] fix newsfile name

---
 changelog.d/{3995.bug => 3995.bugfix} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename changelog.d/{3995.bug => 3995.bugfix} (100%)

diff --git a/changelog.d/3995.bug b/changelog.d/3995.bugfix
similarity index 100%
rename from changelog.d/3995.bug
rename to changelog.d/3995.bugfix

From 9693625e556df1af66ba376d49411064c2d0f47e Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 3 Oct 2018 10:19:41 +0100
Subject: [PATCH 782/830] actually exclude outliers

---
 synapse/storage/events.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 8822dc7bcbd5..03cedf3a75ed 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -560,7 +560,7 @@ def _calculate_new_extremities(self, room_id, event_contexts, latest_event_ids):
     @defer.inlineCallbacks
     def _get_events_which_are_prevs(self, event_ids):
         """Filter the supplied list of event_ids to get those which are prev_events of
-        existing (non-outlier) events.
+        existing (non-outlier/rejected) events.
 
         Args:
             event_ids (Iterable[str]): event ids to filter
@@ -578,6 +578,7 @@ def _get_events(txn, batch):
                 LEFT JOIN rejections USING (event_id)
             WHERE
                 prev_event_id IN (%s)
+                AND NOT events.outlier
                 AND rejections.event_id IS NULL
             """ % (
                 ",".join("?" for _ in batch),

From 7c570bff749eaf72be6f982c89facab3b16c42ae Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 3 Oct 2018 11:28:01 +0100
Subject: [PATCH 783/830] Fix exception in background metrics collection

We attempted to iterate through a list on a separate thread without
doing the necessary copying.
---
 synapse/metrics/background_process_metrics.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/synapse/metrics/background_process_metrics.py b/synapse/metrics/background_process_metrics.py
index 173908299c4a..037f1c490ed4 100644
--- a/synapse/metrics/background_process_metrics.py
+++ b/synapse/metrics/background_process_metrics.py
@@ -101,9 +101,13 @@ def collect(self):
             labels=["name"],
         )
 
-        # We copy the dict so that it doesn't change from underneath us
+        # We copy the dict so that it doesn't change from underneath us.
+        # We also copy the process lists as that can also change
         with _bg_metrics_lock:
-            _background_processes_copy = dict(_background_processes)
+            _background_processes_copy = {
+                k: list(v)
+                for k, v in six.iteritems(_background_processes)
+            }
 
         for desc, processes in six.iteritems(_background_processes_copy):
             background_process_in_flight_count.add_metric(

From c69faf8c4a045e4c53bb50b4dce4369fb586f0e0 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 3 Oct 2018 11:29:44 +0100
Subject: [PATCH 784/830] Newsfile

---
 changelog.d/3996.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3996.bugfix

diff --git a/changelog.d/3996.bugfix b/changelog.d/3996.bugfix
new file mode 100644
index 000000000000..a056485ea143
--- /dev/null
+++ b/changelog.d/3996.bugfix
@@ -0,0 +1 @@
+Fix exception in background metrics collection

From 495a9d06bb21cf30376292d6592aa5fd59a52634 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 3 Oct 2018 11:34:30 +0100
Subject: [PATCH 785/830] Fix exception handling in fetching remote profiles

---
 synapse/handlers/profile.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py
index f284d5a38560..1dfbde84fd7a 100644
--- a/synapse/handlers/profile.py
+++ b/synapse/handlers/profile.py
@@ -142,10 +142,8 @@ def get_displayname(self, target_user):
                 if e.code != 404:
                     logger.exception("Failed to get displayname")
                 raise
-            except Exception:
-                logger.exception("Failed to get displayname")
-            else:
-                defer.returnValue(result["displayname"])
+
+            defer.returnValue(result["displayname"])
 
     @defer.inlineCallbacks
     def set_displayname(self, target_user, requester, new_displayname, by_admin=False):
@@ -199,8 +197,6 @@ def get_avatar_url(self, target_user):
                 if e.code != 404:
                     logger.exception("Failed to get avatar_url")
                 raise
-            except Exception:
-                logger.exception("Failed to get avatar_url")
 
             defer.returnValue(result["avatar_url"])
 

From 6e0c66f6515bbbf4e21f6f8837d5ee0d48bbd71a Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 3 Oct 2018 11:40:02 +0100
Subject: [PATCH 786/830] Newsfile

---
 changelog.d/3997.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3997.bugfix

diff --git a/changelog.d/3997.bugfix b/changelog.d/3997.bugfix
new file mode 100644
index 000000000000..b060ee8c1863
--- /dev/null
+++ b/changelog.d/3997.bugfix
@@ -0,0 +1 @@
+Fix exception handling in fetching remote profiles

From d1b7c0ca05bdcc3629cc3af97756ab71a94da294 Mon Sep 17 00:00:00 2001
From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com>
Date: Wed, 3 Oct 2018 11:52:05 +0100
Subject: [PATCH 787/830] Docker build all tags starting vX.Y.Z, inc RCs

Note the regex must match the complete string anyway,
so the leading ^ was useless.
---
 .circleci/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 6ae3a42235a7..ec3848b04889 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -156,7 +156,7 @@ workflows:
       - dockerhubuploadrelease:
           filters:
             tags:
-              only: /^v[0-9].[0-9]+.[0-9]+(.[0-9]+)?/
+              only: /v[0-9].[0-9]+.[0-9]+.*/
             branches:
               ignore: /.*/
       - dockerhubuploadlatest:

From 69e857853fe91d22fa792c9e26edd87840526c22 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 3 Oct 2018 11:57:30 +0100
Subject: [PATCH 788/830] Fix handling of rejected threepid invites

---
 synapse/event_auth.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/event_auth.py b/synapse/event_auth.py
index 02fa46ef7f82..c6c011b6b644 100644
--- a/synapse/event_auth.py
+++ b/synapse/event_auth.py
@@ -157,7 +157,7 @@ def check(event, auth_events, do_sig_check=True, do_size_check=True):
             raise AuthError(
                 403, (
                     "You cannot issue a third party invite for %s." %
-                    (event.content.display_name,)
+                    (event.content.get("display_name", "<Unknown>"),)
                 )
             )
         else:

From 93a86039040665ec6fbe7bf0c6d7c5024fa95dd5 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 3 Oct 2018 11:59:05 +0100
Subject: [PATCH 789/830] Newsfile

---
 changelog.d/3999.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/3999.bugfix

diff --git a/changelog.d/3999.bugfix b/changelog.d/3999.bugfix
new file mode 100644
index 000000000000..dc3b2caffa58
--- /dev/null
+++ b/changelog.d/3999.bugfix
@@ -0,0 +1 @@
+Fix handling of rejected threepid invites

From 52e6e815bef4379773fde257c85c4fc4d11cd181 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 3 Oct 2018 14:13:07 +0100
Subject: [PATCH 790/830] Sanitise error messages when user doesn't have
 permission to invite

---
 synapse/event_auth.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/synapse/event_auth.py b/synapse/event_auth.py
index c6c011b6b644..af3eee95b977 100644
--- a/synapse/event_auth.py
+++ b/synapse/event_auth.py
@@ -155,10 +155,7 @@ def check(event, auth_events, do_sig_check=True, do_size_check=True):
 
         if user_level < invite_level:
             raise AuthError(
-                403, (
-                    "You cannot issue a third party invite for %s." %
-                    (event.content.get("display_name", "<Unknown>"),)
-                )
+                403, "You don't have permission to invite users",
             )
         else:
             logger.debug("Allowing! %s", event)
@@ -305,7 +302,7 @@ def _is_membership_change_allowed(event, auth_events):
 
             if user_level < invite_level:
                 raise AuthError(
-                    403, "You cannot invite user %s." % target_user_id
+                    403, "You don't have permission to invite users",
                 )
     elif Membership.JOIN == membership:
         # Joins are valid iff caller == target and they were:

From a59d8996682b33454e1afcf2ab1bc23c5224e325 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 3 Oct 2018 17:15:35 +0100
Subject: [PATCH 791/830] Pin to prometheus_client<0.4 to avoid renaming all of
 our metrics

---
 changelog.d/4002.misc          | 1 +
 synapse/python_dependencies.py | 5 ++++-
 2 files changed, 5 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/4002.misc

diff --git a/changelog.d/4002.misc b/changelog.d/4002.misc
new file mode 100644
index 000000000000..545abe5d5835
--- /dev/null
+++ b/changelog.d/4002.misc
@@ -0,0 +1 @@
+Pin to prometheus_client<0.4 to avoid renaming all of our metrics
\ No newline at end of file
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index 0f339a03203e..d4d983b00a4c 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -58,7 +58,10 @@
     "msgpack-python>=0.3.0": ["msgpack"],
     "phonenumbers>=8.2.0": ["phonenumbers"],
     "six>=1.10": ["six"],
-    "prometheus_client>=0.0.18": ["prometheus_client"],
+
+    # prometheus_client 0.4.0 changed the format of counter metrics
+    # (cf https://github.com/matrix-org/synapse/issues/4001)
+    "prometheus_client>=0.0.18,<0.4.0": ["prometheus_client"],
 
     # we use attr.s(slots), which arrived in 16.0.0
     "attrs>=16.0.0": ["attr>=16.0.0"],

From dd59dfc51ffdcb49e3ae46b9072cff86dfc32d94 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Thu, 4 Oct 2018 22:37:55 +1000
Subject: [PATCH 792/830] full version

---
 synapse/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/__init__.py b/synapse/__init__.py
index 312e4e2fa61e..43c5821ade06 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -27,4 +27,4 @@
 except ImportError:
     pass
 
-__version__ = "0.33.6rc1"
+__version__ = "0.33.6"

From 92faeb2a3fc73bd9307b3192aeacec4afb1aad1a Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Thu, 4 Oct 2018 22:38:10 +1000
Subject: [PATCH 793/830] changelog

---
 CHANGES.md            | 9 +++++++++
 changelog.d/4002.misc | 1 -
 2 files changed, 9 insertions(+), 1 deletion(-)
 delete mode 100644 changelog.d/4002.misc

diff --git a/CHANGES.md b/CHANGES.md
index 03d7412874a8..048b9f95db95 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,12 @@
+Synapse 0.33.6 (2018-10-04)
+===========================
+
+Internal Changes
+----------------
+
+- Pin to prometheus_client<0.4 to avoid renaming all of our metrics ([\#4002](https://github.com/matrix-org/synapse/issues/4002))
+
+
 Synapse 0.33.6rc1 (2018-10-03)
 ==============================
 
diff --git a/changelog.d/4002.misc b/changelog.d/4002.misc
deleted file mode 100644
index 545abe5d5835..000000000000
--- a/changelog.d/4002.misc
+++ /dev/null
@@ -1 +0,0 @@
-Pin to prometheus_client<0.4 to avoid renaming all of our metrics
\ No newline at end of file

From f7199e873448794ffc88a08a9d7c01f1be0dfca1 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 5 Oct 2018 11:23:08 +0100
Subject: [PATCH 794/830] Log looping call exceptions

If a looping call function errors, then it kills the loop entirely.
Currently it throws away the exception logs, so we should make it
actually log them.

Fixes #3929
---
 synapse/util/__init__.py | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py
index 680ea928c772..c237d003bc8e 100644
--- a/synapse/util/__init__.py
+++ b/synapse/util/__init__.py
@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import functools
 import logging
 from itertools import islice
 
@@ -66,7 +67,7 @@ def looping_call(self, f, msec):
             f(function): The function to call repeatedly.
             msec(float): How long to wait between calls in milliseconds.
         """
-        call = task.LoopingCall(f)
+        call = task.LoopingCall(_log_exception_wrapper(f))
         call.clock = self._reactor
         call.start(msec / 1000.0, now=False)
         return call
@@ -109,3 +110,19 @@ def batch_iter(iterable, size):
     sourceiter = iter(iterable)
     # call islice until it returns an empty tuple
     return iter(lambda: tuple(islice(sourceiter, size)), ())
+
+
+def _log_exception_wrapper(f):
+    """Used to wrap looping calls to log loudly if they get killed
+    """
+
+    @functools.wraps(f)
+    def wrap(*args, **kwargs):
+        try:
+            logger.info("Running looping call")
+            return f(*args, **kwargs)
+        except:  # noqa: E722, as we reraise the exception this is fine.
+            logger.exception("Looping called died")
+            raise
+
+    return wrap

From 8164f6daf362b7039d6dcc8b645f306cfd5c49a8 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 5 Oct 2018 11:25:09 +0100
Subject: [PATCH 795/830] Newsfile

---
 changelog.d/4008.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/4008.misc

diff --git a/changelog.d/4008.misc b/changelog.d/4008.misc
new file mode 100644
index 000000000000..573021005490
--- /dev/null
+++ b/changelog.d/4008.misc
@@ -0,0 +1 @@
+Log exceptions in looping calls

From 497444f1fdd2c39906179b1dde8c67415e465398 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Fri, 5 Oct 2018 15:08:36 +0100
Subject: [PATCH 796/830] Don't reuse backup versions

Since we don't actually delete the keys, just mark the versions
as deleted in the db rather than actually deleting them, then we
won't reuse versions.

Fixes https://github.com/vector-im/riot-web/issues/7448
---
 synapse/storage/e2e_room_keys.py                  | 9 +++++++--
 synapse/storage/schema/delta/51/e2e_room_keys.sql | 3 ++-
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py
index 969f4aef9ca3..4d439bb16473 100644
--- a/synapse/storage/e2e_room_keys.py
+++ b/synapse/storage/e2e_room_keys.py
@@ -193,7 +193,8 @@ def delete_e2e_room_keys(
     @staticmethod
     def _get_current_version(txn, user_id):
         txn.execute(
-            "SELECT MAX(version) FROM e2e_room_keys_versions WHERE user_id=?",
+            "SELECT MAX(version) FROM e2e_room_keys_versions "
+            "WHERE user_id=? AND deleted=0",
             (user_id,)
         )
         row = txn.fetchone()
@@ -226,6 +227,7 @@ def _get_e2e_room_keys_version_info_txn(txn):
                 keyvalues={
                     "user_id": user_id,
                     "version": this_version,
+                    "deleted": 0,
                 },
                 retcols=(
                     "version",
@@ -300,13 +302,16 @@ def _delete_e2e_room_keys_version_txn(txn):
             else:
                 this_version = version
 
-            return self._simple_delete_one_txn(
+            return self._simple_update_one_txn(
                 txn,
                 table="e2e_room_keys_versions",
                 keyvalues={
                     "user_id": user_id,
                     "version": this_version,
                 },
+                updatevalues={
+                    "deleted": 1,
+                }
             )
 
         return self.runInteraction(
diff --git a/synapse/storage/schema/delta/51/e2e_room_keys.sql b/synapse/storage/schema/delta/51/e2e_room_keys.sql
index 4531fd56ee8a..c0e66a697d88 100644
--- a/synapse/storage/schema/delta/51/e2e_room_keys.sql
+++ b/synapse/storage/schema/delta/51/e2e_room_keys.sql
@@ -32,7 +32,8 @@ CREATE TABLE e2e_room_keys_versions (
     user_id TEXT NOT NULL,
     version TEXT NOT NULL,
     algorithm TEXT NOT NULL,
-    auth_data TEXT NOT NULL
+    auth_data TEXT NOT NULL,
+    deleted SMALLINT DEFAULT 0 NOT NULL
 );
 
 CREATE UNIQUE INDEX e2e_room_keys_versions_idx ON e2e_room_keys_versions(user_id, version);

From 8a1817f0d29308a233783d43cbf1ad27891120c1 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 8 Oct 2018 13:26:54 +0100
Subject: [PATCH 797/830] Use errback pattern and catch async failures

---
 synapse/handlers/appservice.py |  7 +++++-
 synapse/util/__init__.py       | 43 +++++++++++++++++++++++-----------
 2 files changed, 35 insertions(+), 15 deletions(-)

diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py
index f0f89af7dcda..16b897eb18cd 100644
--- a/synapse/handlers/appservice.py
+++ b/synapse/handlers/appservice.py
@@ -28,6 +28,7 @@
     event_processing_loop_room_count,
 )
 from synapse.metrics.background_process_metrics import run_as_background_process
+from synapse.util import make_log_failure_errback
 from synapse.util.logcontext import make_deferred_yieldable, run_in_background
 from synapse.util.metrics import Measure
 
@@ -112,7 +113,11 @@ def handle_event(event):
 
                         if not self.started_scheduler:
                             def start_scheduler():
-                                return self.scheduler.start().addErrback(log_failure)
+                                return self.scheduler.start().addErrback(
+                                    make_log_failure_errback(
+                                        "Application Services Failure",
+                                    )
+                                )
                             run_as_background_process("as_scheduler", start_scheduler)
                             self.started_scheduler = True
 
diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py
index c237d003bc8e..964078aed47f 100644
--- a/synapse/util/__init__.py
+++ b/synapse/util/__init__.py
@@ -13,7 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import functools
 import logging
 from itertools import islice
 
@@ -67,9 +66,12 @@ def looping_call(self, f, msec):
             f(function): The function to call repeatedly.
             msec(float): How long to wait between calls in milliseconds.
         """
-        call = task.LoopingCall(_log_exception_wrapper(f))
+        call = task.LoopingCall(f)
         call.clock = self._reactor
-        call.start(msec / 1000.0, now=False)
+        d = call.start(msec / 1000.0, now=False)
+        d.addErrback(make_log_failure_errback(
+            "Looping call died", consumeErrors=False,
+        ))
         return call
 
     def call_later(self, delay, callback, *args, **kwargs):
@@ -112,17 +114,30 @@ def batch_iter(iterable, size):
     return iter(lambda: tuple(islice(sourceiter, size)), ())
 
 
-def _log_exception_wrapper(f):
-    """Used to wrap looping calls to log loudly if they get killed
+def make_log_failure_errback(msg, consumeErrors=True):
+    """Creates a function suitable for passing to `Deferred.addErrback` that
+    logs any failures that occur.
+
+    Args:
+        msg (str): Message to log
+        consumeErrors (bool): If true consumes the failure, otherwise passes
+            on down the callback chain
+
+    Returns:
+        func(Failure)
     """
 
-    @functools.wraps(f)
-    def wrap(*args, **kwargs):
-        try:
-            logger.info("Running looping call")
-            return f(*args, **kwargs)
-        except:  # noqa: E722, as we reraise the exception this is fine.
-            logger.exception("Looping called died")
-            raise
+    def log_failure(failure):
+        logger.error(
+            msg,
+            exc_info=(
+                failure.type,
+                failure.value,
+                failure.getTracebackObject()
+            )
+        )
+
+        if not consumeErrors:
+            return failure
 
-    return wrap
+    return log_failure

From 495975e231bd429eb0114d9258423a5e202dcb2b Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 8 Oct 2018 13:44:58 +0100
Subject: [PATCH 798/830] Optimisation for filter_events_for_server

We're better off hashing just the event_id than the whole ((type, state_key),
event_id) tuple - so use a dict instead of a set.

Also, iteritems > items.
---
 changelog.d/4017.misc |  1 +
 synapse/visibility.py | 13 ++++++-------
 2 files changed, 7 insertions(+), 7 deletions(-)
 create mode 100644 changelog.d/4017.misc

diff --git a/changelog.d/4017.misc b/changelog.d/4017.misc
new file mode 100644
index 000000000000..b1ceb0656062
--- /dev/null
+++ b/changelog.d/4017.misc
@@ -0,0 +1 @@
+Optimisation for serving federation requests
\ No newline at end of file
diff --git a/synapse/visibility.py b/synapse/visibility.py
index d4680863d3e7..c64ad2144c8b 100644
--- a/synapse/visibility.py
+++ b/synapse/visibility.py
@@ -324,14 +324,13 @@ def redact_disallowed(event, state):
     # server's domain.
     #
     # event_to_state_ids contains lots of duplicates, so it turns out to be
-    # cheaper to build a complete set of unique
-    # ((type, state_key), event_id) tuples, and then filter out the ones we
-    # don't want.
+    # cheaper to build a complete event_id => (type, state_key) dict, and then
+    # filter out the ones we don't want
     #
-    state_key_to_event_id_set = {
-        e
+    event_id_to_state_key = {
+        event_id: key
         for key_to_eid in itervalues(event_to_state_ids)
-        for e in key_to_eid.items()
+        for key, event_id in iteritems(key_to_eid)
     }
 
     def include(typ, state_key):
@@ -346,7 +345,7 @@ def include(typ, state_key):
 
     event_map = yield store.get_events([
         e_id
-        for key, e_id in state_key_to_event_id_set
+        for e_id, key in iteritems(event_id_to_state_key)
         if include(key[0], key[1])
     ])
 

From 69823205722662a72e4203ec304ff595a0f6ecf7 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 8 Oct 2018 14:06:19 +0100
Subject: [PATCH 799/830] Remove unnecessary extra function call layer

---
 synapse/handlers/appservice.py | 18 +++---------------
 synapse/util/__init__.py       | 29 +++++++++++++----------------
 2 files changed, 16 insertions(+), 31 deletions(-)

diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py
index 16b897eb18cd..17eedf4dbf92 100644
--- a/synapse/handlers/appservice.py
+++ b/synapse/handlers/appservice.py
@@ -28,7 +28,7 @@
     event_processing_loop_room_count,
 )
 from synapse.metrics.background_process_metrics import run_as_background_process
-from synapse.util import make_log_failure_errback
+from synapse.util import log_failure
 from synapse.util.logcontext import make_deferred_yieldable, run_in_background
 from synapse.util.metrics import Measure
 
@@ -37,17 +37,6 @@
 events_processed_counter = Counter("synapse_handlers_appservice_events_processed", "")
 
 
-def log_failure(failure):
-    logger.error(
-        "Application Services Failure",
-        exc_info=(
-            failure.type,
-            failure.value,
-            failure.getTracebackObject()
-        )
-    )
-
-
 class ApplicationServicesHandler(object):
 
     def __init__(self, hs):
@@ -114,10 +103,9 @@ def handle_event(event):
                         if not self.started_scheduler:
                             def start_scheduler():
                                 return self.scheduler.start().addErrback(
-                                    make_log_failure_errback(
-                                        "Application Services Failure",
-                                    )
+                                    log_failure, "Application Services Failure",
                                 )
+
                             run_as_background_process("as_scheduler", start_scheduler)
                             self.started_scheduler = True
 
diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py
index 964078aed47f..9a8fae049737 100644
--- a/synapse/util/__init__.py
+++ b/synapse/util/__init__.py
@@ -69,9 +69,9 @@ def looping_call(self, f, msec):
         call = task.LoopingCall(f)
         call.clock = self._reactor
         d = call.start(msec / 1000.0, now=False)
-        d.addErrback(make_log_failure_errback(
-            "Looping call died", consumeErrors=False,
-        ))
+        d.addErrback(
+            log_failure, "Looping call died", consumeErrors=False,
+        )
         return call
 
     def call_later(self, delay, callback, *args, **kwargs):
@@ -114,7 +114,7 @@ def batch_iter(iterable, size):
     return iter(lambda: tuple(islice(sourceiter, size)), ())
 
 
-def make_log_failure_errback(msg, consumeErrors=True):
+def log_failure(failure, msg, consumeErrors=True):
     """Creates a function suitable for passing to `Deferred.addErrback` that
     logs any failures that occur.
 
@@ -127,17 +127,14 @@ def make_log_failure_errback(msg, consumeErrors=True):
         func(Failure)
     """
 
-    def log_failure(failure):
-        logger.error(
-            msg,
-            exc_info=(
-                failure.type,
-                failure.value,
-                failure.getTracebackObject()
-            )
+    logger.error(
+        msg,
+        exc_info=(
+            failure.type,
+            failure.value,
+            failure.getTracebackObject()
         )
+    )
 
-        if not consumeErrors:
-            return failure
-
-    return log_failure
+    if not consumeErrors:
+        return failure

From 0c905ee015fdbf0d61f46db600fd79a18d0b6376 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Tue, 9 Oct 2018 09:39:13 +0100
Subject: [PATCH 800/830] be python3 compatible

---
 synapse/handlers/e2e_room_keys.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py
index 2c330382cfc6..117144bb6eb3 100644
--- a/synapse/handlers/e2e_room_keys.py
+++ b/synapse/handlers/e2e_room_keys.py
@@ -15,6 +15,7 @@
 
 import logging
 
+from six import iteritems
 from twisted.internet import defer
 
 from synapse.api.errors import RoomKeysVersionError, StoreError, SynapseError
@@ -137,8 +138,8 @@ def upload_room_keys(self, user_id, version, room_keys):
 
             # go through the room_keys.
             # XXX: this should/could be done concurrently, given we're in a lock.
-            for room_id, room in room_keys['rooms'].iteritems():
-                for session_id, session in room['sessions'].iteritems():
+            for room_id, room in iteritems(room_keys['rooms']):
+                for session_id, session in iteritems(room['sessions']):
                     yield self._upload_room_key(
                         user_id, version, room_id, session_id, session
                     )

From f4a4dbcad106b41e3307eb5ca541ff3087b788cf Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Tue, 9 Oct 2018 09:47:04 +0100
Subject: [PATCH 801/830] Apparently this blank line is Very Important

---
 synapse/handlers/e2e_room_keys.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py
index 117144bb6eb3..bf2a83cc31c9 100644
--- a/synapse/handlers/e2e_room_keys.py
+++ b/synapse/handlers/e2e_room_keys.py
@@ -16,6 +16,7 @@
 import logging
 
 from six import iteritems
+
 from twisted.internet import defer
 
 from synapse.api.errors import RoomKeysVersionError, StoreError, SynapseError

From d3464ce708cd857216d88d3e67867f1cf0c8bf60 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Tue, 9 Oct 2018 10:33:59 +0100
Subject: [PATCH 802/830] isort

---
 synapse/rest/client/v2_alpha/room_keys.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py
index ea114bc8b48e..539893a5d64b 100644
--- a/synapse/rest/client/v2_alpha/room_keys.py
+++ b/synapse/rest/client/v2_alpha/room_keys.py
@@ -17,7 +17,7 @@
 
 from twisted.internet import defer
 
-from synapse.api.errors import SynapseError, Codes
+from synapse.api.errors import Codes, SynapseError
 from synapse.http.servlet import (
     RestServlet,
     parse_json_object_from_request,

From d34657e1f27df042a33cee4ba8c0b7cbaa112186 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Tue, 9 Oct 2018 11:15:54 +0100
Subject: [PATCH 803/830] Add changelog

---
 changelog.d/4019.feature | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/4019.feature

diff --git a/changelog.d/4019.feature b/changelog.d/4019.feature
new file mode 100644
index 000000000000..49e066d269ef
--- /dev/null
+++ b/changelog.d/4019.feature
@@ -0,0 +1 @@
+Add support for end-to-end key backup (MSC1687)

From bdc27d6716d14128e25737865cc36c8adf42aeaa Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 9 Oct 2018 14:15:49 +0100
Subject: [PATCH 804/830] Add metric to count lazy member sync requests

---
 synapse/handlers/sync.py | 26 ++++++++++++++++++++------
 1 file changed, 20 insertions(+), 6 deletions(-)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 67b8ca28c744..58edf2147205 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -20,6 +20,8 @@
 
 from six import iteritems, itervalues
 
+from prometheus_client import Counter
+
 from twisted.internet import defer
 
 from synapse.api.constants import EventTypes, Membership
@@ -36,6 +38,13 @@
 
 logger = logging.getLogger(__name__)
 
+
+# Counts the number of times we got asked for a lazy loaded sync. Type is one of
+# initial_sync, full_sate_sync or incremental_sync
+lazy_member_sync_counter = Counter(
+    "synapse_handlers_sync_lazy_member_sync", "", ["type"],
+)
+
 # Store the cache that tracks which lazy-loaded members have been sent to a given
 # client for no more than 30 minutes.
 LAZY_LOADED_MEMBERS_CACHE_MAX_AGE = 30 * 60 * 1000
@@ -227,14 +236,19 @@ def wait_for_sync_for_user(self, sync_config, since_token=None, timeout=0,
     @defer.inlineCallbacks
     def _wait_for_sync_for_user(self, sync_config, since_token, timeout,
                                 full_state):
+        if since_token is None:
+            sync_type = "initial_sync"
+        elif full_state:
+            sync_type = "full_state_sync"
+        else:
+            sync_type = "incremental_sync"
+
         context = LoggingContext.current_context()
         if context:
-            if since_token is None:
-                context.tag = "initial_sync"
-            elif full_state:
-                context.tag = "full_state_sync"
-            else:
-                context.tag = "incremental_sync"
+            context.tag = sync_type
+
+        if sync_config.filter_collection.lazy_load_members():
+            lazy_member_sync_counter.labels(sync_type).inc()
 
         if timeout == 0 or since_token is None or full_state:
             # we are going to return immediately, so don't bother calling

From 20733857abb01aee20ec9fd30dbd4b006aa5de23 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 9 Oct 2018 14:16:48 +0100
Subject: [PATCH 805/830] Newsfile

---
 changelog.d/4022.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/4022.misc

diff --git a/changelog.d/4022.misc b/changelog.d/4022.misc
new file mode 100644
index 000000000000..f1d49932e137
--- /dev/null
+++ b/changelog.d/4022.misc
@@ -0,0 +1 @@
+Add metric to count lazy member sync requests

From b8d9e108be60b3d14a57238562bfb5f49781c2a6 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Tue, 9 Oct 2018 18:04:21 +0100
Subject: [PATCH 806/830] Fix mergefail

---
 synapse/api/errors.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index 4124469442af..0a6e78711f29 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -344,7 +344,7 @@ def error_dict(self):
         return cs_error(
             self.msg,
             self.errcode,
-            room_version=self.current_version,
+            room_version=self._room_version,
         )
 
 

From 395276b4051123d369aab5deab020f816479c63b Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 10 Oct 2018 09:24:39 +0100
Subject: [PATCH 807/830] Append _total to metric and fix up spelling

---
 synapse/handlers/sync.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 58edf2147205..41daa1d888f3 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -40,9 +40,9 @@
 
 
 # Counts the number of times we got asked for a lazy loaded sync. Type is one of
-# initial_sync, full_sate_sync or incremental_sync
+# initial_sync, full_state_sync or incremental_sync
 lazy_member_sync_counter = Counter(
-    "synapse_handlers_sync_lazy_member_sync", "", ["type"],
+    "synapse_handlers_sync_lazy_member_sync_total", "", ["type"],
 )
 
 # Store the cache that tracks which lazy-loaded members have been sent to a given

From 3cbe8331e60559008875679fab51423eedb242c1 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 10 Oct 2018 11:23:17 +0100
Subject: [PATCH 808/830] Track number of non-empty sync responses instead

---
 synapse/handlers/sync.py | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 41daa1d888f3..5cae48436fac 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -39,10 +39,12 @@
 logger = logging.getLogger(__name__)
 
 
-# Counts the number of times we got asked for a lazy loaded sync. Type is one of
-# initial_sync, full_state_sync or incremental_sync
-lazy_member_sync_counter = Counter(
-    "synapse_handlers_sync_lazy_member_sync_total", "", ["type"],
+# Counts the number of times we returned a non-empty sync. `type` is one of
+# "initial_sync", "full_state_sync" or "incremental_sync", `lazy_loaded` is
+# "true" or "false" depending on if the request asked for lazy loaded members or
+# not.
+non_empty_sync_counter = Counter(
+    "synapse_handlers_sync_nonempty_total", "", ["type", "lazy_loaded"],
 )
 
 # Store the cache that tracks which lazy-loaded members have been sent to a given
@@ -247,16 +249,12 @@ def _wait_for_sync_for_user(self, sync_config, since_token, timeout,
         if context:
             context.tag = sync_type
 
-        if sync_config.filter_collection.lazy_load_members():
-            lazy_member_sync_counter.labels(sync_type).inc()
-
         if timeout == 0 or since_token is None or full_state:
             # we are going to return immediately, so don't bother calling
             # notifier.wait_for_events.
             result = yield self.current_sync_for_user(
                 sync_config, since_token, full_state=full_state,
             )
-            defer.returnValue(result)
         else:
             def current_sync_callback(before_token, after_token):
                 return self.current_sync_for_user(sync_config, since_token)
@@ -265,7 +263,15 @@ def current_sync_callback(before_token, after_token):
                 sync_config.user.to_string(), timeout, current_sync_callback,
                 from_token=since_token,
             )
-            defer.returnValue(result)
+
+        if result:
+            if sync_config.filter_collection.lazy_load_members():
+                lazy_loaded = "true"
+            else:
+                lazy_loaded = "false"
+            non_empty_sync_counter.labels(sync_type, lazy_loaded).inc()
+
+        defer.returnValue(result)
 
     def current_sync_for_user(self, sync_config, since_token=None,
                               full_state=False):

From 49840f5ab23f7c3bdaee72e2bdad2d458f4446d9 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 10 Oct 2018 11:31:02 +0100
Subject: [PATCH 809/830] Update newsfile

---
 changelog.d/4022.misc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/changelog.d/4022.misc b/changelog.d/4022.misc
index f1d49932e137..5b0e136795fc 100644
--- a/changelog.d/4022.misc
+++ b/changelog.d/4022.misc
@@ -1 +1 @@
-Add metric to count lazy member sync requests
+Add metric to count number of non-empty sync responses

From 7e561b5c1a0cf0177f14e67851bb7e0ffaeda042 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 10 Oct 2018 11:40:43 +0100
Subject: [PATCH 810/830] Add description to counter metric

---
 synapse/handlers/sync.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 5cae48436fac..351892a94f25 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -44,7 +44,11 @@
 # "true" or "false" depending on if the request asked for lazy loaded members or
 # not.
 non_empty_sync_counter = Counter(
-    "synapse_handlers_sync_nonempty_total", "", ["type", "lazy_loaded"],
+    "synapse_handlers_sync_nonempty_total",
+    "Count of non empty sync responses. type is initial_sync/full_state_sync"
+    "/incremental_sync. lazy_loaded indicates if lazy loaded members were "
+    "enabled for that request.",
+    ["type", "lazy_loaded"],
 )
 
 # Store the cache that tracks which lazy-loaded members have been sent to a given

From 8ddd0f273c501b83b75a58e379d73aef3cfab35e Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 12 Oct 2018 09:55:41 +0100
Subject: [PATCH 811/830] Comments on get_all_new_events_stream

just some docstrings to clarify the behaviour here
---
 synapse/storage/stream.py | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py
index 4c296d72c0f8..d6cfdba51961 100644
--- a/synapse/storage/stream.py
+++ b/synapse/storage/stream.py
@@ -630,7 +630,21 @@ def _get_events_around_txn(
 
     @defer.inlineCallbacks
     def get_all_new_events_stream(self, from_id, current_id, limit):
-        """Get all new events"""
+        """Get all new events
+
+         Returns all events with from_id < stream_ordering <= current_id.
+
+         Args:
+             from_id (int):  the stream_ordering of the last event we processed
+             current_id (int):  the stream_ordering of the most recently processed event
+             limit (int): the maximum number of events to return
+
+         Returns:
+             Deferred[Tuple[int, list[FrozenEvent]]]: A tuple of (next_id, events), where
+             `next_id` is the next value to pass as `from_id` (it will either be the
+             stream_ordering of the last returned event, or, if fewer than `limit` events
+             were found, `current_id`.
+         """
 
         def get_all_new_events_stream_txn(txn):
             sql = (

From 83e72bb2f0c6ef282190a378941c856afbb33c16 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Fri, 12 Oct 2018 11:26:18 +0100
Subject: [PATCH 812/830] PR feedback pt. 1

---
 synapse/api/errors.py                     |  8 -----
 synapse/handlers/e2e_room_keys.py         | 41 ++++++++++++-----------
 synapse/rest/client/v2_alpha/room_keys.py |  2 +-
 3 files changed, 22 insertions(+), 29 deletions(-)

diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index 0a6e78711f29..48b903374d30 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -348,14 +348,6 @@ def error_dict(self):
         )
 
 
-def cs_exception(exception):
-    if isinstance(exception, CodeMessageException):
-        return exception.error_dict()
-    else:
-        logger.error("Unknown exception type: %s", type(exception))
-        return {}
-
-
 def cs_error(msg, code=Codes.UNKNOWN, **kwargs):
     """ Utility method for constructing an error response for client-server
     interactions.
diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py
index bf2a83cc31c9..4e3141dac898 100644
--- a/synapse/handlers/e2e_room_keys.py
+++ b/synapse/handlers/e2e_room_keys.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2017 New Vector Ltd
+# Copyright 2017, 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -49,6 +49,11 @@ def get_room_keys(self, user_id, version, room_id=None, session_id=None):
         room, or a given session.
         See EndToEndRoomKeyStore.get_e2e_room_keys for full details.
 
+        Args:
+            user_id(str): the user whose keys we're getting
+            version(str): the version ID of the backup we're getting keys from
+            room_id(string): room ID to get keys for, for None to get keys for all rooms
+            session_id(string): session ID to get keys for, for None to get keys for all sessions
         Returns:
             A deferred list of dicts giving the session_data and message metadata for
             these room keys.
@@ -72,6 +77,11 @@ def delete_room_keys(self, user_id, version, room_id=None, session_id=None):
         room or a given session.
         See EndToEndRoomKeyStore.delete_e2e_room_keys for full details.
 
+        Args:
+            user_id(str): the user whose backup we're deleting
+            version(str): the version ID of the backup we're deleting
+            room_id(string): room ID to delete keys for, for None to delete keys for all rooms
+            session_id(string): session ID to delete keys for, for None to delete keys for all sessions
         Returns:
             A deferred of the deletion transaction
         """
@@ -118,24 +128,24 @@ def upload_room_keys(self, user_id, version, room_keys):
 
             # Check that the version we're trying to upload is the current version
             try:
-                version_info = yield self._get_version_info_unlocked(user_id)
+                version_info = yield self.store.get_e2e_room_keys_version_info(user_id)
             except StoreError as e:
                 if e.code == 404:
                     raise SynapseError(404, "Version '%s' not found" % (version,))
                 else:
-                    raise e
+                    raise
 
             if version_info['version'] != version:
                 # Check that the version we're trying to upload actually exists
                 try:
-                    version_info = yield self._get_version_info_unlocked(user_id, version)
+                    version_info = yield self.store.get_e2e_room_keys_version_info(user_id, version)
                     # if we get this far, the version must exist
                     raise RoomKeysVersionError(current_version=version_info['version'])
                 except StoreError as e:
                     if e.code == 404:
                         raise SynapseError(404, "Version '%s' not found" % (version,))
                     else:
-                        raise e
+                        raise
 
             # go through the room_keys.
             # XXX: this should/could be done concurrently, given we're in a lock.
@@ -168,9 +178,9 @@ def _upload_room_key(self, user_id, version, room_id, session_id, room_key):
             if e.code == 404:
                 pass
             else:
-                raise e
+                raise
 
-        if E2eRoomKeysHandler._should_replace_room_key(current_room_key, room_key):
+        if self._should_replace_room_key(current_room_key, room_key):
             yield self.store.set_e2e_room_key(
                 user_id, version, room_id, session_id, room_key
             )
@@ -195,14 +205,14 @@ def _should_replace_room_key(current_room_key, room_key):
             # purely for legibility.
 
             if room_key['is_verified'] and not current_room_key['is_verified']:
-                pass
+                return True
             elif (
                 room_key['first_message_index'] <
                 current_room_key['first_message_index']
             ):
-                pass
+                return True
             elif room_key['forwarded_count'] < current_room_key['forwarded_count']:
-                pass
+                return True
             else:
                 return False
         return True
@@ -256,18 +266,9 @@ def get_version_info(self, user_id, version=None):
         """
 
         with (yield self._upload_linearizer.queue(user_id)):
-            res = yield self._get_version_info_unlocked(user_id, version)
+            res = yield self.store.get_e2e_room_keys_version_info(user_id, version)
             defer.returnValue(res)
 
-    @defer.inlineCallbacks
-    def _get_version_info_unlocked(self, user_id, version=None):
-        """Get the info about a given version of the user's backup
-        without obtaining the upload_linearizer lock.  For params see get_version_info
-        """
-
-        results = yield self.store.get_e2e_room_keys_version_info(user_id, version)
-        defer.returnValue(results)
-
     @defer.inlineCallbacks
     def delete_version(self, user_id, version=None):
         """Deletes a given version of the user's e2e_room_keys backup
diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py
index 539893a5d64b..4807170ea6cb 100644
--- a/synapse/rest/client/v2_alpha/room_keys.py
+++ b/synapse/rest/client/v2_alpha/room_keys.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2017 New Vector Ltd
+# Copyright 2017, 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

From 86ef9760a786e7de128120c11d623e3303178505 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Fri, 12 Oct 2018 11:35:08 +0100
Subject: [PATCH 813/830] Split /room_keys/version into 2 servlets

---
 synapse/rest/client/v2_alpha/room_keys.py | 30 ++++++++++++++++-------
 1 file changed, 21 insertions(+), 9 deletions(-)

diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py
index 4807170ea6cb..aee641917972 100644
--- a/synapse/rest/client/v2_alpha/room_keys.py
+++ b/synapse/rest/client/v2_alpha/room_keys.py
@@ -240,9 +240,9 @@ def on_DELETE(self, request, room_id, session_id):
         defer.returnValue((200, {}))
 
 
-class RoomKeysVersionServlet(RestServlet):
+class RoomKeysNewVersionServlet(RestServlet):
     PATTERNS = client_v2_patterns(
-        "/room_keys/version(/(?P<version>[^/]+))?$"
+        "/room_keys/version$"
     )
 
     def __init__(self, hs):
@@ -250,12 +250,12 @@ def __init__(self, hs):
         Args:
             hs (synapse.server.HomeServer): server
         """
-        super(RoomKeysVersionServlet, self).__init__()
+        super(RoomKeysNewVersionServlet, self).__init__()
         self.auth = hs.get_auth()
         self.e2e_room_keys_handler = hs.get_e2e_room_keys_handler()
 
     @defer.inlineCallbacks
-    def on_POST(self, request, version):
+    def on_POST(self, request):
         """
         Create a new backup version for this user's room_keys with the given
         info.  The version is allocated by the server and returned to the user
@@ -285,10 +285,6 @@ def on_POST(self, request, version):
             "version": 12345
         }
         """
-
-        if version:
-            raise SynapseError(405, "Cannot POST to a specific version")
-
         requester = yield self.auth.get_user_by_req(request, allow_guest=False)
         user_id = requester.user.to_string()
         info = parse_json_object_from_request(request)
@@ -301,6 +297,20 @@ def on_POST(self, request, version):
     # we deliberately don't have a PUT /version, as these things really should
     # be immutable to avoid people footgunning
 
+class RoomKeysVersionServlet(RestServlet):
+    PATTERNS = client_v2_patterns(
+        "/room_keys/version(/(?P<version>[^/]+))?$"
+    )
+
+    def __init__(self, hs):
+        """
+        Args:
+            hs (synapse.server.HomeServer): server
+        """
+        super(RoomKeysVersionServlet, self).__init__()
+        self.auth = hs.get_auth()
+        self.e2e_room_keys_handler = hs.get_e2e_room_keys_handler()
+
     @defer.inlineCallbacks
     def on_GET(self, request, version):
         """
@@ -320,7 +330,6 @@ def on_GET(self, request, version):
             "auth_data": "dGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgZW5jcnlwdGVkIGpzb24K"
         }
         """
-
         requester = yield self.auth.get_user_by_req(request, allow_guest=False)
         user_id = requester.user.to_string()
 
@@ -346,6 +355,8 @@ def on_DELETE(self, request, version):
         HTTP/1.1 200 OK
         {}
         """
+        if version is None:
+            raise SynapseError(400, "No version specified to delete")
 
         requester = yield self.auth.get_user_by_req(request, allow_guest=False)
         user_id = requester.user.to_string()
@@ -359,3 +370,4 @@ def on_DELETE(self, request, version):
 def register_servlets(hs, http_server):
     RoomKeysServlet(hs).register(http_server)
     RoomKeysVersionServlet(hs).register(http_server)
+    RoomKeysNewVersionServlet(hs).register(http_server)

From bddfad253a2ca8ea0645d10e89055c4ce2ff1c38 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Fri, 12 Oct 2018 11:48:02 +0100
Subject: [PATCH 814/830] Don't mangle exceptions

---
 synapse/rest/client/v2_alpha/room_keys.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py
index aee641917972..8d006d819da7 100644
--- a/synapse/rest/client/v2_alpha/room_keys.py
+++ b/synapse/rest/client/v2_alpha/room_keys.py
@@ -339,9 +339,7 @@ def on_GET(self, request, version):
             )
         except SynapseError as e:
             if e.code == 404:
-                e.errcode = Codes.NOT_FOUND
-                e.msg = "No backup found"
-                raise e
+                raise SynapseError(404, "No backup found", Codes.NOT_FOUND)
         defer.returnValue((200, info))
 
     @defer.inlineCallbacks
@@ -356,7 +354,7 @@ def on_DELETE(self, request, version):
         {}
         """
         if version is None:
-            raise SynapseError(400, "No version specified to delete")
+            raise SynapseError(400, "No version specified to delete", Codes.NOT_FOUND)
 
         requester = yield self.auth.get_user_by_req(request, allow_guest=False)
         user_id = requester.user.to_string()

From 306361b31bc0fb229de8db29209b9da38374d6f2 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Fri, 12 Oct 2018 11:48:56 +0100
Subject: [PATCH 815/830] Misc PR feedback bits

---
 synapse/storage/e2e_room_keys.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py
index 4d439bb16473..f25ded229520 100644
--- a/synapse/storage/e2e_room_keys.py
+++ b/synapse/storage/e2e_room_keys.py
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import simplejson as json
+import json
 
 from twisted.internet import defer
 
@@ -76,7 +76,7 @@ def set_e2e_room_key(self, user_id, version, room_id, session_id, room_key):
             session_id(str): the session whose room_key we're setting
             room_key(dict): the room_key being set
         Raises:
-            StoreError if stuff goes wrong, probably
+            StoreError
         """
 
         yield self._simple_upsert(

From 8c0ff0287af19cbae40fe7dd413f1e50578908c3 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Fri, 12 Oct 2018 13:47:43 +0100
Subject: [PATCH 816/830] Linting soothes the savage PEP8 monster

---
 synapse/handlers/e2e_room_keys.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py
index 4e3141dac898..5edb3cfe0445 100644
--- a/synapse/handlers/e2e_room_keys.py
+++ b/synapse/handlers/e2e_room_keys.py
@@ -53,7 +53,8 @@ def get_room_keys(self, user_id, version, room_id=None, session_id=None):
             user_id(str): the user whose keys we're getting
             version(str): the version ID of the backup we're getting keys from
             room_id(string): room ID to get keys for, for None to get keys for all rooms
-            session_id(string): session ID to get keys for, for None to get keys for all sessions
+            session_id(string): session ID to get keys for, for None to get keys for all
+                sessions
         Returns:
             A deferred list of dicts giving the session_data and message metadata for
             these room keys.
@@ -80,8 +81,10 @@ def delete_room_keys(self, user_id, version, room_id=None, session_id=None):
         Args:
             user_id(str): the user whose backup we're deleting
             version(str): the version ID of the backup we're deleting
-            room_id(string): room ID to delete keys for, for None to delete keys for all rooms
-            session_id(string): session ID to delete keys for, for None to delete keys for all sessions
+            room_id(string): room ID to delete keys for, for None to delete keys for all
+                rooms
+            session_id(string): session ID to delete keys for, for None to delete keys
+                for all sessions
         Returns:
             A deferred of the deletion transaction
         """
@@ -138,7 +141,9 @@ def upload_room_keys(self, user_id, version, room_keys):
             if version_info['version'] != version:
                 # Check that the version we're trying to upload actually exists
                 try:
-                    version_info = yield self.store.get_e2e_room_keys_version_info(user_id, version)
+                    version_info = yield self.store.get_e2e_room_keys_version_info(
+                        user_id, version,
+                    )
                     # if we get this far, the version must exist
                     raise RoomKeysVersionError(current_version=version_info['version'])
                 except StoreError as e:

From 381d2cfdf0f02935b743f4b6dc1b5133d7ed27b7 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Sat, 13 Oct 2018 00:14:08 +1100
Subject: [PATCH 817/830] Make workers work on Py3 (#4027)

---
 changelog.d/4027.bugfix                       |  1 +
 synapse/app/_base.py                          | 30 +++++++++----------
 synapse/app/event_creator.py                  |  3 ++
 synapse/app/pusher.py                         | 15 +++++-----
 synapse/app/synchrotron.py                    | 12 ++++----
 synapse/python_dependencies.py                |  3 --
 synapse/replication/slave/storage/_base.py    |  9 ++++++
 .../replication/slave/storage/deviceinbox.py  | 12 ++++----
 synapse/replication/slave/storage/devices.py  | 11 +------
 synapse/replication/slave/storage/groups.py   |  8 ++---
 synapse/replication/slave/storage/keys.py     | 14 ++++-----
 synapse/replication/slave/storage/presence.py |  6 ++--
 synctl                                        |  2 +-
 13 files changed, 64 insertions(+), 62 deletions(-)
 create mode 100644 changelog.d/4027.bugfix

diff --git a/changelog.d/4027.bugfix b/changelog.d/4027.bugfix
new file mode 100644
index 000000000000..c9bbff3f685c
--- /dev/null
+++ b/changelog.d/4027.bugfix
@@ -0,0 +1 @@
+Workers now start on Python 3.
diff --git a/synapse/app/_base.py b/synapse/app/_base.py
index 7c866e246af8..18584226e9c4 100644
--- a/synapse/app/_base.py
+++ b/synapse/app/_base.py
@@ -17,6 +17,7 @@
 import logging
 import sys
 
+import psutil
 from daemonize import Daemonize
 
 from twisted.internet import error, reactor
@@ -24,12 +25,6 @@
 from synapse.util import PreserveLoggingContext
 from synapse.util.rlimit import change_resource_limit
 
-try:
-    import affinity
-except Exception:
-    affinity = None
-
-
 logger = logging.getLogger(__name__)
 
 
@@ -89,15 +84,20 @@ def run():
         with PreserveLoggingContext():
             logger.info("Running")
             if cpu_affinity is not None:
-                if not affinity:
-                    quit_with_error(
-                        "Missing package 'affinity' required for cpu_affinity\n"
-                        "option\n\n"
-                        "Install by running:\n\n"
-                        "   pip install affinity\n\n"
-                    )
-                logger.info("Setting CPU affinity to %s" % cpu_affinity)
-                affinity.set_process_affinity_mask(0, cpu_affinity)
+                # Turn the bitmask into bits, reverse it so we go from 0 up
+                mask_to_bits = bin(cpu_affinity)[2:][::-1]
+
+                cpus = []
+                cpu_num = 0
+
+                for i in mask_to_bits:
+                    if i == "1":
+                        cpus.append(cpu_num)
+                    cpu_num += 1
+
+                p = psutil.Process()
+                p.cpu_affinity(cpus)
+
             change_resource_limit(soft_file_limit)
             if gc_thresholds:
                 gc.set_threshold(*gc_thresholds)
diff --git a/synapse/app/event_creator.py b/synapse/app/event_creator.py
index 9060ab14f688..e4a68715aa98 100644
--- a/synapse/app/event_creator.py
+++ b/synapse/app/event_creator.py
@@ -178,6 +178,9 @@ def start(config_options):
 
     setup_logging(config, use_worker_options=True)
 
+    # This should only be done on the user directory worker or the master
+    config.update_user_directory = False
+
     events.USE_FROZEN_DICTS = config.use_frozen_dicts
 
     database_engine = create_engine(config.database_config)
diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py
index 630dcda478fe..0f9f8e19f6ed 100644
--- a/synapse/app/pusher.py
+++ b/synapse/app/pusher.py
@@ -28,6 +28,7 @@
 from synapse.http.site import SynapseSite
 from synapse.metrics import RegistryProxy
 from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
+from synapse.replication.slave.storage._base import __func__
 from synapse.replication.slave.storage.account_data import SlavedAccountDataStore
 from synapse.replication.slave.storage.events import SlavedEventStore
 from synapse.replication.slave.storage.pushers import SlavedPusherStore
@@ -49,31 +50,31 @@ class PusherSlaveStore(
     SlavedAccountDataStore
 ):
     update_pusher_last_stream_ordering_and_success = (
-        DataStore.update_pusher_last_stream_ordering_and_success.__func__
+        __func__(DataStore.update_pusher_last_stream_ordering_and_success)
     )
 
     update_pusher_failing_since = (
-        DataStore.update_pusher_failing_since.__func__
+        __func__(DataStore.update_pusher_failing_since)
     )
 
     update_pusher_last_stream_ordering = (
-        DataStore.update_pusher_last_stream_ordering.__func__
+        __func__(DataStore.update_pusher_last_stream_ordering)
     )
 
     get_throttle_params_by_room = (
-        DataStore.get_throttle_params_by_room.__func__
+        __func__(DataStore.get_throttle_params_by_room)
     )
 
     set_throttle_params = (
-        DataStore.set_throttle_params.__func__
+        __func__(DataStore.set_throttle_params)
     )
 
     get_time_of_last_push_action_before = (
-        DataStore.get_time_of_last_push_action_before.__func__
+        __func__(DataStore.get_time_of_last_push_action_before)
     )
 
     get_profile_displayname = (
-        DataStore.get_profile_displayname.__func__
+        __func__(DataStore.get_profile_displayname)
     )
 
 
diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py
index 9a7fc6ee9d8c..3926c7f2636f 100644
--- a/synapse/app/synchrotron.py
+++ b/synapse/app/synchrotron.py
@@ -33,7 +33,7 @@
 from synapse.http.site import SynapseSite
 from synapse.metrics import RegistryProxy
 from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
-from synapse.replication.slave.storage._base import BaseSlavedStore
+from synapse.replication.slave.storage._base import BaseSlavedStore, __func__
 from synapse.replication.slave.storage.account_data import SlavedAccountDataStore
 from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore
 from synapse.replication.slave.storage.client_ips import SlavedClientIpStore
@@ -147,7 +147,7 @@ def send_stop_syncing(self):
         and haven't come back yet. If there are poke the master about them.
         """
         now = self.clock.time_msec()
-        for user_id, last_sync_ms in self.users_going_offline.items():
+        for user_id, last_sync_ms in list(self.users_going_offline.items()):
             if now - last_sync_ms > 10 * 1000:
                 self.users_going_offline.pop(user_id, None)
                 self.send_user_sync(user_id, False, last_sync_ms)
@@ -156,9 +156,9 @@ def set_state(self, user, state, ignore_status_msg=False):
         # TODO Hows this supposed to work?
         pass
 
-    get_states = PresenceHandler.get_states.__func__
-    get_state = PresenceHandler.get_state.__func__
-    current_state_for_users = PresenceHandler.current_state_for_users.__func__
+    get_states = __func__(PresenceHandler.get_states)
+    get_state = __func__(PresenceHandler.get_state)
+    current_state_for_users = __func__(PresenceHandler.current_state_for_users)
 
     def user_syncing(self, user_id, affect_presence):
         if affect_presence:
@@ -208,7 +208,7 @@ def process_replication_rows(self, token, rows):
         ) for row in rows]
 
         for state in states:
-            self.user_to_current_state[row.user_id] = state
+            self.user_to_current_state[state.user_id] = state
 
         stream_id = token
         yield self.notify_from_replication(states, stream_id)
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index d4d983b00a4c..2947f37f1a3a 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -82,9 +82,6 @@
     "psutil": {
         "psutil>=2.0.0": ["psutil>=2.0.0"],
     },
-    "affinity": {
-        "affinity": ["affinity"],
-    },
     "postgres": {
         "psycopg2>=2.6": ["psycopg2"]
     }
diff --git a/synapse/replication/slave/storage/_base.py b/synapse/replication/slave/storage/_base.py
index 3f7be74e0258..2d81d49e9ade 100644
--- a/synapse/replication/slave/storage/_base.py
+++ b/synapse/replication/slave/storage/_base.py
@@ -15,6 +15,8 @@
 
 import logging
 
+import six
+
 from synapse.storage._base import SQLBaseStore
 from synapse.storage.engines import PostgresEngine
 
@@ -23,6 +25,13 @@
 logger = logging.getLogger(__name__)
 
 
+def __func__(inp):
+    if six.PY3:
+        return inp
+    else:
+        return inp.__func__
+
+
 class BaseSlavedStore(SQLBaseStore):
     def __init__(self, db_conn, hs):
         super(BaseSlavedStore, self).__init__(db_conn, hs)
diff --git a/synapse/replication/slave/storage/deviceinbox.py b/synapse/replication/slave/storage/deviceinbox.py
index 87eaa5300435..4f19fd35aa09 100644
--- a/synapse/replication/slave/storage/deviceinbox.py
+++ b/synapse/replication/slave/storage/deviceinbox.py
@@ -17,7 +17,7 @@
 from synapse.util.caches.expiringcache import ExpiringCache
 from synapse.util.caches.stream_change_cache import StreamChangeCache
 
-from ._base import BaseSlavedStore
+from ._base import BaseSlavedStore, __func__
 from ._slaved_id_tracker import SlavedIdTracker
 
 
@@ -43,11 +43,11 @@ def __init__(self, db_conn, hs):
             expiry_ms=30 * 60 * 1000,
         )
 
-    get_to_device_stream_token = DataStore.get_to_device_stream_token.__func__
-    get_new_messages_for_device = DataStore.get_new_messages_for_device.__func__
-    get_new_device_msgs_for_remote = DataStore.get_new_device_msgs_for_remote.__func__
-    delete_messages_for_device = DataStore.delete_messages_for_device.__func__
-    delete_device_msgs_for_remote = DataStore.delete_device_msgs_for_remote.__func__
+    get_to_device_stream_token = __func__(DataStore.get_to_device_stream_token)
+    get_new_messages_for_device = __func__(DataStore.get_new_messages_for_device)
+    get_new_device_msgs_for_remote = __func__(DataStore.get_new_device_msgs_for_remote)
+    delete_messages_for_device = __func__(DataStore.delete_messages_for_device)
+    delete_device_msgs_for_remote = __func__(DataStore.delete_device_msgs_for_remote)
 
     def stream_positions(self):
         result = super(SlavedDeviceInboxStore, self).stream_positions()
diff --git a/synapse/replication/slave/storage/devices.py b/synapse/replication/slave/storage/devices.py
index 21b8c468facd..ec2fd561cc3e 100644
--- a/synapse/replication/slave/storage/devices.py
+++ b/synapse/replication/slave/storage/devices.py
@@ -13,23 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import six
-
 from synapse.storage import DataStore
 from synapse.storage.end_to_end_keys import EndToEndKeyStore
 from synapse.util.caches.stream_change_cache import StreamChangeCache
 
-from ._base import BaseSlavedStore
+from ._base import BaseSlavedStore, __func__
 from ._slaved_id_tracker import SlavedIdTracker
 
 
-def __func__(inp):
-    if six.PY3:
-        return inp
-    else:
-        return inp.__func__
-
-
 class SlavedDeviceStore(BaseSlavedStore):
     def __init__(self, db_conn, hs):
         super(SlavedDeviceStore, self).__init__(db_conn, hs)
diff --git a/synapse/replication/slave/storage/groups.py b/synapse/replication/slave/storage/groups.py
index 5777f07c8d68..e933b170bb15 100644
--- a/synapse/replication/slave/storage/groups.py
+++ b/synapse/replication/slave/storage/groups.py
@@ -16,7 +16,7 @@
 from synapse.storage import DataStore
 from synapse.util.caches.stream_change_cache import StreamChangeCache
 
-from ._base import BaseSlavedStore
+from ._base import BaseSlavedStore, __func__
 from ._slaved_id_tracker import SlavedIdTracker
 
 
@@ -33,9 +33,9 @@ def __init__(self, db_conn, hs):
             "_group_updates_stream_cache", self._group_updates_id_gen.get_current_token(),
         )
 
-    get_groups_changes_for_user = DataStore.get_groups_changes_for_user.__func__
-    get_group_stream_token = DataStore.get_group_stream_token.__func__
-    get_all_groups_for_user = DataStore.get_all_groups_for_user.__func__
+    get_groups_changes_for_user = __func__(DataStore.get_groups_changes_for_user)
+    get_group_stream_token = __func__(DataStore.get_group_stream_token)
+    get_all_groups_for_user = __func__(DataStore.get_all_groups_for_user)
 
     def stream_positions(self):
         result = super(SlavedGroupServerStore, self).stream_positions()
diff --git a/synapse/replication/slave/storage/keys.py b/synapse/replication/slave/storage/keys.py
index 05ed16846367..8032f53fec1b 100644
--- a/synapse/replication/slave/storage/keys.py
+++ b/synapse/replication/slave/storage/keys.py
@@ -16,7 +16,7 @@
 from synapse.storage import DataStore
 from synapse.storage.keys import KeyStore
 
-from ._base import BaseSlavedStore
+from ._base import BaseSlavedStore, __func__
 
 
 class SlavedKeyStore(BaseSlavedStore):
@@ -24,11 +24,11 @@ class SlavedKeyStore(BaseSlavedStore):
         "_get_server_verify_key"
     ]
 
-    get_server_verify_keys = DataStore.get_server_verify_keys.__func__
-    store_server_verify_key = DataStore.store_server_verify_key.__func__
+    get_server_verify_keys = __func__(DataStore.get_server_verify_keys)
+    store_server_verify_key = __func__(DataStore.store_server_verify_key)
 
-    get_server_certificate = DataStore.get_server_certificate.__func__
-    store_server_certificate = DataStore.store_server_certificate.__func__
+    get_server_certificate = __func__(DataStore.get_server_certificate)
+    store_server_certificate = __func__(DataStore.store_server_certificate)
 
-    get_server_keys_json = DataStore.get_server_keys_json.__func__
-    store_server_keys_json = DataStore.store_server_keys_json.__func__
+    get_server_keys_json = __func__(DataStore.get_server_keys_json)
+    store_server_keys_json = __func__(DataStore.store_server_keys_json)
diff --git a/synapse/replication/slave/storage/presence.py b/synapse/replication/slave/storage/presence.py
index 80b744082a95..92447b00d461 100644
--- a/synapse/replication/slave/storage/presence.py
+++ b/synapse/replication/slave/storage/presence.py
@@ -17,7 +17,7 @@
 from synapse.storage.presence import PresenceStore
 from synapse.util.caches.stream_change_cache import StreamChangeCache
 
-from ._base import BaseSlavedStore
+from ._base import BaseSlavedStore, __func__
 from ._slaved_id_tracker import SlavedIdTracker
 
 
@@ -34,8 +34,8 @@ def __init__(self, db_conn, hs):
             "PresenceStreamChangeCache", self._presence_id_gen.get_current_token()
         )
 
-    _get_active_presence = DataStore._get_active_presence.__func__
-    take_presence_startup_info = DataStore.take_presence_startup_info.__func__
+    _get_active_presence = __func__(DataStore._get_active_presence)
+    take_presence_startup_info = __func__(DataStore.take_presence_startup_info)
     _get_presence_for_user = PresenceStore.__dict__["_get_presence_for_user"]
     get_presence_for_users = PresenceStore.__dict__["get_presence_for_users"]
 
diff --git a/synctl b/synctl
index 356e5cb6a725..09b64459b17e 100755
--- a/synctl
+++ b/synctl
@@ -280,7 +280,7 @@ def main():
             if worker.cache_factor:
                 os.environ["SYNAPSE_CACHE_FACTOR"] = str(worker.cache_factor)
 
-            for cache_name, factor in worker.cache_factors.iteritems():
+            for cache_name, factor in iteritems(worker.cache_factors):
                 os.environ["SYNAPSE_CACHE_FACTOR_" + cache_name.upper()] = str(factor)
 
             start_worker(worker.app, configfile, worker.configfile)

From a45f2c3a002dd50ba4dc6dcbe45a51f2da4057b2 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Fri, 12 Oct 2018 14:33:55 +0100
Subject: [PATCH 818/830] missed one

---
 synapse/rest/client/v2_alpha/room_keys.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py
index 8d006d819da7..45b5817d8b7e 100644
--- a/synapse/rest/client/v2_alpha/room_keys.py
+++ b/synapse/rest/client/v2_alpha/room_keys.py
@@ -297,6 +297,7 @@ def on_POST(self, request):
     # we deliberately don't have a PUT /version, as these things really should
     # be immutable to avoid people footgunning
 
+
 class RoomKeysVersionServlet(RestServlet):
     PATTERNS = client_v2_patterns(
         "/room_keys/version(/(?P<version>[^/]+))?$"

From fb216a22dbceebeee59aedfc6f888b73f4492908 Mon Sep 17 00:00:00 2001
From: Ivan Shapovalov <intelfx@intelfx.name>
Date: Sun, 14 Oct 2018 11:37:04 +0300
Subject: [PATCH 819/830] synapse/visibility.py: fix SyntaxError on py3.7

---
 changelog.d/4033.bugfix | 1 +
 synapse/visibility.py   | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/4033.bugfix

diff --git a/changelog.d/4033.bugfix b/changelog.d/4033.bugfix
new file mode 100644
index 000000000000..8f7a0bbbe911
--- /dev/null
+++ b/changelog.d/4033.bugfix
@@ -0,0 +1 @@
+Synapse now starts on Python 3.7.
diff --git a/synapse/visibility.py b/synapse/visibility.py
index c64ad2144c8b..43f48196bee4 100644
--- a/synapse/visibility.py
+++ b/synapse/visibility.py
@@ -219,7 +219,7 @@ def filter_events_for_server(store, server_name, events):
     # Whatever else we do, we need to check for senders which have requested
     # erasure of their data.
     erased_senders = yield store.are_users_erased(
-        e.sender for e in events,
+        (e.sender for e in events),
     )
 
     def redact_disallowed(event, state):

From 06bc8d2fe551b4390cbca9838bc547c5d0021f6d Mon Sep 17 00:00:00 2001
From: Ivan Shapovalov <intelfx@intelfx.name>
Date: Sun, 14 Oct 2018 11:37:38 +0300
Subject: [PATCH 820/830] synapse/app: frontend_proxy.py: actually make workers
 work on py3

---
 changelog.d/4033.bugfix       | 1 +
 synapse/app/frontend_proxy.py | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/changelog.d/4033.bugfix b/changelog.d/4033.bugfix
index 8f7a0bbbe911..4e9e38cdcf66 100644
--- a/changelog.d/4033.bugfix
+++ b/changelog.d/4033.bugfix
@@ -1 +1,2 @@
 Synapse now starts on Python 3.7.
+_All_ workers now start on Python 3.
diff --git a/synapse/app/frontend_proxy.py b/synapse/app/frontend_proxy.py
index fc4b25de1c38..f5c61dec5b83 100644
--- a/synapse/app/frontend_proxy.py
+++ b/synapse/app/frontend_proxy.py
@@ -68,7 +68,7 @@ def on_GET(self, request, user_id):
             "Authorization": auth_headers,
         }
         result = yield self.http_client.get_json(
-            self.main_uri + request.uri,
+            self.main_uri + request.uri.decode('ascii'),
             headers=headers,
         )
         defer.returnValue((200, result))
@@ -125,7 +125,7 @@ def on_POST(self, request, device_id):
                 "Authorization": auth_headers,
             }
             result = yield self.http_client.post_json_get_json(
-                self.main_uri + request.uri,
+                self.main_uri + request.uri.decode('ascii'),
                 body,
                 headers=headers,
             )

From f726f2dc6c398eb5fdf9d2dc0c56e96d930537ba Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Mon, 15 Oct 2018 22:21:45 +1100
Subject: [PATCH 821/830] version bump

---
 synapse/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/__init__.py b/synapse/__init__.py
index 43c5821ade06..ea07fda14aef 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -27,4 +27,4 @@
 except ImportError:
     pass
 
-__version__ = "0.33.6"
+__version__ = "0.33.7rc1"

From 4e50fe3edb94125d496f2f77da0d653054e3ac3c Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Mon, 15 Oct 2018 22:22:49 +1100
Subject: [PATCH 822/830] update changelog

---
 changelog.d/4033.bugfix | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/changelog.d/4033.bugfix b/changelog.d/4033.bugfix
index 4e9e38cdcf66..05d54d9524f2 100644
--- a/changelog.d/4033.bugfix
+++ b/changelog.d/4033.bugfix
@@ -1,2 +1 @@
-Synapse now starts on Python 3.7.
-_All_ workers now start on Python 3.
+Synapse now starts on Python 3.7.
\ No newline at end of file

From 24bc15eab46992faff9976ddfe9b15c611eab600 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Mon, 15 Oct 2018 22:23:13 +1100
Subject: [PATCH 823/830] update changelog

---
 CHANGES.md               | 28 ++++++++++++++++++++++++++++
 changelog.d/3995.bugfix  |  1 -
 changelog.d/3996.bugfix  |  1 -
 changelog.d/3997.bugfix  |  1 -
 changelog.d/3999.bugfix  |  1 -
 changelog.d/4008.misc    |  1 -
 changelog.d/4017.misc    |  1 -
 changelog.d/4019.feature |  1 -
 changelog.d/4022.misc    |  1 -
 changelog.d/4027.bugfix  |  1 -
 changelog.d/4033.bugfix  |  1 -
 11 files changed, 28 insertions(+), 10 deletions(-)
 delete mode 100644 changelog.d/3995.bugfix
 delete mode 100644 changelog.d/3996.bugfix
 delete mode 100644 changelog.d/3997.bugfix
 delete mode 100644 changelog.d/3999.bugfix
 delete mode 100644 changelog.d/4008.misc
 delete mode 100644 changelog.d/4017.misc
 delete mode 100644 changelog.d/4019.feature
 delete mode 100644 changelog.d/4022.misc
 delete mode 100644 changelog.d/4027.bugfix
 delete mode 100644 changelog.d/4033.bugfix

diff --git a/CHANGES.md b/CHANGES.md
index 048b9f95db95..6c80a8646185 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,31 @@
+Synapse 0.33.7rc1 (2018-10-15)
+==============================
+
+Features
+--------
+
+- Add support for end-to-end key backup (MSC1687) ([\#4019](https://github.com/matrix-org/synapse/issues/4019))
+
+
+Bugfixes
+--------
+
+- Fix bug in event persistence logic which caused 'NoneType is not iterable' ([\#3995](https://github.com/matrix-org/synapse/issues/3995))
+- Fix exception in background metrics collection ([\#3996](https://github.com/matrix-org/synapse/issues/3996))
+- Fix exception handling in fetching remote profiles ([\#3997](https://github.com/matrix-org/synapse/issues/3997))
+- Fix handling of rejected threepid invites ([\#3999](https://github.com/matrix-org/synapse/issues/3999))
+- Workers now start on Python 3. ([\#4027](https://github.com/matrix-org/synapse/issues/4027))
+- Synapse now starts on Python 3.7. ([\#4033](https://github.com/matrix-org/synapse/issues/4033))
+
+
+Internal Changes
+----------------
+
+- Log exceptions in looping calls ([\#4008](https://github.com/matrix-org/synapse/issues/4008))
+- Optimisation for serving federation requests ([\#4017](https://github.com/matrix-org/synapse/issues/4017))
+- Add metric to count number of non-empty sync responses ([\#4022](https://github.com/matrix-org/synapse/issues/4022))
+
+
 Synapse 0.33.6 (2018-10-04)
 ===========================
 
diff --git a/changelog.d/3995.bugfix b/changelog.d/3995.bugfix
deleted file mode 100644
index 2adc36756b79..000000000000
--- a/changelog.d/3995.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix bug in event persistence logic which caused 'NoneType is not iterable'
\ No newline at end of file
diff --git a/changelog.d/3996.bugfix b/changelog.d/3996.bugfix
deleted file mode 100644
index a056485ea143..000000000000
--- a/changelog.d/3996.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix exception in background metrics collection
diff --git a/changelog.d/3997.bugfix b/changelog.d/3997.bugfix
deleted file mode 100644
index b060ee8c1863..000000000000
--- a/changelog.d/3997.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix exception handling in fetching remote profiles
diff --git a/changelog.d/3999.bugfix b/changelog.d/3999.bugfix
deleted file mode 100644
index dc3b2caffa58..000000000000
--- a/changelog.d/3999.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix handling of rejected threepid invites
diff --git a/changelog.d/4008.misc b/changelog.d/4008.misc
deleted file mode 100644
index 573021005490..000000000000
--- a/changelog.d/4008.misc
+++ /dev/null
@@ -1 +0,0 @@
-Log exceptions in looping calls
diff --git a/changelog.d/4017.misc b/changelog.d/4017.misc
deleted file mode 100644
index b1ceb0656062..000000000000
--- a/changelog.d/4017.misc
+++ /dev/null
@@ -1 +0,0 @@
-Optimisation for serving federation requests
\ No newline at end of file
diff --git a/changelog.d/4019.feature b/changelog.d/4019.feature
deleted file mode 100644
index 49e066d269ef..000000000000
--- a/changelog.d/4019.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add support for end-to-end key backup (MSC1687)
diff --git a/changelog.d/4022.misc b/changelog.d/4022.misc
deleted file mode 100644
index 5b0e136795fc..000000000000
--- a/changelog.d/4022.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add metric to count number of non-empty sync responses
diff --git a/changelog.d/4027.bugfix b/changelog.d/4027.bugfix
deleted file mode 100644
index c9bbff3f685c..000000000000
--- a/changelog.d/4027.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Workers now start on Python 3.
diff --git a/changelog.d/4033.bugfix b/changelog.d/4033.bugfix
deleted file mode 100644
index 05d54d9524f2..000000000000
--- a/changelog.d/4033.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Synapse now starts on Python 3.7.
\ No newline at end of file

From 1519572961d22aeaacbe9fccae7788f0f9e72b1c Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 17 Oct 2018 15:44:34 +0100
Subject: [PATCH 824/830] Ship the email templates as package_data

move the example email templates into the synapse package so that they can be
used as package data, which should mean that all of the packaging mechanisms
(pip, docker, debian, arch, etc) should now come with the example templates.

In order to grandfather in people who relied on the templates being in the old
place, check for that situation and fall back to using the defaults if the
templates directory does not exist.
---
 MANIFEST.in                                   |  2 +-
 changelog.d/4052.feature                      | 12 +++++++
 docker/conf/homeserver.yaml                   |  4 ++-
 synapse/config/emailconfig.py                 | 33 +++++++++++++++++--
 synapse/push/mailer.py                        |  5 ++-
 .../res}/templates/mail-Vector.css            |  0
 {res => synapse/res}/templates/mail.css       |  0
 {res => synapse/res}/templates/notif.html     |  0
 {res => synapse/res}/templates/notif.txt      |  0
 .../res}/templates/notif_mail.html            |  0
 {res => synapse/res}/templates/notif_mail.txt |  0
 {res => synapse/res}/templates/room.html      |  0
 {res => synapse/res}/templates/room.txt       |  0
 13 files changed, 50 insertions(+), 6 deletions(-)
 create mode 100644 changelog.d/4052.feature
 rename {res => synapse/res}/templates/mail-Vector.css (100%)
 rename {res => synapse/res}/templates/mail.css (100%)
 rename {res => synapse/res}/templates/notif.html (100%)
 rename {res => synapse/res}/templates/notif.txt (100%)
 rename {res => synapse/res}/templates/notif_mail.html (100%)
 rename {res => synapse/res}/templates/notif_mail.txt (100%)
 rename {res => synapse/res}/templates/room.html (100%)
 rename {res => synapse/res}/templates/room.txt (100%)

diff --git a/MANIFEST.in b/MANIFEST.in
index c6a37ac685a7..25cdf0a61bd6 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -12,12 +12,12 @@ recursive-include synapse/storage/schema *.sql
 recursive-include synapse/storage/schema *.py
 
 recursive-include docs *
-recursive-include res *
 recursive-include scripts *
 recursive-include scripts-dev *
 recursive-include synapse *.pyi
 recursive-include tests *.py
 
+recursive-include synapse/res *
 recursive-include synapse/static *.css
 recursive-include synapse/static *.gif
 recursive-include synapse/static *.html
diff --git a/changelog.d/4052.feature b/changelog.d/4052.feature
new file mode 100644
index 000000000000..0c01f06880cf
--- /dev/null
+++ b/changelog.d/4052.feature
@@ -0,0 +1,12 @@
+Ship the example email templates as part of the package
+
+**Note**: if you deploy your Synapse instance from a git checkout or a github
+snapshot URL, then this means that the example email templates will no longer
+be installed in `res/templates`. If you have email notifications enabled, you
+should ensure that `email.template_dir` is either configured to point at a
+directory where you have installed customised templates, or leave it unset to
+use the default templates.
+
+The configuration parser will try to detect the situation where
+`email.template_dir` is incorrectly set to `res/templates` and do the right
+thing, but will warn about this.
\ No newline at end of file
diff --git a/docker/conf/homeserver.yaml b/docker/conf/homeserver.yaml
index cfe88788f2cc..a38b929f509c 100644
--- a/docker/conf/homeserver.yaml
+++ b/docker/conf/homeserver.yaml
@@ -211,7 +211,9 @@ email:
    require_transport_security: False
    notif_from: "{{ SYNAPSE_SMTP_FROM or "hostmaster@" + SYNAPSE_SERVER_NAME }}"
    app_name: Matrix
-   template_dir: res/templates
+   # if template_dir is unset, uses the example templates that are part of
+   # the Synapse distribution.
+   #template_dir: res/templates
    notif_template_html: notif_mail.html
    notif_template_text: notif_mail.txt
    notif_for_new_users: True
diff --git a/synapse/config/emailconfig.py b/synapse/config/emailconfig.py
index fe156b693056..862f92c6aee7 100644
--- a/synapse/config/emailconfig.py
+++ b/synapse/config/emailconfig.py
@@ -13,11 +13,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
+
 # This file can't be called email.py because if it is, we cannot:
 import email.utils
+import logging
+import os
+import sys
+import textwrap
 
 from ._base import Config
 
+logger = logging.getLogger(__name__)
+
 
 class EmailConfig(Config):
     def read_config(self, config):
@@ -38,7 +46,6 @@ def read_config(self, config):
                 "smtp_host",
                 "smtp_port",
                 "notif_from",
-                "template_dir",
                 "notif_template_html",
                 "notif_template_text",
             ]
@@ -62,9 +69,27 @@ def read_config(self, config):
             self.email_smtp_host = email_config["smtp_host"]
             self.email_smtp_port = email_config["smtp_port"]
             self.email_notif_from = email_config["notif_from"]
-            self.email_template_dir = email_config["template_dir"]
             self.email_notif_template_html = email_config["notif_template_html"]
             self.email_notif_template_text = email_config["notif_template_text"]
+
+            self.email_template_dir = email_config.get("template_dir")
+
+            # backwards-compatibility hack
+            if (
+                self.email_template_dir == "res/templates"
+                and not os.path.isfile(
+                    os.path.join(self.email_template_dir, self.email_notif_template_text)
+                )
+            ):
+                t = """\
+WARNING: The email notifier is configured to look for templates in '%s', but no templates
+could be found there. We will fall back to using the example templates; to get rid of this
+warning, leave 'email.template_dir' unset.
+""" % (self.email_template_dir,)
+
+                print(textwrap.fill(t, width=80) + "\n", file=sys.stderr)
+                self.email_template_dir = None
+
             self.email_notif_for_new_users = email_config.get(
                 "notif_for_new_users", True
             )
@@ -113,7 +138,9 @@ def default_config(self, config_dir_path, server_name, **kwargs):
         #   require_transport_security: False
         #   notif_from: "Your Friendly %(app)s Home Server <noreply@example.com>"
         #   app_name: Matrix
-        #   template_dir: res/templates
+        #   # if template_dir is unset, uses the example templates that are part of
+        #   # the Synapse distribution.
+        #   #template_dir: res/templates
         #   notif_template_html: notif_mail.html
         #   notif_template_text: notif_mail.txt
         #   notif_for_new_users: True
diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py
index 1a5a10d97476..b9dcfee74056 100644
--- a/synapse/push/mailer.py
+++ b/synapse/push/mailer.py
@@ -528,7 +528,10 @@ def load_jinja2_templates(config):
     """
     logger.info("loading jinja2")
 
-    loader = jinja2.FileSystemLoader(config.email_template_dir)
+    if config.email_template_dir:
+        loader = jinja2.FileSystemLoader(config.email_template_dir)
+    else:
+        loader = jinja2.PackageLoader('synapse', 'res/templates')
     env = jinja2.Environment(loader=loader)
     env.filters["format_ts"] = format_ts_filter
     env.filters["mxc_to_http"] = _create_mxc_to_http_filter(config)
diff --git a/res/templates/mail-Vector.css b/synapse/res/templates/mail-Vector.css
similarity index 100%
rename from res/templates/mail-Vector.css
rename to synapse/res/templates/mail-Vector.css
diff --git a/res/templates/mail.css b/synapse/res/templates/mail.css
similarity index 100%
rename from res/templates/mail.css
rename to synapse/res/templates/mail.css
diff --git a/res/templates/notif.html b/synapse/res/templates/notif.html
similarity index 100%
rename from res/templates/notif.html
rename to synapse/res/templates/notif.html
diff --git a/res/templates/notif.txt b/synapse/res/templates/notif.txt
similarity index 100%
rename from res/templates/notif.txt
rename to synapse/res/templates/notif.txt
diff --git a/res/templates/notif_mail.html b/synapse/res/templates/notif_mail.html
similarity index 100%
rename from res/templates/notif_mail.html
rename to synapse/res/templates/notif_mail.html
diff --git a/res/templates/notif_mail.txt b/synapse/res/templates/notif_mail.txt
similarity index 100%
rename from res/templates/notif_mail.txt
rename to synapse/res/templates/notif_mail.txt
diff --git a/res/templates/room.html b/synapse/res/templates/room.html
similarity index 100%
rename from res/templates/room.html
rename to synapse/res/templates/room.html
diff --git a/res/templates/room.txt b/synapse/res/templates/room.txt
similarity index 100%
rename from res/templates/room.txt
rename to synapse/res/templates/room.txt

From c8f2c199910b70a5d94acf5597a4f3ba94cd3969 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 17 Oct 2018 16:56:22 +0100
Subject: [PATCH 825/830] Put the warning blob at the top of the file

---
 synapse/config/emailconfig.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/synapse/config/emailconfig.py b/synapse/config/emailconfig.py
index 862f92c6aee7..e2582cfecc2a 100644
--- a/synapse/config/emailconfig.py
+++ b/synapse/config/emailconfig.py
@@ -26,6 +26,12 @@
 
 logger = logging.getLogger(__name__)
 
+TEMPLATE_DIR_WARNING = """\
+WARNING: The email notifier is configured to look for templates in '%(template_dir)s',
+but no templates could be found there. We will fall back to using the example templates;
+to get rid of this warning, leave 'email.template_dir' unset.
+"""
+
 
 class EmailConfig(Config):
     def read_config(self, config):
@@ -81,12 +87,9 @@ def read_config(self, config):
                     os.path.join(self.email_template_dir, self.email_notif_template_text)
                 )
             ):
-                t = """\
-WARNING: The email notifier is configured to look for templates in '%s', but no templates
-could be found there. We will fall back to using the example templates; to get rid of this
-warning, leave 'email.template_dir' unset.
-""" % (self.email_template_dir,)
-
+                t = TEMPLATE_DIR_WARNING % {
+                    "template_dir": self.email_template_dir,
+                }
                 print(textwrap.fill(t, width=80) + "\n", file=sys.stderr)
                 self.email_template_dir = None
 

From 0fd2321629f4a06bec8b552299d3ced32a7a021c Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 16 Oct 2018 20:37:16 +0100
Subject: [PATCH 826/830] Fix incorrect truncation in get_missing_events

It's quite important that get_missing_events returns the *latest* events in the
room; however we were pulling event ids out of the database until we got *at
least* 10, and then taking the *earliest* of the results.

We also shouldn't really be relying on depth, and should be checking the
room_id.
---
 changelog.d/4045.bugfix                 |  1 +
 synapse/federation/federation_server.py |  8 +++---
 synapse/federation/transport/server.py  |  2 --
 synapse/handlers/federation.py          | 12 ++++----
 synapse/storage/event_federation.py     | 38 +++++++++++--------------
 5 files changed, 26 insertions(+), 35 deletions(-)
 create mode 100644 changelog.d/4045.bugfix

diff --git a/changelog.d/4045.bugfix b/changelog.d/4045.bugfix
new file mode 100644
index 000000000000..fa50eb5aff8c
--- /dev/null
+++ b/changelog.d/4045.bugfix
@@ -0,0 +1 @@
+Fix bug which made get_missing_events return too few events
\ No newline at end of file
diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index 819e8f73315e..4efe95faa40d 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -507,19 +507,19 @@ def on_claim_client_keys(self, origin, content):
     @defer.inlineCallbacks
     @log_function
     def on_get_missing_events(self, origin, room_id, earliest_events,
-                              latest_events, limit, min_depth):
+                              latest_events, limit):
         with (yield self._server_linearizer.queue((origin, room_id))):
             origin_host, _ = parse_server_name(origin)
             yield self.check_server_matches_acl(origin_host, room_id)
 
             logger.info(
                 "on_get_missing_events: earliest_events: %r, latest_events: %r,"
-                " limit: %d, min_depth: %d",
-                earliest_events, latest_events, limit, min_depth
+                " limit: %d",
+                earliest_events, latest_events, limit,
             )
 
             missing_events = yield self.handler.on_get_missing_events(
-                origin, room_id, earliest_events, latest_events, limit, min_depth
+                origin, room_id, earliest_events, latest_events, limit,
             )
 
             if len(missing_events) < 5:
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 2f874b483815..7288d49074ef 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -560,7 +560,6 @@ class FederationGetMissingEventsServlet(BaseFederationServlet):
     @defer.inlineCallbacks
     def on_POST(self, origin, content, query, room_id):
         limit = int(content.get("limit", 10))
-        min_depth = int(content.get("min_depth", 0))
         earliest_events = content.get("earliest_events", [])
         latest_events = content.get("latest_events", [])
 
@@ -569,7 +568,6 @@ def on_POST(self, origin, content, query, room_id):
             room_id=room_id,
             earliest_events=earliest_events,
             latest_events=latest_events,
-            min_depth=min_depth,
             limit=limit,
         )
 
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 45d955e6f545..cab57a88490e 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -309,8 +309,8 @@ def on_receive_pdu(
 
                 if sent_to_us_directly:
                     logger.warn(
-                        "[%s %s] Failed to fetch %d prev events: rejecting",
-                        room_id, event_id, len(prevs - seen),
+                        "[%s %s] Rejecting: failed to fetch %d prev events: %s",
+                        room_id, event_id, len(prevs - seen), shortstr(prevs - seen)
                     )
                     raise FederationError(
                         "ERROR",
@@ -452,8 +452,8 @@ def _get_missing_events_for_pdu(self, origin, pdu, prevs, min_depth):
         latest |= seen
 
         logger.info(
-            "[%s %s]: Requesting %d prev_events: %s",
-            room_id, event_id, len(prevs - seen), shortstr(prevs - seen)
+            "[%s %s]: Requesting missing events between %s and %s",
+            room_id, event_id, shortstr(latest), event_id,
         )
 
         # XXX: we set timeout to 10s to help workaround
@@ -1852,7 +1852,7 @@ def on_query_auth(self, origin, event_id, room_id, remote_auth_chain, rejects,
 
     @defer.inlineCallbacks
     def on_get_missing_events(self, origin, room_id, earliest_events,
-                              latest_events, limit, min_depth):
+                              latest_events, limit):
         in_room = yield self.auth.check_host_in_room(
             room_id,
             origin
@@ -1861,14 +1861,12 @@ def on_get_missing_events(self, origin, room_id, earliest_events,
             raise AuthError(403, "Host not in room.")
 
         limit = min(limit, 20)
-        min_depth = max(min_depth, 0)
 
         missing_events = yield self.store.get_missing_events(
             room_id=room_id,
             earliest_events=earliest_events,
             latest_events=latest_events,
             limit=limit,
-            min_depth=min_depth,
         )
 
         missing_events = yield filter_events_for_server(
diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py
index 24345b20a6fa..3faca2a04282 100644
--- a/synapse/storage/event_federation.py
+++ b/synapse/storage/event_federation.py
@@ -376,33 +376,25 @@ def _get_backfill_events(self, txn, room_id, event_list, limit):
 
     @defer.inlineCallbacks
     def get_missing_events(self, room_id, earliest_events, latest_events,
-                           limit, min_depth):
+                           limit):
         ids = yield self.runInteraction(
             "get_missing_events",
             self._get_missing_events,
-            room_id, earliest_events, latest_events, limit, min_depth
+            room_id, earliest_events, latest_events, limit,
         )
-
         events = yield self._get_events(ids)
-
-        events = sorted(
-            [ev for ev in events if ev.depth >= min_depth],
-            key=lambda e: e.depth,
-        )
-
-        defer.returnValue(events[:limit])
+        defer.returnValue(events)
 
     def _get_missing_events(self, txn, room_id, earliest_events, latest_events,
-                            limit, min_depth):
-
-        earliest_events = set(earliest_events)
-        front = set(latest_events) - earliest_events
+                            limit):
 
-        event_results = set()
+        seen_events = set(earliest_events)
+        front = set(latest_events) - seen_events
+        event_results = []
 
         query = (
             "SELECT prev_event_id FROM event_edges "
-            "WHERE event_id = ? AND is_state = ? "
+            "WHERE room_id = ? AND event_id = ? AND is_state = ? "
             "LIMIT ?"
         )
 
@@ -411,18 +403,20 @@ def _get_missing_events(self, txn, room_id, earliest_events, latest_events,
             for event_id in front:
                 txn.execute(
                     query,
-                    (event_id, False, limit - len(event_results))
+                    (room_id, event_id, False, limit - len(event_results))
                 )
 
-                for e_id, in txn:
-                    new_front.add(e_id)
+                new_results = set(t[0] for t in txn) - seen_events
 
-            new_front -= earliest_events
-            new_front -= event_results
+                new_front |= new_results
+                seen_events |= new_results
+                event_results.extend(new_results)
 
             front = new_front
-            event_results |= new_front
 
+        # we built the list working backwards from latest_events; we now need to
+        # reverse it so that the events are approximately chronological.
+        event_results.reverse()
         return event_results
 
 

From 52e3d3813baf82dc16aae42bdd108cf296c3ead7 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 17 Oct 2018 17:40:01 +0100
Subject: [PATCH 827/830] prep changelog

---
 CHANGES.md               | 25 +++++++++++++++++++++++++
 changelog.d/4045.bugfix  |  1 -
 changelog.d/4052.feature | 12 ------------
 3 files changed, 25 insertions(+), 13 deletions(-)
 delete mode 100644 changelog.d/4045.bugfix
 delete mode 100644 changelog.d/4052.feature

diff --git a/CHANGES.md b/CHANGES.md
index 6c80a8646185..2598bfc3330c 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,28 @@
+Synapse 0.33.7rc2 (2018-10-17)
+==============================
+
+**Warning**: This release removes the example email notification templates from `res/templates` (they are now internal to the python package). This should only affect you if you (a) deploy your Synapse instance from a git checkout or a github snapshot URL, and (b) have email notifications enabled.
+
+If you have email notifications enabled, you should ensure that
+`email.template_dir` is either configured to point at a directory where you
+have installed customised templates, or leave it unset to use the default
+templates.
+
+The configuration parser will try to detect the situation where
+`email.template_dir` is incorrectly set to `res/templates` and do the right
+thing, but will warn about this.
+
+Features
+--------
+
+- Ship the example email templates as part of the package ([\#4052](https://github.com/matrix-org/synapse/issues/4052))
+
+Bugfixes
+--------
+
+- Fix bug which made get_missing_events return too few events ([\#4045](https://github.com/matrix-org/synapse/issues/4045))
+
+
 Synapse 0.33.7rc1 (2018-10-15)
 ==============================
 
diff --git a/changelog.d/4045.bugfix b/changelog.d/4045.bugfix
deleted file mode 100644
index fa50eb5aff8c..000000000000
--- a/changelog.d/4045.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix bug which made get_missing_events return too few events
\ No newline at end of file
diff --git a/changelog.d/4052.feature b/changelog.d/4052.feature
deleted file mode 100644
index 0c01f06880cf..000000000000
--- a/changelog.d/4052.feature
+++ /dev/null
@@ -1,12 +0,0 @@
-Ship the example email templates as part of the package
-
-**Note**: if you deploy your Synapse instance from a git checkout or a github
-snapshot URL, then this means that the example email templates will no longer
-be installed in `res/templates`. If you have email notifications enabled, you
-should ensure that `email.template_dir` is either configured to point at a
-directory where you have installed customised templates, or leave it unset to
-use the default templates.
-
-The configuration parser will try to detect the situation where
-`email.template_dir` is incorrectly set to `res/templates` and do the right
-thing, but will warn about this.
\ No newline at end of file

From c7d0f34a3c0217be4ca06d934b66fcdfc22a7219 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 17 Oct 2018 17:40:19 +0100
Subject: [PATCH 828/830] v0.33.7rc2

---
 synapse/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/__init__.py b/synapse/__init__.py
index ea07fda14aef..a0c4ef64883a 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -27,4 +27,4 @@
 except ImportError:
     pass
 
-__version__ = "0.33.7rc1"
+__version__ = "0.33.7rc2"

From 8c2b8d7f0bfb5043c5e22f09fa69f01b4656d3d2 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 17 Oct 2018 17:42:12 +0100
Subject: [PATCH 829/830] fix changelog formatting

---
 CHANGES.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/CHANGES.md b/CHANGES.md
index 2598bfc3330c..7de66d6ab49d 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,7 +1,10 @@
 Synapse 0.33.7rc2 (2018-10-17)
 ==============================
 
-**Warning**: This release removes the example email notification templates from `res/templates` (they are now internal to the python package). This should only affect you if you (a) deploy your Synapse instance from a git checkout or a github snapshot URL, and (b) have email notifications enabled.
+**Warning**: This release removes the example email notification templates from
+`res/templates` (they are now internal to the python package). This should only
+affect you if you (a) deploy your Synapse instance from a git checkout or a
+github snapshot URL, and (b) have email notifications enabled.
 
 If you have email notifications enabled, you should ensure that
 `email.template_dir` is either configured to point at a directory where you

From 926da4dda84178b83fa00813d31f1bceef25171f Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 18 Oct 2018 14:57:32 +0100
Subject: [PATCH 830/830] 0.33.7

---
 CHANGES.md          | 7 +++++--
 synapse/__init__.py | 2 +-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 7de66d6ab49d..5f598559a035 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,5 @@
-Synapse 0.33.7rc2 (2018-10-17)
-==============================
+Synapse 0.33.7 (2018-10-18)
+===========================
 
 **Warning**: This release removes the example email notification templates from
 `res/templates` (they are now internal to the python package). This should only
@@ -15,6 +15,9 @@ The configuration parser will try to detect the situation where
 `email.template_dir` is incorrectly set to `res/templates` and do the right
 thing, but will warn about this.
 
+Synapse 0.33.7rc2 (2018-10-17)
+==============================
+
 Features
 --------
 
diff --git a/synapse/__init__.py b/synapse/__init__.py
index a0c4ef64883a..1ddbbbebfbf9 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -27,4 +27,4 @@
 except ImportError:
     pass
 
-__version__ = "0.33.7rc2"
+__version__ = "0.33.7"