Merge branch 'develop' into travis/rver-cap

UPGRADE.rst

@@ -18,7 +18,7 @@ instructions that may be required are listed later in this document.
 
    .. code:: bash
 
-       pip install --upgrade --process-dependency-links matrix-synapse
+       pip install --upgrade matrix-synapse
 
        # restart synapse
        synctl restart

changelog.d/4405.bugfix

@@ -0,0 +1 @@
+Fix bug when rejecting remote invites

changelog.d/4412.bugfix

@@ -0,0 +1 @@
+Copy over whether a room is a direct message and any associated room tags on room upgrade.

changelog.d/4415.feature

@@ -0,0 +1 @@
+Search now includes results from predecessor rooms after a room upgrade.

changelog.d/4437.misc

@@ -0,0 +1 @@
+Add infrastructure to support different event formats

changelog.d/4447.misc

@@ -0,0 +1 @@
+Add infrastructure to support different event formats

changelog.d/4448.misc

@@ -0,0 +1 @@
+Add infrastructure to support different event formats

changelog.d/4466.misc

@@ -0,0 +1 @@
+Synapse will now take advantage of native UPSERT functionality in PostgreSQL 9.5+ and SQLite 3.24+.

changelog.d/4468.misc

@@ -0,0 +1 @@
+Move SRV logic into the Agent layer

changelog.d/4470.misc

@@ -0,0 +1 @@
+Add infrastructure to support different event formats

changelog.d/4471.misc

@@ -0,0 +1 @@
+ Synapse will now take advantage of native UPSERT functionality in PostgreSQL 9.5+ and SQLite 3.24+.

changelog.d/4476.misc

@@ -0,0 +1 @@
+Fix quoting for allowed_local_3pids example config

changelog.d/4477.misc

@@ -0,0 +1 @@
+Synapse will now take advantage of native UPSERT functionality in PostgreSQL 9.5+ and SQLite 3.24+.

changelog.d/4482.misc

@@ -0,0 +1 @@
+Add infrastructure to support different event formats

changelog.d/4485.misc

@@ -0,0 +1 @@
+Remove deprecated --process-dependency-links option from UPGRADE.rst

changelog.d/4487.misc

@@ -0,0 +1 @@
+Fix idna and ipv6 literal handling in MatrixFederationAgent

changelog.d/4488.feature

@@ -0,0 +1 @@
+Implement MSC1708 (.well-known routing for server-server federation)

changelog.d/4492.feature

@@ -0,0 +1 @@
+ Synapse can now automatically provision TLS certificates via ACME (the protocol used by CAs like Let's Encrypt).

synapse/api/auth.py

@@ -65,7 +65,7 @@ def __init__(self, hs):
         register_cache("cache", "token_cache", self.token_cache)
 
     @defer.inlineCallbacks
-    def check_from_context(self, event, context, do_sig_check=True):
+    def check_from_context(self, room_version, event, context, do_sig_check=True):
         prev_state_ids = yield context.get_prev_state_ids(self.store)
         auth_events_ids = yield self.compute_auth_events(
             event, prev_state_ids, for_verification=True,
@@ -74,12 +74,16 @@ def check_from_context(self, event, context, do_sig_check=True):
         auth_events = {
             (e.type, e.state_key): e for e in itervalues(auth_events)
         }
-        self.check(event, auth_events=auth_events, do_sig_check=do_sig_check)
+        self.check(
+            room_version, event,
+            auth_events=auth_events, do_sig_check=do_sig_check,
+        )
 
-    def check(self, event, auth_events, do_sig_check=True):
+    def check(self, room_version, event, auth_events, do_sig_check=True):
         """ Checks if this event is correctly authed.
 
         Args:
+            room_version (str): version of the room
             event: the event being checked.
             auth_events (dict: event-key -> event): the existing room state.
 
@@ -88,7 +92,9 @@ def check(self, event, auth_events, do_sig_check=True):
             True if the auth checks pass.
         """
         with Measure(self.clock, "auth.check"):
-            event_auth.check(event, auth_events, do_sig_check=do_sig_check)
+            event_auth.check(
+                room_version, event, auth_events, do_sig_check=do_sig_check
+            )
 
     @defer.inlineCallbacks
     def check_joined_room(self, room_id, user_id, current_state=None):

synapse/api/constants.py

@@ -121,6 +121,19 @@ class RoomVersions(object):
     RoomVersions.STATE_V2_TEST,
 }
 
+
+class EventFormatVersions(object):
+    """This is an internal enum for tracking the version of the event format,
+    independently from the room version.
+    """
+    V1 = 1
+
+
+KNOWN_EVENT_FORMAT_VERSIONS = {
+    EventFormatVersions.V1,
+}
+
+
 ServerNoticeMsgType = "m.server_notice"
 ServerNoticeLimitReached = "m.server_notice.usage_limit_reached"
 

synapse/api/filtering.py

@@ -444,6 +444,20 @@ def lazy_load_members(self):
     def include_redundant_members(self):
         return self.filter_json.get("include_redundant_members", False)
 
+    def with_room_ids(self, room_ids):
+        """Returns a new filter with the given room IDs appended.
+
+        Args:
+            room_ids (iterable[unicode]): The room_ids to add
+
+        Returns:
+            filter: A new filter including the given rooms and the old
+                    filter's rooms.
+        """
+        newFilter = Filter(self.filter_json)
+        newFilter.rooms += room_ids
+        return newFilter
+
 
 def _matches_wildcard(actual_value, filter_value):
     if filter_value.endswith("*"):

synapse/app/client_reader.py

@@ -164,23 +164,23 @@ def start(config_options):
 
     database_engine = create_engine(config.database_config)
 
-    tls_server_context_factory = context_factory.ServerContextFactory(config)
-    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
-
     ss = ClientReaderServer(
         config.server_name,
         db_config=config.database_config,
-        tls_server_context_factory=tls_server_context_factory,
-        tls_client_options_factory=tls_client_options_factory,
         config=config,
         version_string="Synapse/" + get_version_string(synapse),
         database_engine=database_engine,
     )
 
     ss.setup()
-    ss.start_listening(config.worker_listeners)
 
     def start():
+        ss.config.read_certificate_from_disk()
+        ss.tls_server_context_factory = context_factory.ServerContextFactory(config)
+        ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
+            config
+        )
+        ss.start_listening(config.worker_listeners)
         ss.get_datastore().start_profiling()
 
     reactor.callWhenRunning(start)

synapse/app/event_creator.py

@@ -185,23 +185,23 @@ def start(config_options):
 
     database_engine = create_engine(config.database_config)
 
-    tls_server_context_factory = context_factory.ServerContextFactory(config)
-    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
-
     ss = EventCreatorServer(
         config.server_name,
         db_config=config.database_config,
-        tls_server_context_factory=tls_server_context_factory,
-        tls_client_options_factory=tls_client_options_factory,
         config=config,
         version_string="Synapse/" + get_version_string(synapse),
         database_engine=database_engine,
     )
 
     ss.setup()
-    ss.start_listening(config.worker_listeners)
 
     def start():
+        ss.config.read_certificate_from_disk()
+        ss.tls_server_context_factory = context_factory.ServerContextFactory(config)
+        ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
+            config
+        )
+        ss.start_listening(config.worker_listeners)
         ss.get_datastore().start_profiling()
 
     reactor.callWhenRunning(start)

synapse/app/federation_reader.py

@@ -151,23 +151,23 @@ def start(config_options):
 
     database_engine = create_engine(config.database_config)
 
-    tls_server_context_factory = context_factory.ServerContextFactory(config)
-    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
-
     ss = FederationReaderServer(
         config.server_name,
         db_config=config.database_config,
-        tls_server_context_factory=tls_server_context_factory,
-        tls_client_options_factory=tls_client_options_factory,
         config=config,
         version_string="Synapse/" + get_version_string(synapse),
         database_engine=database_engine,
     )
 
     ss.setup()
-    ss.start_listening(config.worker_listeners)
 
     def start():
+        ss.config.read_certificate_from_disk()
+        ss.tls_server_context_factory = context_factory.ServerContextFactory(config)
+        ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
+            config
+        )
+        ss.start_listening(config.worker_listeners)
         ss.get_datastore().start_profiling()
 
     reactor.callWhenRunning(start)

synapse/app/federation_sender.py

@@ -183,24 +183,24 @@ def start(config_options):
     # Force the pushers to start since they will be disabled in the main config
     config.send_federation = True
 
-    tls_server_context_factory = context_factory.ServerContextFactory(config)
-    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
-
-    ps = FederationSenderServer(
+    ss = FederationSenderServer(
         config.server_name,
         db_config=config.database_config,
-        tls_server_context_factory=tls_server_context_factory,
-        tls_client_options_factory=tls_client_options_factory,
         config=config,
         version_string="Synapse/" + get_version_string(synapse),
         database_engine=database_engine,
     )
 
-    ps.setup()
-    ps.start_listening(config.worker_listeners)
+    ss.setup()
 
     def start():
-        ps.get_datastore().start_profiling()
+        ss.config.read_certificate_from_disk()
+        ss.tls_server_context_factory = context_factory.ServerContextFactory(config)
+        ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
+            config
+        )
+        ss.start_listening(config.worker_listeners)
+        ss.get_datastore().start_profiling()
 
     reactor.callWhenRunning(start)
     _base.start_worker_reactor("synapse-federation-sender", config)

synapse/app/frontend_proxy.py

@@ -241,23 +241,23 @@ def start(config_options):
 
     database_engine = create_engine(config.database_config)
 
-    tls_server_context_factory = context_factory.ServerContextFactory(config)
-    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
-
     ss = FrontendProxyServer(
         config.server_name,
         db_config=config.database_config,
-        tls_server_context_factory=tls_server_context_factory,
-        tls_client_options_factory=tls_client_options_factory,
         config=config,
         version_string="Synapse/" + get_version_string(synapse),
         database_engine=database_engine,
     )
 
     ss.setup()
-    ss.start_listening(config.worker_listeners)
 
     def start():
+        ss.config.read_certificate_from_disk()
+        ss.tls_server_context_factory = context_factory.ServerContextFactory(config)
+        ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
+            config
+        )
+        ss.start_listening(config.worker_listeners)
         ss.get_datastore().start_profiling()
 
     reactor.callWhenRunning(start)

synapse/app/media_repository.py

@@ -151,23 +151,23 @@ def start(config_options):
 
     database_engine = create_engine(config.database_config)
 
-    tls_server_context_factory = context_factory.ServerContextFactory(config)
-    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
-
     ss = MediaRepositoryServer(
         config.server_name,
         db_config=config.database_config,
-        tls_server_context_factory=tls_server_context_factory,
-        tls_client_options_factory=tls_client_options_factory,
         config=config,
         version_string="Synapse/" + get_version_string(synapse),
         database_engine=database_engine,
     )
 
     ss.setup()
-    ss.start_listening(config.worker_listeners)
 
     def start():
+        ss.config.read_certificate_from_disk()
+        ss.tls_server_context_factory = context_factory.ServerContextFactory(config)
+        ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
+            config
+        )
+        ss.start_listening(config.worker_listeners)
         ss.get_datastore().start_profiling()
 
     reactor.callWhenRunning(start)

synapse/app/user_dir.py

@@ -211,24 +211,24 @@ def start(config_options):
     # Force the pushers to start since they will be disabled in the main config
     config.update_user_directory = True
 
-    tls_server_context_factory = context_factory.ServerContextFactory(config)
-    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
-
-    ps = UserDirectoryServer(
+    ss = UserDirectoryServer(
         config.server_name,
         db_config=config.database_config,
-        tls_server_context_factory=tls_server_context_factory,
-        tls_client_options_factory=tls_client_options_factory,
         config=config,
         version_string="Synapse/" + get_version_string(synapse),
         database_engine=database_engine,
     )
 
-    ps.setup()
-    ps.start_listening(config.worker_listeners)
+    ss.setup()
 
     def start():
-        ps.get_datastore().start_profiling()
+        ss.config.read_certificate_from_disk()
+        ss.tls_server_context_factory = context_factory.ServerContextFactory(config)
+        ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
+            config
+        )
+        ss.start_listening(config.worker_listeners)
+        ss.get_datastore().start_profiling()
 
     reactor.callWhenRunning(start)
 

synapse/config/registration.py

@@ -84,11 +84,11 @@ def default_config(self, generate_secrets=False, **kwargs):
         #
         # allowed_local_3pids:
         #     - medium: email
-        #       pattern: ".*@matrix\\.org"
+        #       pattern: '.*@matrix\\.org'
         #     - medium: email
-        #       pattern: ".*@vector\\.im"
+        #       pattern: '.*@vector\\.im'
         #     - medium: msisdn
-        #       pattern: "\\+44"
+        #       pattern: '\\+44'
 
         # If set, allows registration by anyone who also has the shared
         # secret, even if registration is otherwise disabled.

synapse/event_auth.py

@@ -27,10 +27,11 @@
 logger = logging.getLogger(__name__)
 
 
-def check(event, auth_events, do_sig_check=True, do_size_check=True):
+def check(room_version, event, auth_events, do_sig_check=True, do_size_check=True):
     """ Checks if this event is correctly authed.
 
     Args:
+        room_version (str): the version of the room
         event: the event being checked.
         auth_events (dict: event-key -> event): the existing room state.