From 8a28ba94b092eacbcfbc259466ad85621209e46c Mon Sep 17 00:00:00 2001 From: Jason Little Date: Sat, 15 Oct 2022 23:20:44 -0500 Subject: [PATCH 01/21] Add templating and comments for missing stream writers to configure_workers_and_start.py. --- docker/configure_workers_and_start.py | 50 ++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 60a5c10ea75c..67405aa72a82 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -50,7 +50,12 @@ MAIN_PROCESS_HTTP_LISTENER_PORT = 8080 - +# Workers with exposed endpoints needs either "client", "federation", or "media" listener_resources +# Watching /_matrix/client needs a "client" listener +# Watching /_matrix/federation needs a "federation" listener +# Watching /_matrix/media and related needs a "media" listener +# Stream Writers require "client" and "replication" listeners because they +# have to attach by instance_map to the master process and have client endpoints. WORKERS_CONFIG: Dict[str, Dict[str, Any]] = { "pusher": { "app": "synapse.app.pusher", @@ -209,6 +214,49 @@ % (MAIN_PROCESS_HTTP_LISTENER_PORT,) ), }, + "account_data": { + "app": "synapse.app.generic_worker", + "listener_resources": ["client", "replication"], + "endpoint_patterns": [ + "^/_matrix/client/(r0|v3|unstable)/.*/tags", + "^/_matrix/client/(r0|v3|unstable)/.*/account_data", + ], + "shared_extra_conf": {}, + "worker_extra_conf": "", + }, + "presence": { + "app": "synapse.app.generic_worker", + "listener_resources": ["client", "replication"], + "endpoint_patterns": ["^/_matrix/client/(api/v1|r0|v3|unstable)/presence/"], + "shared_extra_conf": {}, + "worker_extra_conf": "", + }, + "receipts": { + "app": "synapse.app.generic_worker", + "listener_resources": ["client", "replication"], + "endpoint_patterns": [ + "^/_matrix/client/(r0|v3|unstable)/rooms/.*/receipt", + "^/_matrix/client/(r0|v3|unstable)/rooms/.*/read_markers", + ], + "shared_extra_conf": {}, + "worker_extra_conf": "", + }, + "to_device": { + "app": "synapse.app.generic_worker", + "listener_resources": ["client", "replication"], + "endpoint_patterns": ["^/_matrix/client/(r0|v3|unstable)/sendToDevice/"], + "shared_extra_conf": {}, + "worker_extra_conf": "", + }, + "typing": { + "app": "synapse.app.generic_worker", + "listener_resources": ["client", "replication"], + "endpoint_patterns": [ + "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/typing" + ], + "shared_extra_conf": {}, + "worker_extra_conf": "", + }, } # Templates for sections that may be inserted multiple times in config files From 52628efb1ac7289991d5b594771d54af55288a29 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Sat, 15 Oct 2022 23:23:06 -0500 Subject: [PATCH 02/21] Add warning comment about non-functional stream writers. --- docker/configure_workers_and_start.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 67405aa72a82..f95150acd019 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -56,6 +56,7 @@ # Watching /_matrix/media and related needs a "media" listener # Stream Writers require "client" and "replication" listeners because they # have to attach by instance_map to the master process and have client endpoints. +# BIG WARNING: typing and receipts stream writers are not working correctly at this time. WORKERS_CONFIG: Dict[str, Dict[str, Any]] = { "pusher": { "app": "synapse.app.pusher", From 2f7fedb6c0a54427d5de320ae16288ad56676220 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Sun, 16 Oct 2022 02:24:52 -0500 Subject: [PATCH 03/21] Add stream_writer map and instance_map handling to add_sharding_to_shared_config(). --- docker/configure_workers_and_start.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index f95150acd019..695238f2e802 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -363,6 +363,20 @@ def add_sharding_to_shared_config( "port": worker_port, } + elif worker_type in ["account_data", "presence", "receipts", "to_device", "typing"]: + # Update the list of stream writers + # It's convienent that the name of the worker type is the same as the event stream + shared_config.setdefault("stream_writers", {}).setdefault( + worker_type, [] + ).append(worker_name) + + # Map of stream writer instance names to host/ports combos + # For now, all stream writers need http replication ports + instance_map[worker_name] = { + "host": "localhost", + "port": worker_port, + } + elif worker_type == "media_repository": # The first configured media worker will run the media background jobs shared_config.setdefault("media_instance_running_background_jobs", worker_name) From adb34bcefdc21ed53741076a20704b7ca0902fb3 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Sun, 16 Oct 2022 03:41:20 -0500 Subject: [PATCH 04/21] Remove unnecessary gating for running add_sharding_to_shared_config(). --- docker/configure_workers_and_start.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 695238f2e802..c8307ff41f61 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -509,11 +509,11 @@ def generate_worker_files( # Check if more than one instance of this worker type has been specified worker_type_total_count = worker_types.count(worker_type) - if worker_type_total_count > 1: - # Update the shared config with sharding-related options if necessary - add_sharding_to_shared_config( - shared_config, worker_type, worker_name, worker_port - ) + + # Update the shared config with sharding-related options if necessary + add_sharding_to_shared_config( + shared_config, worker_type, worker_name, worker_port + ) # Enable the worker in supervisord worker_descriptors.append(worker_config) From 92a1345f27ddc8d3477980cb2c364f0c7e3552ad Mon Sep 17 00:00:00 2001 From: Jason Little Date: Sun, 16 Oct 2022 04:35:00 -0500 Subject: [PATCH 05/21] Changelog --- changelog.d/14197.docker | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/14197.docker diff --git a/changelog.d/14197.docker b/changelog.d/14197.docker new file mode 100644 index 000000000000..529ccd99c501 --- /dev/null +++ b/changelog.d/14197.docker @@ -0,0 +1 @@ +Add all Stream Writer worker types to configure_workers_and_start.py. From 0c073d643420ee797f11b9d3cf93f95c424ee16f Mon Sep 17 00:00:00 2001 From: Jason Little Date: Thu, 20 Oct 2022 17:05:42 -0500 Subject: [PATCH 06/21] Rename add_sharding_to_shared_config to process_sharding_and_stream_writers_for_shared_config For the record, I think this function name is to long but seems to get the point across. --- docker/configure_workers_and_start.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index c8307ff41f61..28f8ab90f8a3 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -325,7 +325,7 @@ def convert(src: str, dst: str, **template_vars: object) -> None: outfile.write(rendered) -def add_sharding_to_shared_config( +def process_sharding_and_stream_writers_for_shared_config( shared_config: dict, worker_type: str, worker_name: str, @@ -511,7 +511,7 @@ def generate_worker_files( worker_type_total_count = worker_types.count(worker_type) # Update the shared config with sharding-related options if necessary - add_sharding_to_shared_config( + process_sharding_and_stream_writers_for_shared_config( shared_config, worker_type, worker_name, worker_port ) From 3a03ce725c39b5babaf24df272b3a1887b31a126 Mon Sep 17 00:00:00 2001 From: realtyem Date: Thu, 20 Oct 2022 21:41:57 -0500 Subject: [PATCH 07/21] Removed outdated comment This is no longer accurate --- docker/configure_workers_and_start.py | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 644f51ffcb70..4afc51829696 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -56,7 +56,6 @@ # Watching /_matrix/media and related needs a "media" listener # Stream Writers require "client" and "replication" listeners because they # have to attach by instance_map to the master process and have client endpoints. -# BIG WARNING: typing and receipts stream writers are not working correctly at this time. WORKERS_CONFIG: Dict[str, Dict[str, Any]] = { "pusher": { "app": "synapse.app.pusher", From 9393fb707e2f59fcd62bd56ac8c4a8bf191aaa0d Mon Sep 17 00:00:00 2001 From: Jason Little Date: Tue, 25 Oct 2022 18:40:19 -0500 Subject: [PATCH 08/21] Change all worker declarations except 'media_repository' to be 'synapse.app.generic_worker'. --- docker/configure_workers_and_start.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 4afc51829696..5b5deae8234a 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -58,7 +58,7 @@ # have to attach by instance_map to the master process and have client endpoints. WORKERS_CONFIG: Dict[str, Dict[str, Any]] = { "pusher": { - "app": "synapse.app.pusher", + "app": "synapse.app.generic_worker", "listener_resources": [], "endpoint_patterns": [], "shared_extra_conf": {"start_pushers": False}, @@ -95,7 +95,7 @@ "worker_extra_conf": "", }, "federation_sender": { - "app": "synapse.app.federation_sender", + "app": "synapse.app.generic_worker", "listener_resources": [], "endpoint_patterns": [], "shared_extra_conf": {"send_federation": False}, @@ -205,7 +205,7 @@ "worker_extra_conf": "", }, "frontend_proxy": { - "app": "synapse.app.frontend_proxy", + "app": "synapse.app.generic_worker", "listener_resources": ["client", "replication"], "endpoint_patterns": ["^/_matrix/client/(api/v1|r0|v3|unstable)/keys/upload"], "shared_extra_conf": {}, From ca1956d4386f5eb1345d7790e0773df846ca0079 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Tue, 25 Oct 2022 18:42:35 -0500 Subject: [PATCH 09/21] Remove 'start_pushers' and 'send_federation' from script, as they are now deprecated. --- docker/configure_workers_and_start.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 5b5deae8234a..c6b1bd310993 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -61,7 +61,7 @@ "app": "synapse.app.generic_worker", "listener_resources": [], "endpoint_patterns": [], - "shared_extra_conf": {"start_pushers": False}, + "shared_extra_conf": {}, "worker_extra_conf": "", }, "user_dir": { @@ -98,7 +98,7 @@ "app": "synapse.app.generic_worker", "listener_resources": [], "endpoint_patterns": [], - "shared_extra_conf": {"send_federation": False}, + "shared_extra_conf": {}, "worker_extra_conf": "", }, "synchrotron": { From ce01c104ec7c60c908efebc4cc102e45567180ed Mon Sep 17 00:00:00 2001 From: Jason Little Date: Tue, 25 Oct 2022 18:46:28 -0500 Subject: [PATCH 10/21] Remove comment that hasn't been true since Apr 14th, 2021. --- docker/configure_workers_and_start.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index c6b1bd310993..3701923e8983 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -20,7 +20,7 @@ # * SYNAPSE_SERVER_NAME: The desired server_name of the homeserver. # * SYNAPSE_REPORT_STATS: Whether to report stats. # * SYNAPSE_WORKER_TYPES: A comma separated list of worker names as specified in WORKER_CONFIG -# below. Leave empty for no workers, or set to '*' for all possible workers. +# below. Leave empty for no workers. # * SYNAPSE_AS_REGISTRATION_DIR: If specified, a directory in which .yaml and .yml files # will be treated as Application Service registration files. # * SYNAPSE_TLS_CERT: Path to a TLS certificate in PEM format. From 6079b66a40ed245cc62469950a67c66d3973b60e Mon Sep 17 00:00:00 2001 From: Jason Little Date: Tue, 25 Oct 2022 19:57:13 -0500 Subject: [PATCH 11/21] Rename function to summarize better it's function. --- docker/configure_workers_and_start.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 3701923e8983..f9377aea7525 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -319,7 +319,7 @@ def convert(src: str, dst: str, **template_vars: object) -> None: outfile.write(rendered) -def process_sharding_and_stream_writers_for_shared_config( +def handle_map_instances_for_for_shared_config( shared_config: dict, worker_type: str, worker_name: str, @@ -505,7 +505,7 @@ def generate_worker_files( worker_type_total_count = worker_types.count(worker_type) # Update the shared config with sharding-related options if necessary - process_sharding_and_stream_writers_for_shared_config( + handle_map_instances_for_shared_config( shared_config, worker_type, worker_name, worker_port ) From 72071518609b92e7ebf38989ac4e1aadea665213 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Tue, 25 Oct 2022 20:00:14 -0500 Subject: [PATCH 12/21] Changelog --- changelog.d/14294.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/14294.misc diff --git a/changelog.d/14294.misc b/changelog.d/14294.misc new file mode 100644 index 000000000000..304faa941b70 --- /dev/null +++ b/changelog.d/14294.misc @@ -0,0 +1 @@ +Update template in configure_workers_and_start.py to reflect settings requirements in source. From d23d9927d4c5b8d7421087cab806490004eaa166 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Tue, 25 Oct 2022 20:05:41 -0500 Subject: [PATCH 13/21] Fix rename derp. --- docker/configure_workers_and_start.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index f9377aea7525..957c5211e534 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -319,7 +319,7 @@ def convert(src: str, dst: str, **template_vars: object) -> None: outfile.write(rendered) -def handle_map_instances_for_for_shared_config( +def handle_map_instances_for_shared_config( shared_config: dict, worker_type: str, worker_name: str, From 8759d179cfb649acd093585b2383cfa59cf348dd Mon Sep 17 00:00:00 2001 From: Jason Little Date: Tue, 25 Oct 2022 20:09:03 -0500 Subject: [PATCH 14/21] Update some comments. --- docker/configure_workers_and_start.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 957c5211e534..8d1bee78de52 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -326,7 +326,7 @@ def handle_map_instances_for_shared_config( worker_port: int, ) -> None: """Given a dictionary representing a config file shared across all workers, - append sharded worker information to it for the current worker_type instance. + append appropriate worker information to it for the current worker_type instance. Args: shared_config: The config dict that all worker instances share (after being converted to YAML) @@ -359,7 +359,7 @@ def handle_map_instances_for_shared_config( elif worker_type in ["account_data", "presence", "receipts", "to_device", "typing"]: # Update the list of stream writers - # It's convienent that the name of the worker type is the same as the event stream + # It's convenient that the name of the worker type is the same as the stream to write shared_config.setdefault("stream_writers", {}).setdefault( worker_type, [] ).append(worker_name) From eced528f16f53d0a06738eefa75168aeed682c2e Mon Sep 17 00:00:00 2001 From: Jason Little Date: Wed, 26 Oct 2022 05:17:16 -0500 Subject: [PATCH 15/21] Adjust changelog and extension. --- changelog.d/14294.docker | 1 + changelog.d/14294.misc | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 changelog.d/14294.docker delete mode 100644 changelog.d/14294.misc diff --git a/changelog.d/14294.docker b/changelog.d/14294.docker new file mode 100644 index 000000000000..1489470408d2 --- /dev/null +++ b/changelog.d/14294.docker @@ -0,0 +1 @@ +Remove references to legacy worker types in the multi-worker Dockerfile. diff --git a/changelog.d/14294.misc b/changelog.d/14294.misc deleted file mode 100644 index 304faa941b70..000000000000 --- a/changelog.d/14294.misc +++ /dev/null @@ -1 +0,0 @@ -Update template in configure_workers_and_start.py to reflect settings requirements in source. From 7b7a928ca521a1ad34ca4c1f39098e9c22b0ed46 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Wed, 26 Oct 2022 05:21:36 -0500 Subject: [PATCH 16/21] Make media_repository shared_extra_conf consistent with other types, like background_worker. --- docker/configure_workers_and_start.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 8d1bee78de52..2d4452bf6656 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -84,7 +84,11 @@ "^/_synapse/admin/v1/media/.*$", "^/_synapse/admin/v1/quarantine_media/.*$", ], - "shared_extra_conf": {"enable_media_repo": False}, + # The first configured media worker will run the media background jobs + "shared_extra_conf": { + "enable_media_repo": False, + "media_instance_running_background_jobs", "media_repository1", + }, "worker_extra_conf": "enable_media_repo: true", }, "appservice": { @@ -371,10 +375,6 @@ def handle_map_instances_for_shared_config( "port": worker_port, } - elif worker_type == "media_repository": - # The first configured media worker will run the media background jobs - shared_config.setdefault("media_instance_running_background_jobs", worker_name) - def generate_base_homeserver_config() -> None: """Starts Synapse and generates a basic homeserver config, which will later be From 268674ff9511c485bff134e43f301a33a92d8280 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Wed, 26 Oct 2022 05:28:40 -0500 Subject: [PATCH 17/21] Need to fix linting after all. --- docker/configure_workers_and_start.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 2d4452bf6656..b5af74ebe105 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -87,7 +87,7 @@ # The first configured media worker will run the media background jobs "shared_extra_conf": { "enable_media_repo": False, - "media_instance_running_background_jobs", "media_repository1", + "media_instance_running_background_jobs": "media_repository1", }, "worker_extra_conf": "enable_media_repo: true", }, From 7da1e196a42e8365cd28ea0934592326cb442870 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Fri, 4 Nov 2022 15:02:15 -0500 Subject: [PATCH 18/21] Rename function to reivilibre's suggestion(This be a no-op when the previous PR is merged). --- docker/configure_workers_and_start.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index b5af74ebe105..fa8684548f08 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -323,7 +323,7 @@ def convert(src: str, dst: str, **template_vars: object) -> None: outfile.write(rendered) -def handle_map_instances_for_shared_config( +def add_worker_roles_to_shared_config( shared_config: dict, worker_type: str, worker_name: str, @@ -505,7 +505,7 @@ def generate_worker_files( worker_type_total_count = worker_types.count(worker_type) # Update the shared config with sharding-related options if necessary - handle_map_instances_for_shared_config( + add_worker_roles_to_shared_config( shared_config, worker_type, worker_name, worker_port ) From 99a206cc59bd27a0d559ad3da36e8a48dca521c6 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Tue, 8 Nov 2022 16:08:11 -0600 Subject: [PATCH 19/21] Because resolving conflicts on github doesn't preserve executable permissions. --- docker/configure_workers_and_start.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 docker/configure_workers_and_start.py diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py old mode 100644 new mode 100755 From aa045a6de702f8140f283434d06b5603af07be7e Mon Sep 17 00:00:00 2001 From: Jason Little Date: Wed, 9 Nov 2022 05:13:21 -0600 Subject: [PATCH 20/21] Almost forgot to make this a show-stopper. --- docker/configure_workers_and_start.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index fa8684548f08..78373155e020 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -483,7 +483,7 @@ def generate_worker_files( if worker_config: worker_config = worker_config.copy() else: - log(worker_type + " is an unknown worker type! It will be ignored") + error(worker_type + " is an unknown worker type! Please fix!") continue new_worker_count = worker_type_counter.setdefault(worker_type, 0) + 1 From 661ad1058973146120eacb68bcbb72110d61f933 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Wed, 9 Nov 2022 05:24:12 -0600 Subject: [PATCH 21/21] Remove unreachable 'continue' --- docker/configure_workers_and_start.py | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 78373155e020..62b1bab297be 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -484,7 +484,6 @@ def generate_worker_files( worker_config = worker_config.copy() else: error(worker_type + " is an unknown worker type! Please fix!") - continue new_worker_count = worker_type_counter.setdefault(worker_type, 0) + 1 worker_type_counter[worker_type] = new_worker_count