From 108bc6cc8f4eeef021902418ddb4471240bbec33 Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Wed, 24 Jul 2019 23:46:47 +1000 Subject: [PATCH 01/21] disable media repo if it's disabled and not an option --- synapse/config/repository.py | 13 +++++++++++++ synapse/rest/media/v1/media_repository.py | 9 ++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/synapse/config/repository.py b/synapse/config/repository.py index 80a628d9b0cb..f64133d18c3a 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -87,6 +87,11 @@ def parse_thumbnail_requirements(thumbnail_sizes): class ContentRepositoryConfig(Config): def read_config(self, config, **kwargs): + # If the media repo is disabled and this is not a worker, do not load + # the further configuration (which checks if files exists). + if self.worker_app is None and not self.enable_media_repo: + return + self.max_upload_size = self.parse_size(config.get("max_upload_size", "10M")) self.max_image_pixels = self.parse_size(config.get("max_image_pixels", "32M")) self.max_spider_size = self.parse_size(config.get("max_spider_size", "10M")) @@ -202,6 +207,14 @@ def generate_config_section(self, data_dir_path, **kwargs): return ( r""" + ## + ## Media Store + ## + + # Enable the media store service inside Synapse. + # + enable_media_store: True + # Directory where uploaded images and attachments are stored. # media_store_path: "%(media_store)s" diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py index 92beefa1766c..a4834e00a177 100644 --- a/synapse/rest/media/v1/media_repository.py +++ b/synapse/rest/media/v1/media_repository.py @@ -753,8 +753,15 @@ class MediaRepositoryResource(Resource): """ def __init__(self, hs): - Resource.__init__(self) + # If this is not the media repo (which will always load this class), + # raise an exception if it is tried to be used. + if ( + not hs.config.enable_media_repo + and not config.worker_app == "synapse.app.media_repository" + ): + raise Exception("This Synapse is not configured for media repo use.") + super().__init__(self) media_repo = hs.get_media_repository() self.putChild(b"upload", UploadResource(hs, media_repo)) From 997fd94e0f6836eeae50b6d087efe6dfd47ec23f Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Wed, 24 Jul 2019 23:48:00 +1000 Subject: [PATCH 02/21] changelog --- changelog.d/5754.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/5754.misc diff --git a/changelog.d/5754.misc b/changelog.d/5754.misc new file mode 100644 index 000000000000..480bef5a5460 --- /dev/null +++ b/changelog.d/5754.misc @@ -0,0 +1 @@ +Do not load the media repository configuration if the media repository is disabled. From bbffae3c0d4d7ed119b99264726424482c9d0824 Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Wed, 24 Jul 2019 23:56:24 +1000 Subject: [PATCH 03/21] config sync --- docs/sample_config.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 7edf15207afc..4d1301a2a3ae 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -548,6 +548,14 @@ log_config: "CONFDIR/SERVERNAME.log.config" +## +## Media Store +## + +# Enable the media store service inside Synapse. +# +enable_media_store: True + # Directory where uploaded images and attachments are stored. # media_store_path: "DATADIR/media_store" From 4f2caf19c30ccda91a64003d238cb4322dbf467b Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Wed, 24 Jul 2019 23:57:06 +1000 Subject: [PATCH 04/21] config sync --- synapse/rest/media/v1/media_repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py index a4834e00a177..c8fb13e6eac9 100644 --- a/synapse/rest/media/v1/media_repository.py +++ b/synapse/rest/media/v1/media_repository.py @@ -757,7 +757,7 @@ def __init__(self, hs): # raise an exception if it is tried to be used. if ( not hs.config.enable_media_repo - and not config.worker_app == "synapse.app.media_repository" + and not hs.config.worker_app == "synapse.app.media_repository" ): raise Exception("This Synapse is not configured for media repo use.") From c463960b209b1ae9feed02a5d739e0e1e116c681 Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Thu, 25 Jul 2019 00:04:16 +1000 Subject: [PATCH 05/21] config sync --- synapse/config/homeserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index 72acad4f1895..daca27820fa9 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -52,7 +52,6 @@ class HomeServerConfig( DatabaseConfig, LoggingConfig, RatelimitConfig, - ContentRepositoryConfig, CaptchaConfig, VoipConfig, RegistrationConfig, @@ -66,6 +65,7 @@ class HomeServerConfig( PasswordConfig, EmailConfig, WorkerConfig, + ContentRepositoryConfig, PasswordAuthProviderConfig, PushConfig, SpamCheckerConfig, From 081e102c6c266deddd98cac45237606f8e8bb4bb Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Thu, 25 Jul 2019 00:15:01 +1000 Subject: [PATCH 06/21] reorder config too --- docs/sample_config.yaml | 294 ++++++++++++++++++++-------------------- 1 file changed, 147 insertions(+), 147 deletions(-) diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 4d1301a2a3ae..e83bd496ff90 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -547,153 +547,6 @@ log_config: "CONFDIR/SERVERNAME.log.config" #federation_rr_transactions_per_room_per_second: 50 - -## -## Media Store -## - -# Enable the media store service inside Synapse. -# -enable_media_store: True - -# Directory where uploaded images and attachments are stored. -# -media_store_path: "DATADIR/media_store" - -# Media storage providers allow media to be stored in different -# locations. -# -#media_storage_providers: -# - module: file_system -# # Whether to write new local files. -# store_local: false -# # Whether to write new remote media -# store_remote: false -# # Whether to block upload requests waiting for write to this -# # provider to complete -# store_synchronous: false -# config: -# directory: /mnt/some/other/directory - -# Directory where in-progress uploads are stored. -# -uploads_path: "DATADIR/uploads" - -# The largest allowed upload size in bytes -# -#max_upload_size: 10M - -# Maximum number of pixels that will be thumbnailed -# -#max_image_pixels: 32M - -# Whether to generate new thumbnails on the fly to precisely match -# the resolution requested by the client. If true then whenever -# a new resolution is requested by the client the server will -# generate a new thumbnail. If false the server will pick a thumbnail -# from a precalculated list. -# -#dynamic_thumbnails: false - -# List of thumbnails to precalculate when an image is uploaded. -# -#thumbnail_sizes: -# - width: 32 -# height: 32 -# method: crop -# - width: 96 -# height: 96 -# method: crop -# - width: 320 -# height: 240 -# method: scale -# - width: 640 -# height: 480 -# method: scale -# - width: 800 -# height: 600 -# method: scale - -# Is the preview URL API enabled? -# -# 'false' by default: uncomment the following to enable it (and specify a -# url_preview_ip_range_blacklist blacklist). -# -#url_preview_enabled: true - -# List of IP address CIDR ranges that the URL preview spider is denied -# from accessing. There are no defaults: you must explicitly -# specify a list for URL previewing to work. You should specify any -# internal services in your network that you do not want synapse to try -# to connect to, otherwise anyone in any Matrix room could cause your -# synapse to issue arbitrary GET requests to your internal services, -# causing serious security issues. -# -# (0.0.0.0 and :: are always blacklisted, whether or not they are explicitly -# listed here, since they correspond to unroutable addresses.) -# -# This must be specified if url_preview_enabled is set. It is recommended that -# you uncomment the following list as a starting point. -# -#url_preview_ip_range_blacklist: -# - '127.0.0.0/8' -# - '10.0.0.0/8' -# - '172.16.0.0/12' -# - '192.168.0.0/16' -# - '100.64.0.0/10' -# - '169.254.0.0/16' -# - '::1/128' -# - 'fe80::/64' -# - 'fc00::/7' - -# List of IP address CIDR ranges that the URL preview spider is allowed -# to access even if they are specified in url_preview_ip_range_blacklist. -# This is useful for specifying exceptions to wide-ranging blacklisted -# target IP ranges - e.g. for enabling URL previews for a specific private -# website only visible in your network. -# -#url_preview_ip_range_whitelist: -# - '192.168.1.1' - -# Optional list of URL matches that the URL preview spider is -# denied from accessing. You should use url_preview_ip_range_blacklist -# in preference to this, otherwise someone could define a public DNS -# entry that points to a private IP address and circumvent the blacklist. -# This is more useful if you know there is an entire shape of URL that -# you know that will never want synapse to try to spider. -# -# Each list entry is a dictionary of url component attributes as returned -# by urlparse.urlsplit as applied to the absolute form of the URL. See -# https://docs.python.org/2/library/urlparse.html#urlparse.urlsplit -# The values of the dictionary are treated as an filename match pattern -# applied to that component of URLs, unless they start with a ^ in which -# case they are treated as a regular expression match. If all the -# specified component matches for a given list item succeed, the URL is -# blacklisted. -# -#url_preview_url_blacklist: -# # blacklist any URL with a username in its URI -# - username: '*' -# -# # blacklist all *.google.com URLs -# - netloc: 'google.com' -# - netloc: '*.google.com' -# -# # blacklist all plain HTTP URLs -# - scheme: 'http' -# -# # blacklist http(s)://www.acme.com/foo -# - netloc: 'www.acme.com' -# path: '/foo' -# -# # blacklist any URL with a literal IPv4 address -# - netloc: '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' - -# The largest allowed URL preview spidering size in bytes -# -#max_spider_size: 10M - - ## Captcha ## # See docs/CAPTCHA_SETUP for full details of configuring this. @@ -1184,6 +1037,153 @@ password_config: # #password_reset_template_failure_html: password_reset_failure.html + +## +## Media Store +## + +# Enable the media store service inside Synapse. +# +enable_media_store: True + +# Directory where uploaded images and attachments are stored. +# +media_store_path: "DATADIR/media_store" + +# Media storage providers allow media to be stored in different +# locations. +# +#media_storage_providers: +# - module: file_system +# # Whether to write new local files. +# store_local: false +# # Whether to write new remote media +# store_remote: false +# # Whether to block upload requests waiting for write to this +# # provider to complete +# store_synchronous: false +# config: +# directory: /mnt/some/other/directory + +# Directory where in-progress uploads are stored. +# +uploads_path: "DATADIR/uploads" + +# The largest allowed upload size in bytes +# +#max_upload_size: 10M + +# Maximum number of pixels that will be thumbnailed +# +#max_image_pixels: 32M + +# Whether to generate new thumbnails on the fly to precisely match +# the resolution requested by the client. If true then whenever +# a new resolution is requested by the client the server will +# generate a new thumbnail. If false the server will pick a thumbnail +# from a precalculated list. +# +#dynamic_thumbnails: false + +# List of thumbnails to precalculate when an image is uploaded. +# +#thumbnail_sizes: +# - width: 32 +# height: 32 +# method: crop +# - width: 96 +# height: 96 +# method: crop +# - width: 320 +# height: 240 +# method: scale +# - width: 640 +# height: 480 +# method: scale +# - width: 800 +# height: 600 +# method: scale + +# Is the preview URL API enabled? +# +# 'false' by default: uncomment the following to enable it (and specify a +# url_preview_ip_range_blacklist blacklist). +# +#url_preview_enabled: true + +# List of IP address CIDR ranges that the URL preview spider is denied +# from accessing. There are no defaults: you must explicitly +# specify a list for URL previewing to work. You should specify any +# internal services in your network that you do not want synapse to try +# to connect to, otherwise anyone in any Matrix room could cause your +# synapse to issue arbitrary GET requests to your internal services, +# causing serious security issues. +# +# (0.0.0.0 and :: are always blacklisted, whether or not they are explicitly +# listed here, since they correspond to unroutable addresses.) +# +# This must be specified if url_preview_enabled is set. It is recommended that +# you uncomment the following list as a starting point. +# +#url_preview_ip_range_blacklist: +# - '127.0.0.0/8' +# - '10.0.0.0/8' +# - '172.16.0.0/12' +# - '192.168.0.0/16' +# - '100.64.0.0/10' +# - '169.254.0.0/16' +# - '::1/128' +# - 'fe80::/64' +# - 'fc00::/7' + +# List of IP address CIDR ranges that the URL preview spider is allowed +# to access even if they are specified in url_preview_ip_range_blacklist. +# This is useful for specifying exceptions to wide-ranging blacklisted +# target IP ranges - e.g. for enabling URL previews for a specific private +# website only visible in your network. +# +#url_preview_ip_range_whitelist: +# - '192.168.1.1' + +# Optional list of URL matches that the URL preview spider is +# denied from accessing. You should use url_preview_ip_range_blacklist +# in preference to this, otherwise someone could define a public DNS +# entry that points to a private IP address and circumvent the blacklist. +# This is more useful if you know there is an entire shape of URL that +# you know that will never want synapse to try to spider. +# +# Each list entry is a dictionary of url component attributes as returned +# by urlparse.urlsplit as applied to the absolute form of the URL. See +# https://docs.python.org/2/library/urlparse.html#urlparse.urlsplit +# The values of the dictionary are treated as an filename match pattern +# applied to that component of URLs, unless they start with a ^ in which +# case they are treated as a regular expression match. If all the +# specified component matches for a given list item succeed, the URL is +# blacklisted. +# +#url_preview_url_blacklist: +# # blacklist any URL with a username in its URI +# - username: '*' +# +# # blacklist all *.google.com URLs +# - netloc: 'google.com' +# - netloc: '*.google.com' +# +# # blacklist all plain HTTP URLs +# - scheme: 'http' +# +# # blacklist http(s)://www.acme.com/foo +# - netloc: 'www.acme.com' +# path: '/foo' +# +# # blacklist any URL with a literal IPv4 address +# - netloc: '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' + +# The largest allowed URL preview spidering size in bytes +# +#max_spider_size: 10M + + #password_providers: # - module: "ldap_auth_provider.LdapAuthProvider" # config: From cd70e82765970fbaf0a59f0554d31e9878ecf6c0 Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Thu, 25 Jul 2019 00:21:19 +1000 Subject: [PATCH 07/21] reorder config too --- synapse/rest/media/v1/media_repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py index c8fb13e6eac9..6b8a76f0e880 100644 --- a/synapse/rest/media/v1/media_repository.py +++ b/synapse/rest/media/v1/media_repository.py @@ -761,7 +761,7 @@ def __init__(self, hs): ): raise Exception("This Synapse is not configured for media repo use.") - super().__init__(self) + super().__init__() media_repo = hs.get_media_repository() self.putChild(b"upload", UploadResource(hs, media_repo)) From 3ff9935cb6e7d2c72b560f43840e141f102c36d0 Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Thu, 25 Jul 2019 03:49:48 +1000 Subject: [PATCH 08/21] fix --- synapse/config/repository.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/config/repository.py b/synapse/config/repository.py index f64133d18c3a..46a96e894a96 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + import os from collections import namedtuple From 85f5cbd6a0df3017c8e500657ee1990a756fbc94 Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Mon, 29 Jul 2019 20:46:03 +1000 Subject: [PATCH 09/21] fix --- docs/sample_config.yaml | 2 +- synapse/config/repository.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index e83bd496ff90..8a290fc20335 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -1044,7 +1044,7 @@ password_config: # Enable the media store service inside Synapse. # -enable_media_store: True +enable_media_repo: True # Directory where uploaded images and attachments are stored. # diff --git a/synapse/config/repository.py b/synapse/config/repository.py index 46a96e894a96..334b870e99e9 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -214,7 +214,7 @@ def generate_config_section(self, data_dir_path, **kwargs): # Enable the media store service inside Synapse. # - enable_media_store: True + enable_media_repo: True # Directory where uploaded images and attachments are stored. # From f528371ac7e8961d8f7e495839815694cb87d2b2 Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Mon, 29 Jul 2019 21:47:56 +1000 Subject: [PATCH 10/21] fix worker mode just exploding --- docs/sample_config.yaml | 7 +++++++ synapse/app/homeserver.py | 17 ++++++++++++----- synapse/app/media_repository.py | 7 +++++++ synapse/config/repository.py | 20 +++++++++++++++++--- synapse/rest/admin/__init__.py | 10 +++++++--- synapse/rest/media/v1/media_repository.py | 5 +---- 6 files changed, 51 insertions(+), 15 deletions(-) diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 8a290fc20335..48a30fe999b3 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -1042,6 +1042,13 @@ password_config: ## Media Store ## +# Is an external media repo in use? +# +# Set 'False' to use Synapse's media repo. If this is set to 'True', +# Synapse's media repo below must be disabled. +# +#external_media_repo: False + # Enable the media store service inside Synapse. # enable_media_repo: True diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 7d6b51b5bc09..a89f2c5a1447 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -219,7 +219,18 @@ def _configure_named_resource(self, name, compress=False): ) if name in ["media", "federation", "client"]: - if self.get_config().enable_media_repo: + if name == "media" and not self.get_config().enable_media_repo: + raise ConfigError( + "'media' resource conflicts with enable_media_repo=False" + ) + elif name == "media" and self.get_config().external_media_repo: + raise ConfigError( + "'media' resource conflicts with external_media_repo=True" + ) + elif ( + self.get_config().enable_media_repo + and not self.get_config().external_media_repo + ): media_repo = self.get_media_repository_resource() resources.update( { @@ -230,10 +241,6 @@ def _configure_named_resource(self, name, compress=False): ), } ) - elif name == "media": - raise ConfigError( - "'media' resource conflicts with enable_media_repo=False" - ) if name in ["keys", "federation"]: resources[SERVER_KEY_V2_PREFIX] = KeyApiV2Resource(self) diff --git a/synapse/app/media_repository.py b/synapse/app/media_repository.py index ea26f29acb88..b7bb35a21c9d 100644 --- a/synapse/app/media_repository.py +++ b/synapse/app/media_repository.py @@ -139,6 +139,13 @@ def start(config_options): assert config.worker_app == "synapse.app.media_repository" + if config.external_media_repo: + _base.quit_with_error( + "external_media_repo must be disabled before the \n" + "media_repository worker can be used\n" + "Please add ``external_media_repo: false`` to the main config\n" + ) + if config.enable_media_repo: _base.quit_with_error( "enable_media_repo must be disabled in the main synapse process\n" diff --git a/synapse/config/repository.py b/synapse/config/repository.py index 334b870e99e9..78915894ce90 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -88,9 +88,16 @@ def parse_thumbnail_requirements(thumbnail_sizes): class ContentRepositoryConfig(Config): def read_config(self, config, **kwargs): - # If the media repo is disabled and this is not a worker, do not load - # the further configuration (which checks if files exists). - if self.worker_app is None and not self.enable_media_repo: + self.external_media_repo = config.get("external_media_repo", False) + + if self.external_media_repo and self.enable_media_repo: + raise ConfigError( + "external_media_repo can't be True if the enable_media_repo is True" + ) + + # If we're using an external media repo, do not load the further + # configuration (which checks if files exists). + if self.external_media_repo: return self.max_upload_size = self.parse_size(config.get("max_upload_size", "10M")) @@ -212,6 +219,13 @@ def generate_config_section(self, data_dir_path, **kwargs): ## Media Store ## + # Is an external media repo in use? + # + # Set 'False' to use Synapse's media repo. If this is set to 'True', + # Synapse's media repo below must be disabled. + # + #external_media_repo: False + # Enable the media store service inside Synapse. # enable_media_repo: True diff --git a/synapse/rest/admin/__init__.py b/synapse/rest/admin/__init__.py index 0a7d9b81b27b..197677ca3b22 100644 --- a/synapse/rest/admin/__init__.py +++ b/synapse/rest/admin/__init__.py @@ -825,7 +825,6 @@ def register_servlets(hs, http_server): def register_servlets_for_client_rest_resource(hs, http_server): """Register only the servlets which need to be exposed on /_matrix/client/xxx""" WhoisRestServlet(hs).register(http_server) - PurgeMediaCacheRestServlet(hs).register(http_server) PurgeHistoryStatusRestServlet(hs).register(http_server) DeactivateAccountRestServlet(hs).register(http_server) PurgeHistoryRestServlet(hs).register(http_server) @@ -834,10 +833,15 @@ def register_servlets_for_client_rest_resource(hs, http_server): GetUsersPaginatedRestServlet(hs).register(http_server) SearchUsersRestServlet(hs).register(http_server) ShutdownRoomRestServlet(hs).register(http_server) - QuarantineMediaInRoom(hs).register(http_server) - ListMediaInRoom(hs).register(http_server) UserRegisterServlet(hs).register(http_server) DeleteGroupAdminRestServlet(hs).register(http_server) AccountValidityRenewServlet(hs).register(http_server) + + # Media URLs -- if we're using an external media repo, don't add them. + if not hs.config.external_media_repo: + PurgeMediaCacheRestServlet(hs).register(http_server) + QuarantineMediaInRoom(hs).register(http_server) + ListMediaInRoom(hs).register(http_server) + # don't add more things here: new servlets should only be exposed on # /_synapse/admin so should not go here. Instead register them in AdminRestResource. diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py index 6b8a76f0e880..5135bd4930dc 100644 --- a/synapse/rest/media/v1/media_repository.py +++ b/synapse/rest/media/v1/media_repository.py @@ -755,10 +755,7 @@ class MediaRepositoryResource(Resource): def __init__(self, hs): # If this is not the media repo (which will always load this class), # raise an exception if it is tried to be used. - if ( - not hs.config.enable_media_repo - and not hs.config.worker_app == "synapse.app.media_repository" - ): + if hs.config.external_media_repo: raise Exception("This Synapse is not configured for media repo use.") super().__init__() From c7561c25c1b218cd03593dc3bd8ce5145efc5a92 Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Mon, 29 Jul 2019 22:51:10 +1000 Subject: [PATCH 11/21] unmove this about --- docs/sample_config.yaml | 308 +++++++++++++++++------------------ synapse/config/homeserver.py | 2 +- 2 files changed, 155 insertions(+), 155 deletions(-) diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 48a30fe999b3..65be2306c727 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -547,6 +547,160 @@ log_config: "CONFDIR/SERVERNAME.log.config" #federation_rr_transactions_per_room_per_second: 50 + +## +## Media Store +## + +# Is an external media repo in use? +# +# Set 'False' to use Synapse's media repo. If this is set to 'True', +# Synapse's media repo below must be disabled. +# +#external_media_repo: False + +# Enable the media store service inside Synapse. +# +enable_media_repo: True + +# Directory where uploaded images and attachments are stored. +# +media_store_path: "DATADIR/media_store" + +# Media storage providers allow media to be stored in different +# locations. +# +#media_storage_providers: +# - module: file_system +# # Whether to write new local files. +# store_local: false +# # Whether to write new remote media +# store_remote: false +# # Whether to block upload requests waiting for write to this +# # provider to complete +# store_synchronous: false +# config: +# directory: /mnt/some/other/directory + +# Directory where in-progress uploads are stored. +# +uploads_path: "DATADIR/uploads" + +# The largest allowed upload size in bytes +# +#max_upload_size: 10M + +# Maximum number of pixels that will be thumbnailed +# +#max_image_pixels: 32M + +# Whether to generate new thumbnails on the fly to precisely match +# the resolution requested by the client. If true then whenever +# a new resolution is requested by the client the server will +# generate a new thumbnail. If false the server will pick a thumbnail +# from a precalculated list. +# +#dynamic_thumbnails: false + +# List of thumbnails to precalculate when an image is uploaded. +# +#thumbnail_sizes: +# - width: 32 +# height: 32 +# method: crop +# - width: 96 +# height: 96 +# method: crop +# - width: 320 +# height: 240 +# method: scale +# - width: 640 +# height: 480 +# method: scale +# - width: 800 +# height: 600 +# method: scale + +# Is the preview URL API enabled? +# +# 'false' by default: uncomment the following to enable it (and specify a +# url_preview_ip_range_blacklist blacklist). +# +#url_preview_enabled: true + +# List of IP address CIDR ranges that the URL preview spider is denied +# from accessing. There are no defaults: you must explicitly +# specify a list for URL previewing to work. You should specify any +# internal services in your network that you do not want synapse to try +# to connect to, otherwise anyone in any Matrix room could cause your +# synapse to issue arbitrary GET requests to your internal services, +# causing serious security issues. +# +# (0.0.0.0 and :: are always blacklisted, whether or not they are explicitly +# listed here, since they correspond to unroutable addresses.) +# +# This must be specified if url_preview_enabled is set. It is recommended that +# you uncomment the following list as a starting point. +# +#url_preview_ip_range_blacklist: +# - '127.0.0.0/8' +# - '10.0.0.0/8' +# - '172.16.0.0/12' +# - '192.168.0.0/16' +# - '100.64.0.0/10' +# - '169.254.0.0/16' +# - '::1/128' +# - 'fe80::/64' +# - 'fc00::/7' + +# List of IP address CIDR ranges that the URL preview spider is allowed +# to access even if they are specified in url_preview_ip_range_blacklist. +# This is useful for specifying exceptions to wide-ranging blacklisted +# target IP ranges - e.g. for enabling URL previews for a specific private +# website only visible in your network. +# +#url_preview_ip_range_whitelist: +# - '192.168.1.1' + +# Optional list of URL matches that the URL preview spider is +# denied from accessing. You should use url_preview_ip_range_blacklist +# in preference to this, otherwise someone could define a public DNS +# entry that points to a private IP address and circumvent the blacklist. +# This is more useful if you know there is an entire shape of URL that +# you know that will never want synapse to try to spider. +# +# Each list entry is a dictionary of url component attributes as returned +# by urlparse.urlsplit as applied to the absolute form of the URL. See +# https://docs.python.org/2/library/urlparse.html#urlparse.urlsplit +# The values of the dictionary are treated as an filename match pattern +# applied to that component of URLs, unless they start with a ^ in which +# case they are treated as a regular expression match. If all the +# specified component matches for a given list item succeed, the URL is +# blacklisted. +# +#url_preview_url_blacklist: +# # blacklist any URL with a username in its URI +# - username: '*' +# +# # blacklist all *.google.com URLs +# - netloc: 'google.com' +# - netloc: '*.google.com' +# +# # blacklist all plain HTTP URLs +# - scheme: 'http' +# +# # blacklist http(s)://www.acme.com/foo +# - netloc: 'www.acme.com' +# path: '/foo' +# +# # blacklist any URL with a literal IPv4 address +# - netloc: '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' + +# The largest allowed URL preview spidering size in bytes +# +#max_spider_size: 10M + + ## Captcha ## # See docs/CAPTCHA_SETUP for full details of configuring this. @@ -1037,160 +1191,6 @@ password_config: # #password_reset_template_failure_html: password_reset_failure.html - -## -## Media Store -## - -# Is an external media repo in use? -# -# Set 'False' to use Synapse's media repo. If this is set to 'True', -# Synapse's media repo below must be disabled. -# -#external_media_repo: False - -# Enable the media store service inside Synapse. -# -enable_media_repo: True - -# Directory where uploaded images and attachments are stored. -# -media_store_path: "DATADIR/media_store" - -# Media storage providers allow media to be stored in different -# locations. -# -#media_storage_providers: -# - module: file_system -# # Whether to write new local files. -# store_local: false -# # Whether to write new remote media -# store_remote: false -# # Whether to block upload requests waiting for write to this -# # provider to complete -# store_synchronous: false -# config: -# directory: /mnt/some/other/directory - -# Directory where in-progress uploads are stored. -# -uploads_path: "DATADIR/uploads" - -# The largest allowed upload size in bytes -# -#max_upload_size: 10M - -# Maximum number of pixels that will be thumbnailed -# -#max_image_pixels: 32M - -# Whether to generate new thumbnails on the fly to precisely match -# the resolution requested by the client. If true then whenever -# a new resolution is requested by the client the server will -# generate a new thumbnail. If false the server will pick a thumbnail -# from a precalculated list. -# -#dynamic_thumbnails: false - -# List of thumbnails to precalculate when an image is uploaded. -# -#thumbnail_sizes: -# - width: 32 -# height: 32 -# method: crop -# - width: 96 -# height: 96 -# method: crop -# - width: 320 -# height: 240 -# method: scale -# - width: 640 -# height: 480 -# method: scale -# - width: 800 -# height: 600 -# method: scale - -# Is the preview URL API enabled? -# -# 'false' by default: uncomment the following to enable it (and specify a -# url_preview_ip_range_blacklist blacklist). -# -#url_preview_enabled: true - -# List of IP address CIDR ranges that the URL preview spider is denied -# from accessing. There are no defaults: you must explicitly -# specify a list for URL previewing to work. You should specify any -# internal services in your network that you do not want synapse to try -# to connect to, otherwise anyone in any Matrix room could cause your -# synapse to issue arbitrary GET requests to your internal services, -# causing serious security issues. -# -# (0.0.0.0 and :: are always blacklisted, whether or not they are explicitly -# listed here, since they correspond to unroutable addresses.) -# -# This must be specified if url_preview_enabled is set. It is recommended that -# you uncomment the following list as a starting point. -# -#url_preview_ip_range_blacklist: -# - '127.0.0.0/8' -# - '10.0.0.0/8' -# - '172.16.0.0/12' -# - '192.168.0.0/16' -# - '100.64.0.0/10' -# - '169.254.0.0/16' -# - '::1/128' -# - 'fe80::/64' -# - 'fc00::/7' - -# List of IP address CIDR ranges that the URL preview spider is allowed -# to access even if they are specified in url_preview_ip_range_blacklist. -# This is useful for specifying exceptions to wide-ranging blacklisted -# target IP ranges - e.g. for enabling URL previews for a specific private -# website only visible in your network. -# -#url_preview_ip_range_whitelist: -# - '192.168.1.1' - -# Optional list of URL matches that the URL preview spider is -# denied from accessing. You should use url_preview_ip_range_blacklist -# in preference to this, otherwise someone could define a public DNS -# entry that points to a private IP address and circumvent the blacklist. -# This is more useful if you know there is an entire shape of URL that -# you know that will never want synapse to try to spider. -# -# Each list entry is a dictionary of url component attributes as returned -# by urlparse.urlsplit as applied to the absolute form of the URL. See -# https://docs.python.org/2/library/urlparse.html#urlparse.urlsplit -# The values of the dictionary are treated as an filename match pattern -# applied to that component of URLs, unless they start with a ^ in which -# case they are treated as a regular expression match. If all the -# specified component matches for a given list item succeed, the URL is -# blacklisted. -# -#url_preview_url_blacklist: -# # blacklist any URL with a username in its URI -# - username: '*' -# -# # blacklist all *.google.com URLs -# - netloc: 'google.com' -# - netloc: '*.google.com' -# -# # blacklist all plain HTTP URLs -# - scheme: 'http' -# -# # blacklist http(s)://www.acme.com/foo -# - netloc: 'www.acme.com' -# path: '/foo' -# -# # blacklist any URL with a literal IPv4 address -# - netloc: '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' - -# The largest allowed URL preview spidering size in bytes -# -#max_spider_size: 10M - - #password_providers: # - module: "ldap_auth_provider.LdapAuthProvider" # config: diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index daca27820fa9..72acad4f1895 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -52,6 +52,7 @@ class HomeServerConfig( DatabaseConfig, LoggingConfig, RatelimitConfig, + ContentRepositoryConfig, CaptchaConfig, VoipConfig, RegistrationConfig, @@ -65,7 +66,6 @@ class HomeServerConfig( PasswordConfig, EmailConfig, WorkerConfig, - ContentRepositoryConfig, PasswordAuthProviderConfig, PushConfig, SpamCheckerConfig, From f695e0bbc7a654512b215d34578a89b3ba9b6a05 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 29 Jul 2019 22:54:51 +1000 Subject: [PATCH 12/21] Update and rename 5754.misc to 5754.feature --- changelog.d/5754.feature | 1 + changelog.d/5754.misc | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 changelog.d/5754.feature delete mode 100644 changelog.d/5754.misc diff --git a/changelog.d/5754.feature b/changelog.d/5754.feature new file mode 100644 index 000000000000..b49a8e59a223 --- /dev/null +++ b/changelog.d/5754.feature @@ -0,0 +1 @@ +Add the `external_media_repo` configuration option. This prevents Synapse from checking or loading media repo specific configuration and from loading in the associated servlets which are not required when a third party media repo is used. diff --git a/changelog.d/5754.misc b/changelog.d/5754.misc deleted file mode 100644 index 480bef5a5460..000000000000 --- a/changelog.d/5754.misc +++ /dev/null @@ -1 +0,0 @@ -Do not load the media repository configuration if the media repository is disabled. From fe5d3373f13ea3a3562d51a08e1cb5a793ea9c80 Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Fri, 9 Aug 2019 22:43:32 +1000 Subject: [PATCH 13/21] fix --- changelog.d/5754.feature | 2 +- docs/sample_config.yaml | 7 -- docs/workers.rst | 7 ++ synapse/app/homeserver.py | 9 +- synapse/app/media_repository.py | 15 +-- synapse/config/repository.py | 25 ++--- synapse/rest/admin/__init__.py | 115 +++++----------------- synapse/rest/admin/_base.py | 25 +++++ synapse/rest/admin/media.py | 92 +++++++++++++++++ synapse/rest/media/v1/media_repository.py | 8 +- 10 files changed, 170 insertions(+), 135 deletions(-) create mode 100644 synapse/rest/admin/media.py diff --git a/changelog.d/5754.feature b/changelog.d/5754.feature index b49a8e59a223..9a019039df76 100644 --- a/changelog.d/5754.feature +++ b/changelog.d/5754.feature @@ -1 +1 @@ -Add the `external_media_repo` configuration option. This prevents Synapse from checking or loading media repo specific configuration and from loading in the associated servlets which are not required when a third party media repo is used. +Synapse will no longer serve any media repo admin endpoints when `enable_media_repo` is set to False in the configuration. \ No newline at end of file diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 4770eac85c6a..4997f465c2f9 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -569,13 +569,6 @@ log_config: "CONFDIR/SERVERNAME.log.config" ## Media Store ## -# Is an external media repo in use? -# -# Set 'False' to use Synapse's media repo. If this is set to 'True', -# Synapse's media repo below must be disabled. -# -#external_media_repo: False - # Enable the media store service inside Synapse. # enable_media_repo: True diff --git a/docs/workers.rst b/docs/workers.rst index 7b2d2db53327..e11e11741820 100644 --- a/docs/workers.rst +++ b/docs/workers.rst @@ -206,6 +206,13 @@ Handles the media repository. It can handle all endpoints starting with:: /_matrix/media/ +And the following regular expressions matching media-specific administration +APIs:: + + ^/_synapse/admin/v1/purge_media_cache$ + ^/_synapse/admin/v1/room/.*/media$ + ^/_synapse/admin/v1/quarantine_media/.*$ + You should also set ``enable_media_repo: False`` in the shared configuration file to stop the main synapse running background jobs related to managing the media repository. diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index a89f2c5a1447..8279334185c3 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -223,14 +223,7 @@ def _configure_named_resource(self, name, compress=False): raise ConfigError( "'media' resource conflicts with enable_media_repo=False" ) - elif name == "media" and self.get_config().external_media_repo: - raise ConfigError( - "'media' resource conflicts with external_media_repo=True" - ) - elif ( - self.get_config().enable_media_repo - and not self.get_config().external_media_repo - ): + elif self.get_config().enable_media_repo: media_repo = self.get_media_repository_resource() resources.update( { diff --git a/synapse/app/media_repository.py b/synapse/app/media_repository.py index b7bb35a21c9d..8e039a148f91 100644 --- a/synapse/app/media_repository.py +++ b/synapse/app/media_repository.py @@ -35,6 +35,7 @@ from synapse.replication.slave.storage.registration import SlavedRegistrationStore from synapse.replication.slave.storage.transactions import SlavedTransactionStore from synapse.replication.tcp.client import ReplicationClientHandler +from synapse.rest.admin import register_servlets_for_media_repo from synapse.rest.media.v0.content_repository import ContentRepoResource from synapse.server import HomeServer from synapse.storage.engines import create_engine @@ -71,6 +72,12 @@ def _listen_http(self, listener_config): resources[METRICS_PREFIX] = MetricsResource(RegistryProxy) elif name == "media": media_repo = self.get_media_repository_resource() + + # We need to serve the admin servlets for media on the + # worker. + admin_resource = JsonResource(self, canonical_json=False) + register_servlets_for_media_repo(self, admin_resource) + resources.update( { MEDIA_PREFIX: media_repo, @@ -78,6 +85,7 @@ def _listen_http(self, listener_config): CONTENT_REPO_PREFIX: ContentRepoResource( self, self.config.uploads_path ), + "/_synapse/admin": admin_resource, } ) @@ -139,13 +147,6 @@ def start(config_options): assert config.worker_app == "synapse.app.media_repository" - if config.external_media_repo: - _base.quit_with_error( - "external_media_repo must be disabled before the \n" - "media_repository worker can be used\n" - "Please add ``external_media_repo: false`` to the main config\n" - ) - if config.enable_media_repo: _base.quit_with_error( "enable_media_repo must be disabled in the main synapse process\n" diff --git a/synapse/config/repository.py b/synapse/config/repository.py index 78915894ce90..52d551a33849 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -88,17 +88,17 @@ def parse_thumbnail_requirements(thumbnail_sizes): class ContentRepositoryConfig(Config): def read_config(self, config, **kwargs): - self.external_media_repo = config.get("external_media_repo", False) - if self.external_media_repo and self.enable_media_repo: - raise ConfigError( - "external_media_repo can't be True if the enable_media_repo is True" - ) - - # If we're using an external media repo, do not load the further - # configuration (which checks if files exists). - if self.external_media_repo: + # Only enable the media repo if either the media repo is enabled or the + # current worker app is the media repo. + if ( + self.enable_media_repo is False + and config.worker_app != "synapse.app.media_repository" + ): + self.can_load_media_repo = False return + else: + self.can_load_media_repo = True self.max_upload_size = self.parse_size(config.get("max_upload_size", "10M")) self.max_image_pixels = self.parse_size(config.get("max_image_pixels", "32M")) @@ -219,13 +219,6 @@ def generate_config_section(self, data_dir_path, **kwargs): ## Media Store ## - # Is an external media repo in use? - # - # Set 'False' to use Synapse's media repo. If this is set to 'True', - # Synapse's media repo below must be disabled. - # - #external_media_repo: False - # Enable the media store service inside Synapse. # enable_media_repo: True diff --git a/synapse/rest/admin/__init__.py b/synapse/rest/admin/__init__.py index 197677ca3b22..783af79711ac 100644 --- a/synapse/rest/admin/__init__.py +++ b/synapse/rest/admin/__init__.py @@ -27,7 +27,7 @@ import synapse from synapse.api.constants import Membership, UserTypes -from synapse.api.errors import AuthError, Codes, NotFoundError, SynapseError +from synapse.api.errors import Codes, NotFoundError, SynapseError from synapse.http.server import JsonResource from synapse.http.servlet import ( RestServlet, @@ -36,7 +36,16 @@ parse_json_object_from_request, parse_string, ) -from synapse.rest.admin._base import assert_requester_is_admin, assert_user_is_admin +from synapse.rest.admin._base import ( + assert_requester_is_admin, + assert_user_is_admin, + historical_admin_path_patterns, +) +from synapse.rest.admin.media import ( + ListMediaInRoom, + PurgeMediaCacheRestServlet, + QuarantineMediaInRoom, +) from synapse.rest.admin.server_notice_servlet import SendServerNoticeServlet from synapse.types import UserID, create_requester from synapse.util.versionstring import get_version_string @@ -44,28 +53,6 @@ logger = logging.getLogger(__name__) -def historical_admin_path_patterns(path_regex): - """Returns the list of patterns for an admin endpoint, including historical ones - - This is a backwards-compatibility hack. Previously, the Admin API was exposed at - various paths under /_matrix/client. This function returns a list of patterns - matching those paths (as well as the new one), so that existing scripts which rely - on the endpoints being available there are not broken. - - Note that this should only be used for existing endpoints: new ones should just - register for the /_synapse/admin path. - """ - return list( - re.compile(prefix + path_regex) - for prefix in ( - "^/_synapse/admin/v1", - "^/_matrix/client/api/v1/admin", - "^/_matrix/client/unstable/admin", - "^/_matrix/client/r0/admin", - ) - ) - - class UsersRestServlet(RestServlet): PATTERNS = historical_admin_path_patterns("/users/(?P[^/]*)") @@ -255,25 +242,6 @@ def on_GET(self, request, user_id): return (200, ret) -class PurgeMediaCacheRestServlet(RestServlet): - PATTERNS = historical_admin_path_patterns("/purge_media_cache") - - def __init__(self, hs): - self.media_repository = hs.get_media_repository() - self.auth = hs.get_auth() - - @defer.inlineCallbacks - def on_POST(self, request): - yield assert_requester_is_admin(self.auth, request) - - before_ts = parse_integer(request, "before_ts", required=True) - logger.info("before_ts: %r", before_ts) - - ret = yield self.media_repository.delete_old_remote_media(before_ts) - - return (200, ret) - - class PurgeHistoryRestServlet(RestServlet): PATTERNS = historical_admin_path_patterns( "/purge_history/(?P[^/]*)(/(?P[^/]+))?" @@ -542,50 +510,6 @@ def on_POST(self, request, room_id): ) -class QuarantineMediaInRoom(RestServlet): - """Quarantines all media in a room so that no one can download it via - this server. - """ - - PATTERNS = historical_admin_path_patterns("/quarantine_media/(?P[^/]+)") - - def __init__(self, hs): - self.store = hs.get_datastore() - self.auth = hs.get_auth() - - @defer.inlineCallbacks - def on_POST(self, request, room_id): - requester = yield self.auth.get_user_by_req(request) - yield assert_user_is_admin(self.auth, requester.user) - - num_quarantined = yield self.store.quarantine_media_ids_in_room( - room_id, requester.user.to_string() - ) - - return (200, {"num_quarantined": num_quarantined}) - - -class ListMediaInRoom(RestServlet): - """Lists all of the media in a given room. - """ - - PATTERNS = historical_admin_path_patterns("/room/(?P[^/]+)/media") - - def __init__(self, hs): - self.store = hs.get_datastore() - - @defer.inlineCallbacks - def on_GET(self, request, room_id): - requester = yield self.auth.get_user_by_req(request) - is_admin = yield self.auth.is_server_admin(requester.user) - if not is_admin: - raise AuthError(403, "You are not a server admin") - - local_mxcs, remote_mxcs = yield self.store.get_media_mxcs_in_room(room_id) - - return (200, {"local": local_mxcs, "remote": remote_mxcs}) - - class ResetPasswordRestServlet(RestServlet): """Post request to allow an administrator reset password for a user. This needs user to have administrator access in Synapse. @@ -822,6 +746,15 @@ def register_servlets(hs, http_server): VersionServlet(hs).register(http_server) +def register_servlets_for_media_repo(hs, http_server): + """ + Media repo specific APIs. + """ + PurgeMediaCacheRestServlet(hs).register(http_server) + QuarantineMediaInRoom(hs).register(http_server) + ListMediaInRoom(hs).register(http_server) + + def register_servlets_for_client_rest_resource(hs, http_server): """Register only the servlets which need to be exposed on /_matrix/client/xxx""" WhoisRestServlet(hs).register(http_server) @@ -837,11 +770,9 @@ def register_servlets_for_client_rest_resource(hs, http_server): DeleteGroupAdminRestServlet(hs).register(http_server) AccountValidityRenewServlet(hs).register(http_server) - # Media URLs -- if we're using an external media repo, don't add them. - if not hs.config.external_media_repo: - PurgeMediaCacheRestServlet(hs).register(http_server) - QuarantineMediaInRoom(hs).register(http_server) - ListMediaInRoom(hs).register(http_server) + # Load the media repo ones if we're using them. + if hs.config.can_load_media_repo: + register_servlets_for_client_rest_resource(hs, http_server) # don't add more things here: new servlets should only be exposed on # /_synapse/admin so should not go here. Instead register them in AdminRestResource. diff --git a/synapse/rest/admin/_base.py b/synapse/rest/admin/_base.py index 881d67b89cc4..5a9b08d3ef9c 100644 --- a/synapse/rest/admin/_base.py +++ b/synapse/rest/admin/_base.py @@ -12,11 +12,36 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +import re + from twisted.internet import defer from synapse.api.errors import AuthError +def historical_admin_path_patterns(path_regex): + """Returns the list of patterns for an admin endpoint, including historical ones + + This is a backwards-compatibility hack. Previously, the Admin API was exposed at + various paths under /_matrix/client. This function returns a list of patterns + matching those paths (as well as the new one), so that existing scripts which rely + on the endpoints being available there are not broken. + + Note that this should only be used for existing endpoints: new ones should just + register for the /_synapse/admin path. + """ + return list( + re.compile(prefix + path_regex) + for prefix in ( + "^/_synapse/admin/v1", + "^/_matrix/client/api/v1/admin", + "^/_matrix/client/unstable/admin", + "^/_matrix/client/r0/admin", + ) + ) + + @defer.inlineCallbacks def assert_requester_is_admin(auth, request): """Verify that the requester is an admin user diff --git a/synapse/rest/admin/media.py b/synapse/rest/admin/media.py new file mode 100644 index 000000000000..b47f65496941 --- /dev/null +++ b/synapse/rest/admin/media.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +# Copyright 2014-2016 OpenMarket Ltd +# Copyright 2018-2019 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from twisted.internet import defer + +from synapse.api.errors import AuthError +from synapse.http.servlet import RestServlet, parse_integer +from synapse.rest.admin._base import ( + assert_requester_is_admin, + assert_user_is_admin, + historical_admin_path_patterns, +) + +logger = logging.getLogger(__name__) + + +class QuarantineMediaInRoom(RestServlet): + """Quarantines all media in a room so that no one can download it via + this server. + """ + + PATTERNS = historical_admin_path_patterns("/quarantine_media/(?P[^/]+)") + + def __init__(self, hs): + self.store = hs.get_datastore() + self.auth = hs.get_auth() + + @defer.inlineCallbacks + def on_POST(self, request, room_id): + requester = yield self.auth.get_user_by_req(request) + yield assert_user_is_admin(self.auth, requester.user) + + num_quarantined = yield self.store.quarantine_media_ids_in_room( + room_id, requester.user.to_string() + ) + + return (200, {"num_quarantined": num_quarantined}) + + +class ListMediaInRoom(RestServlet): + """Lists all of the media in a given room. + """ + + PATTERNS = historical_admin_path_patterns("/room/(?P[^/]+)/media") + + def __init__(self, hs): + self.store = hs.get_datastore() + + @defer.inlineCallbacks + def on_GET(self, request, room_id): + requester = yield self.auth.get_user_by_req(request) + is_admin = yield self.auth.is_server_admin(requester.user) + if not is_admin: + raise AuthError(403, "You are not a server admin") + + local_mxcs, remote_mxcs = yield self.store.get_media_mxcs_in_room(room_id) + + return (200, {"local": local_mxcs, "remote": remote_mxcs}) + + +class PurgeMediaCacheRestServlet(RestServlet): + PATTERNS = historical_admin_path_patterns("/purge_media_cache") + + def __init__(self, hs): + self.media_repository = hs.get_media_repository() + self.auth = hs.get_auth() + + @defer.inlineCallbacks + def on_POST(self, request): + yield assert_requester_is_admin(self.auth, request) + + before_ts = parse_integer(request, "before_ts", required=True) + logger.info("before_ts: %r", before_ts) + + ret = yield self.media_repository.delete_old_remote_media(before_ts) + + return (200, ret) diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py index 5135bd4930dc..cf5759e9a693 100644 --- a/synapse/rest/media/v1/media_repository.py +++ b/synapse/rest/media/v1/media_repository.py @@ -33,6 +33,7 @@ RequestSendFailed, SynapseError, ) +from synapse.config._base import ConfigError from synapse.logging.context import defer_to_thread from synapse.metrics.background_process_metrics import run_as_background_process from synapse.util.async_helpers import Linearizer @@ -753,10 +754,9 @@ class MediaRepositoryResource(Resource): """ def __init__(self, hs): - # If this is not the media repo (which will always load this class), - # raise an exception if it is tried to be used. - if hs.config.external_media_repo: - raise Exception("This Synapse is not configured for media repo use.") + # If we're not configured to use it, raise if we somehow got here. + if not hs.config.can_load_media_repo: + raise ConfigError("Synapse is not configured to use a media repo.") super().__init__() media_repo = hs.get_media_repository() From 5f6b68740fb8a329783a9bdecacf6ad089747e4a Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Fri, 9 Aug 2019 22:46:13 +1000 Subject: [PATCH 14/21] cleanup config --- docs/sample_config.yaml | 288 +++++++++++++++++------------------ synapse/config/repository.py | 8 +- 2 files changed, 146 insertions(+), 150 deletions(-) diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 4997f465c2f9..36bb532a03b8 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -564,151 +564,149 @@ log_config: "CONFDIR/SERVERNAME.log.config" #federation_rr_transactions_per_room_per_second: 50 - -## -## Media Store -## - -# Enable the media store service inside Synapse. -# -enable_media_repo: True - -# Directory where uploaded images and attachments are stored. -# -media_store_path: "DATADIR/media_store" - -# Media storage providers allow media to be stored in different -# locations. -# -#media_storage_providers: -# - module: file_system -# # Whether to write new local files. -# store_local: false -# # Whether to write new remote media -# store_remote: false -# # Whether to block upload requests waiting for write to this -# # provider to complete -# store_synchronous: false -# config: -# directory: /mnt/some/other/directory - -# Directory where in-progress uploads are stored. -# -uploads_path: "DATADIR/uploads" - -# The largest allowed upload size in bytes -# -#max_upload_size: 10M - -# Maximum number of pixels that will be thumbnailed -# -#max_image_pixels: 32M - -# Whether to generate new thumbnails on the fly to precisely match -# the resolution requested by the client. If true then whenever -# a new resolution is requested by the client the server will -# generate a new thumbnail. If false the server will pick a thumbnail -# from a precalculated list. -# -#dynamic_thumbnails: false - -# List of thumbnails to precalculate when an image is uploaded. -# -#thumbnail_sizes: -# - width: 32 -# height: 32 -# method: crop -# - width: 96 -# height: 96 -# method: crop -# - width: 320 -# height: 240 -# method: scale -# - width: 640 -# height: 480 -# method: scale -# - width: 800 -# height: 600 -# method: scale - -# Is the preview URL API enabled? -# -# 'false' by default: uncomment the following to enable it (and specify a -# url_preview_ip_range_blacklist blacklist). -# -#url_preview_enabled: true - -# List of IP address CIDR ranges that the URL preview spider is denied -# from accessing. There are no defaults: you must explicitly -# specify a list for URL previewing to work. You should specify any -# internal services in your network that you do not want synapse to try -# to connect to, otherwise anyone in any Matrix room could cause your -# synapse to issue arbitrary GET requests to your internal services, -# causing serious security issues. -# -# (0.0.0.0 and :: are always blacklisted, whether or not they are explicitly -# listed here, since they correspond to unroutable addresses.) -# -# This must be specified if url_preview_enabled is set. It is recommended that -# you uncomment the following list as a starting point. -# -#url_preview_ip_range_blacklist: -# - '127.0.0.0/8' -# - '10.0.0.0/8' -# - '172.16.0.0/12' -# - '192.168.0.0/16' -# - '100.64.0.0/10' -# - '169.254.0.0/16' -# - '::1/128' -# - 'fe80::/64' -# - 'fc00::/7' - -# List of IP address CIDR ranges that the URL preview spider is allowed -# to access even if they are specified in url_preview_ip_range_blacklist. -# This is useful for specifying exceptions to wide-ranging blacklisted -# target IP ranges - e.g. for enabling URL previews for a specific private -# website only visible in your network. -# -#url_preview_ip_range_whitelist: -# - '192.168.1.1' - -# Optional list of URL matches that the URL preview spider is -# denied from accessing. You should use url_preview_ip_range_blacklist -# in preference to this, otherwise someone could define a public DNS -# entry that points to a private IP address and circumvent the blacklist. -# This is more useful if you know there is an entire shape of URL that -# you know that will never want synapse to try to spider. -# -# Each list entry is a dictionary of url component attributes as returned -# by urlparse.urlsplit as applied to the absolute form of the URL. See -# https://docs.python.org/2/library/urlparse.html#urlparse.urlsplit -# The values of the dictionary are treated as an filename match pattern -# applied to that component of URLs, unless they start with a ^ in which -# case they are treated as a regular expression match. If all the -# specified component matches for a given list item succeed, the URL is -# blacklisted. -# -#url_preview_url_blacklist: -# # blacklist any URL with a username in its URI -# - username: '*' -# -# # blacklist all *.google.com URLs -# - netloc: 'google.com' -# - netloc: '*.google.com' -# -# # blacklist all plain HTTP URLs -# - scheme: 'http' -# -# # blacklist http(s)://www.acme.com/foo -# - netloc: 'www.acme.com' -# path: '/foo' -# -# # blacklist any URL with a literal IPv4 address -# - netloc: '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' - -# The largest allowed URL preview spidering size in bytes -# -#max_spider_size: 10M +\ + ## Media Store ## + + # Enable the media store service inside Synapse. + # + # enable_media_repo: False + + # Directory where uploaded images and attachments are stored. + # + media_store_path: "DATADIR/media_store" + + # Media storage providers allow media to be stored in different + # locations. + # + #media_storage_providers: + # - module: file_system + # # Whether to write new local files. + # store_local: false + # # Whether to write new remote media + # store_remote: false + # # Whether to block upload requests waiting for write to this + # # provider to complete + # store_synchronous: false + # config: + # directory: /mnt/some/other/directory + + # Directory where in-progress uploads are stored. + # + uploads_path: "DATADIR/uploads" + + # The largest allowed upload size in bytes + # + #max_upload_size: 10M + + # Maximum number of pixels that will be thumbnailed + # + #max_image_pixels: 32M + + # Whether to generate new thumbnails on the fly to precisely match + # the resolution requested by the client. If true then whenever + # a new resolution is requested by the client the server will + # generate a new thumbnail. If false the server will pick a thumbnail + # from a precalculated list. + # + #dynamic_thumbnails: false + + # List of thumbnails to precalculate when an image is uploaded. + # + #thumbnail_sizes: + # - width: 32 + # height: 32 + # method: crop + # - width: 96 + # height: 96 + # method: crop + # - width: 320 + # height: 240 + # method: scale + # - width: 640 + # height: 480 + # method: scale + # - width: 800 + # height: 600 + # method: scale + + # Is the preview URL API enabled? + # + # 'false' by default: uncomment the following to enable it (and specify a + # url_preview_ip_range_blacklist blacklist). + # + #url_preview_enabled: true + + # List of IP address CIDR ranges that the URL preview spider is denied + # from accessing. There are no defaults: you must explicitly + # specify a list for URL previewing to work. You should specify any + # internal services in your network that you do not want synapse to try + # to connect to, otherwise anyone in any Matrix room could cause your + # synapse to issue arbitrary GET requests to your internal services, + # causing serious security issues. + # + # (0.0.0.0 and :: are always blacklisted, whether or not they are explicitly + # listed here, since they correspond to unroutable addresses.) + # + # This must be specified if url_preview_enabled is set. It is recommended that + # you uncomment the following list as a starting point. + # + #url_preview_ip_range_blacklist: + # - '127.0.0.0/8' + # - '10.0.0.0/8' + # - '172.16.0.0/12' + # - '192.168.0.0/16' + # - '100.64.0.0/10' + # - '169.254.0.0/16' + # - '::1/128' + # - 'fe80::/64' + # - 'fc00::/7' + + # List of IP address CIDR ranges that the URL preview spider is allowed + # to access even if they are specified in url_preview_ip_range_blacklist. + # This is useful for specifying exceptions to wide-ranging blacklisted + # target IP ranges - e.g. for enabling URL previews for a specific private + # website only visible in your network. + # + #url_preview_ip_range_whitelist: + # - '192.168.1.1' + + # Optional list of URL matches that the URL preview spider is + # denied from accessing. You should use url_preview_ip_range_blacklist + # in preference to this, otherwise someone could define a public DNS + # entry that points to a private IP address and circumvent the blacklist. + # This is more useful if you know there is an entire shape of URL that + # you know that will never want synapse to try to spider. + # + # Each list entry is a dictionary of url component attributes as returned + # by urlparse.urlsplit as applied to the absolute form of the URL. See + # https://docs.python.org/2/library/urlparse.html#urlparse.urlsplit + # The values of the dictionary are treated as an filename match pattern + # applied to that component of URLs, unless they start with a ^ in which + # case they are treated as a regular expression match. If all the + # specified component matches for a given list item succeed, the URL is + # blacklisted. + # + #url_preview_url_blacklist: + # # blacklist any URL with a username in its URI + # - username: '*' + # + # # blacklist all *.google.com URLs + # - netloc: 'google.com' + # - netloc: '*.google.com' + # + # # blacklist all plain HTTP URLs + # - scheme: 'http' + # + # # blacklist http(s)://www.acme.com/foo + # - netloc: 'www.acme.com' + # path: '/foo' + # + # # blacklist any URL with a literal IPv4 address + # - netloc: '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' + + # The largest allowed URL preview spidering size in bytes + # + #max_spider_size: 10M ## Captcha ## diff --git a/synapse/config/repository.py b/synapse/config/repository.py index 52d551a33849..cbbffaed72df 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -214,14 +214,12 @@ def generate_config_section(self, data_dir_path, **kwargs): formatted_thumbnail_sizes = formatted_thumbnail_sizes[:-1] return ( - r""" - ## - ## Media Store - ## + r"""\ + ## Media Store ## # Enable the media store service inside Synapse. # - enable_media_repo: True + # enable_media_repo: False # Directory where uploaded images and attachments are stored. # From 7eeed4bec3aec9ad50219ca28bcc39b9677712b1 Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Fri, 9 Aug 2019 22:47:36 +1000 Subject: [PATCH 15/21] fix import --- synapse/app/media_repository.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/app/media_repository.py b/synapse/app/media_repository.py index 8e039a148f91..3a168577c737 100644 --- a/synapse/app/media_repository.py +++ b/synapse/app/media_repository.py @@ -26,6 +26,7 @@ from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging +from synapse.http.server import JsonResource from synapse.http.site import SynapseSite from synapse.logging.context import LoggingContext from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy From 74232f50715413f92934c9f35d31f1cf69e53783 Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Fri, 9 Aug 2019 22:50:21 +1000 Subject: [PATCH 16/21] lol oops --- docs/sample_config.yaml | 286 +++++++++++++++++------------------ synapse/config/repository.py | 2 +- 2 files changed, 144 insertions(+), 144 deletions(-) diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 36bb532a03b8..3dbc757c8480 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -564,149 +564,149 @@ log_config: "CONFDIR/SERVERNAME.log.config" #federation_rr_transactions_per_room_per_second: 50 -\ - ## Media Store ## - - # Enable the media store service inside Synapse. - # - # enable_media_repo: False - - # Directory where uploaded images and attachments are stored. - # - media_store_path: "DATADIR/media_store" - - # Media storage providers allow media to be stored in different - # locations. - # - #media_storage_providers: - # - module: file_system - # # Whether to write new local files. - # store_local: false - # # Whether to write new remote media - # store_remote: false - # # Whether to block upload requests waiting for write to this - # # provider to complete - # store_synchronous: false - # config: - # directory: /mnt/some/other/directory - - # Directory where in-progress uploads are stored. - # - uploads_path: "DATADIR/uploads" - - # The largest allowed upload size in bytes - # - #max_upload_size: 10M - - # Maximum number of pixels that will be thumbnailed - # - #max_image_pixels: 32M - - # Whether to generate new thumbnails on the fly to precisely match - # the resolution requested by the client. If true then whenever - # a new resolution is requested by the client the server will - # generate a new thumbnail. If false the server will pick a thumbnail - # from a precalculated list. - # - #dynamic_thumbnails: false - - # List of thumbnails to precalculate when an image is uploaded. - # - #thumbnail_sizes: - # - width: 32 - # height: 32 - # method: crop - # - width: 96 - # height: 96 - # method: crop - # - width: 320 - # height: 240 - # method: scale - # - width: 640 - # height: 480 - # method: scale - # - width: 800 - # height: 600 - # method: scale - - # Is the preview URL API enabled? - # - # 'false' by default: uncomment the following to enable it (and specify a - # url_preview_ip_range_blacklist blacklist). - # - #url_preview_enabled: true - - # List of IP address CIDR ranges that the URL preview spider is denied - # from accessing. There are no defaults: you must explicitly - # specify a list for URL previewing to work. You should specify any - # internal services in your network that you do not want synapse to try - # to connect to, otherwise anyone in any Matrix room could cause your - # synapse to issue arbitrary GET requests to your internal services, - # causing serious security issues. - # - # (0.0.0.0 and :: are always blacklisted, whether or not they are explicitly - # listed here, since they correspond to unroutable addresses.) - # - # This must be specified if url_preview_enabled is set. It is recommended that - # you uncomment the following list as a starting point. - # - #url_preview_ip_range_blacklist: - # - '127.0.0.0/8' - # - '10.0.0.0/8' - # - '172.16.0.0/12' - # - '192.168.0.0/16' - # - '100.64.0.0/10' - # - '169.254.0.0/16' - # - '::1/128' - # - 'fe80::/64' - # - 'fc00::/7' - - # List of IP address CIDR ranges that the URL preview spider is allowed - # to access even if they are specified in url_preview_ip_range_blacklist. - # This is useful for specifying exceptions to wide-ranging blacklisted - # target IP ranges - e.g. for enabling URL previews for a specific private - # website only visible in your network. - # - #url_preview_ip_range_whitelist: - # - '192.168.1.1' - - # Optional list of URL matches that the URL preview spider is - # denied from accessing. You should use url_preview_ip_range_blacklist - # in preference to this, otherwise someone could define a public DNS - # entry that points to a private IP address and circumvent the blacklist. - # This is more useful if you know there is an entire shape of URL that - # you know that will never want synapse to try to spider. - # - # Each list entry is a dictionary of url component attributes as returned - # by urlparse.urlsplit as applied to the absolute form of the URL. See - # https://docs.python.org/2/library/urlparse.html#urlparse.urlsplit - # The values of the dictionary are treated as an filename match pattern - # applied to that component of URLs, unless they start with a ^ in which - # case they are treated as a regular expression match. If all the - # specified component matches for a given list item succeed, the URL is - # blacklisted. - # - #url_preview_url_blacklist: - # # blacklist any URL with a username in its URI - # - username: '*' - # - # # blacklist all *.google.com URLs - # - netloc: 'google.com' - # - netloc: '*.google.com' - # - # # blacklist all plain HTTP URLs - # - scheme: 'http' - # - # # blacklist http(s)://www.acme.com/foo - # - netloc: 'www.acme.com' - # path: '/foo' - # - # # blacklist any URL with a literal IPv4 address - # - netloc: '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' - - # The largest allowed URL preview spidering size in bytes - # - #max_spider_size: 10M + +## Media Store ## + +# Enable the media store service inside Synapse. +# +# enable_media_repo: False + +# Directory where uploaded images and attachments are stored. +# +media_store_path: "DATADIR/media_store" + +# Media storage providers allow media to be stored in different +# locations. +# +#media_storage_providers: +# - module: file_system +# # Whether to write new local files. +# store_local: false +# # Whether to write new remote media +# store_remote: false +# # Whether to block upload requests waiting for write to this +# # provider to complete +# store_synchronous: false +# config: +# directory: /mnt/some/other/directory + +# Directory where in-progress uploads are stored. +# +uploads_path: "DATADIR/uploads" + +# The largest allowed upload size in bytes +# +#max_upload_size: 10M + +# Maximum number of pixels that will be thumbnailed +# +#max_image_pixels: 32M + +# Whether to generate new thumbnails on the fly to precisely match +# the resolution requested by the client. If true then whenever +# a new resolution is requested by the client the server will +# generate a new thumbnail. If false the server will pick a thumbnail +# from a precalculated list. +# +#dynamic_thumbnails: false + +# List of thumbnails to precalculate when an image is uploaded. +# +#thumbnail_sizes: +# - width: 32 +# height: 32 +# method: crop +# - width: 96 +# height: 96 +# method: crop +# - width: 320 +# height: 240 +# method: scale +# - width: 640 +# height: 480 +# method: scale +# - width: 800 +# height: 600 +# method: scale + +# Is the preview URL API enabled? +# +# 'false' by default: uncomment the following to enable it (and specify a +# url_preview_ip_range_blacklist blacklist). +# +#url_preview_enabled: true + +# List of IP address CIDR ranges that the URL preview spider is denied +# from accessing. There are no defaults: you must explicitly +# specify a list for URL previewing to work. You should specify any +# internal services in your network that you do not want synapse to try +# to connect to, otherwise anyone in any Matrix room could cause your +# synapse to issue arbitrary GET requests to your internal services, +# causing serious security issues. +# +# (0.0.0.0 and :: are always blacklisted, whether or not they are explicitly +# listed here, since they correspond to unroutable addresses.) +# +# This must be specified if url_preview_enabled is set. It is recommended that +# you uncomment the following list as a starting point. +# +#url_preview_ip_range_blacklist: +# - '127.0.0.0/8' +# - '10.0.0.0/8' +# - '172.16.0.0/12' +# - '192.168.0.0/16' +# - '100.64.0.0/10' +# - '169.254.0.0/16' +# - '::1/128' +# - 'fe80::/64' +# - 'fc00::/7' + +# List of IP address CIDR ranges that the URL preview spider is allowed +# to access even if they are specified in url_preview_ip_range_blacklist. +# This is useful for specifying exceptions to wide-ranging blacklisted +# target IP ranges - e.g. for enabling URL previews for a specific private +# website only visible in your network. +# +#url_preview_ip_range_whitelist: +# - '192.168.1.1' + +# Optional list of URL matches that the URL preview spider is +# denied from accessing. You should use url_preview_ip_range_blacklist +# in preference to this, otherwise someone could define a public DNS +# entry that points to a private IP address and circumvent the blacklist. +# This is more useful if you know there is an entire shape of URL that +# you know that will never want synapse to try to spider. +# +# Each list entry is a dictionary of url component attributes as returned +# by urlparse.urlsplit as applied to the absolute form of the URL. See +# https://docs.python.org/2/library/urlparse.html#urlparse.urlsplit +# The values of the dictionary are treated as an filename match pattern +# applied to that component of URLs, unless they start with a ^ in which +# case they are treated as a regular expression match. If all the +# specified component matches for a given list item succeed, the URL is +# blacklisted. +# +#url_preview_url_blacklist: +# # blacklist any URL with a username in its URI +# - username: '*' +# +# # blacklist all *.google.com URLs +# - netloc: 'google.com' +# - netloc: '*.google.com' +# +# # blacklist all plain HTTP URLs +# - scheme: 'http' +# +# # blacklist http(s)://www.acme.com/foo +# - netloc: 'www.acme.com' +# path: '/foo' +# +# # blacklist any URL with a literal IPv4 address +# - netloc: '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' + +# The largest allowed URL preview spidering size in bytes +# +#max_spider_size: 10M ## Captcha ## diff --git a/synapse/config/repository.py b/synapse/config/repository.py index cbbffaed72df..f154c2e14fbc 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -214,7 +214,7 @@ def generate_config_section(self, data_dir_path, **kwargs): formatted_thumbnail_sizes = formatted_thumbnail_sizes[:-1] return ( - r"""\ + r""" ## Media Store ## # Enable the media store service inside Synapse. From 7c2149fee217f73d1fd3a9fc581daec4da79965e Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Fri, 9 Aug 2019 22:51:58 +1000 Subject: [PATCH 17/21] reduce diff --- synapse/app/homeserver.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 8279334185c3..7d6b51b5bc09 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -219,11 +219,7 @@ def _configure_named_resource(self, name, compress=False): ) if name in ["media", "federation", "client"]: - if name == "media" and not self.get_config().enable_media_repo: - raise ConfigError( - "'media' resource conflicts with enable_media_repo=False" - ) - elif self.get_config().enable_media_repo: + if self.get_config().enable_media_repo: media_repo = self.get_media_repository_resource() resources.update( { @@ -234,6 +230,10 @@ def _configure_named_resource(self, name, compress=False): ), } ) + elif name == "media": + raise ConfigError( + "'media' resource conflicts with enable_media_repo=False" + ) if name in ["keys", "federation"]: resources[SERVER_KEY_V2_PREFIX] = KeyApiV2Resource(self) From 3a23f6a7a2509824496f083531556f351f460955 Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Fri, 9 Aug 2019 22:53:39 +1000 Subject: [PATCH 18/21] doc --- changelog.d/5754.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/5754.feature b/changelog.d/5754.feature index 9a019039df76..c1a09a4dcee5 100644 --- a/changelog.d/5754.feature +++ b/changelog.d/5754.feature @@ -1 +1 @@ -Synapse will no longer serve any media repo admin endpoints when `enable_media_repo` is set to False in the configuration. \ No newline at end of file +Synapse will no longer serve any media repo admin endpoints when `enable_media_repo` is set to False in the configuration. If a media repo worker is used, the admin APIs relating to the media repo will be served from it instead. \ No newline at end of file From afb31e957508e24cc82ff95c398127700bc41fcf Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 9 Aug 2019 23:58:44 +1000 Subject: [PATCH 19/21] Update docs/sample_config.yaml Co-Authored-By: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- docs/sample_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 3dbc757c8480..1b50d2e88271 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -569,7 +569,7 @@ log_config: "CONFDIR/SERVERNAME.log.config" # Enable the media store service inside Synapse. # -# enable_media_repo: False +# enable_media_repo: false # Directory where uploaded images and attachments are stored. # From 4a0380bb802fc0c25eb75f573bf32586a3a4d3e6 Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Sat, 10 Aug 2019 02:02:01 +1000 Subject: [PATCH 20/21] fix --- docs/sample_config.yaml | 2 +- synapse/rest/admin/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 1b50d2e88271..3dbc757c8480 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -569,7 +569,7 @@ log_config: "CONFDIR/SERVERNAME.log.config" # Enable the media store service inside Synapse. # -# enable_media_repo: false +# enable_media_repo: False # Directory where uploaded images and attachments are stored. # diff --git a/synapse/rest/admin/__init__.py b/synapse/rest/admin/__init__.py index 783af79711ac..f62738c72bad 100644 --- a/synapse/rest/admin/__init__.py +++ b/synapse/rest/admin/__init__.py @@ -772,7 +772,7 @@ def register_servlets_for_client_rest_resource(hs, http_server): # Load the media repo ones if we're using them. if hs.config.can_load_media_repo: - register_servlets_for_client_rest_resource(hs, http_server) + register_servlets_for_media_repo(hs, http_server) # don't add more things here: new servlets should only be exposed on # /_synapse/admin so should not go here. Instead register them in AdminRestResource. From 80d6c4888075bc32b53f1c7d32673576979d383f Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Tue, 13 Aug 2019 21:33:57 +1000 Subject: [PATCH 21/21] review cleanups --- .gitignore | 1 + docs/sample_config.yaml | 5 +++-- synapse/config/repository.py | 5 +++-- synapse/rest/admin/__init__.py | 15 +-------------- synapse/rest/admin/media.py | 9 +++++++++ 5 files changed, 17 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index a84c41b0c95a..f6168a8819a3 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ _trial_temp*/ /*.log /*.log.config /*.pid +/.python-version /*.signing.key /env/ /homeserver*.yaml diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 3dbc757c8480..0c6be30e513d 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -567,9 +567,10 @@ log_config: "CONFDIR/SERVERNAME.log.config" ## Media Store ## -# Enable the media store service inside Synapse. +# Enable the media store service in the Synapse master. Uncomment the +# following if you are using a separate media store worker. # -# enable_media_repo: False +#enable_media_repo: false # Directory where uploaded images and attachments are stored. # diff --git a/synapse/config/repository.py b/synapse/config/repository.py index f154c2e14fbc..db39697e45b1 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -217,9 +217,10 @@ def generate_config_section(self, data_dir_path, **kwargs): r""" ## Media Store ## - # Enable the media store service inside Synapse. + # Enable the media store service in the Synapse master. Uncomment the + # following if you are using a separate media store worker. # - # enable_media_repo: False + #enable_media_repo: false # Directory where uploaded images and attachments are stored. # diff --git a/synapse/rest/admin/__init__.py b/synapse/rest/admin/__init__.py index f62738c72bad..5720cab42588 100644 --- a/synapse/rest/admin/__init__.py +++ b/synapse/rest/admin/__init__.py @@ -41,11 +41,7 @@ assert_user_is_admin, historical_admin_path_patterns, ) -from synapse.rest.admin.media import ( - ListMediaInRoom, - PurgeMediaCacheRestServlet, - QuarantineMediaInRoom, -) +from synapse.rest.admin.media import register_servlets_for_media_repo from synapse.rest.admin.server_notice_servlet import SendServerNoticeServlet from synapse.types import UserID, create_requester from synapse.util.versionstring import get_version_string @@ -746,15 +742,6 @@ def register_servlets(hs, http_server): VersionServlet(hs).register(http_server) -def register_servlets_for_media_repo(hs, http_server): - """ - Media repo specific APIs. - """ - PurgeMediaCacheRestServlet(hs).register(http_server) - QuarantineMediaInRoom(hs).register(http_server) - ListMediaInRoom(hs).register(http_server) - - def register_servlets_for_client_rest_resource(hs, http_server): """Register only the servlets which need to be exposed on /_matrix/client/xxx""" WhoisRestServlet(hs).register(http_server) diff --git a/synapse/rest/admin/media.py b/synapse/rest/admin/media.py index b47f65496941..824df919f269 100644 --- a/synapse/rest/admin/media.py +++ b/synapse/rest/admin/media.py @@ -90,3 +90,12 @@ def on_POST(self, request): ret = yield self.media_repository.delete_old_remote_media(before_ts) return (200, ret) + + +def register_servlets_for_media_repo(hs, http_server): + """ + Media repo specific APIs. + """ + PurgeMediaCacheRestServlet(hs).register(http_server) + QuarantineMediaInRoom(hs).register(http_server) + ListMediaInRoom(hs).register(http_server)