From 38d518c3313fe83eeda71a0c3b2ea507e2ae3185 Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Thu, 10 Feb 2022 10:04:31 -0800 Subject: [PATCH 01/19] add config section --- synapse/config/background_updates.py | 65 ++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 synapse/config/background_updates.py diff --git a/synapse/config/background_updates.py b/synapse/config/background_updates.py new file mode 100644 index 000000000000..0e84bf1428c4 --- /dev/null +++ b/synapse/config/background_updates.py @@ -0,0 +1,65 @@ +# Copyright 2019-2021 Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._base import Config + + +class BackgroundUpdateConfig(Config): + section = "background_updates" + + def generate_config_section(self, **kwargs) -> str: + return """\ + + + ## Background Updates ## + + # Background updates are database updates that are run in the background in batches. + # The duration, minimum batch size, default batch size, whether to sleep between batches and if so, how long to + # sleep can all be configured. This is helpful to speed up or slow down the updates. + + # How long in milliseconds to run a batch of background updates for. Defaults to 100. Uncomment and set + # a time to change the default. + # + #background_update_duration_ms: 500 + + # Whether to sleep between updates. Defaults to True. Uncomment to change the default. + # + #sleep: false + + # If sleeping between updates, how long in milliseconds to sleep for. Defaults to 1000. Uncomment + # and set a duration to change the default. + # + #sleep_duration_ms: 300 + + # Minimum size a batch of background updates can be. Must be greater than 0. Defaults to 1. Uncomment and + # set a size to change the default. + # + #min_batch_size: 10 + + # The batch size to use for the first iteration of a new background update. The default is 100. + # Uncomment and set a size to change the default. + # + #default_batch_size: 50 + """ + + def read_config(self, config, **kwargs) -> None: + self.update_duration_ms = config.get("background_update_duration_ms", 100) + + self.sleep_enabled = config.get("sleep", True) + + self.sleep_duration_ms = config.get("sleep_duration_ms", 1000) + + self.min_batch_size = config.get("min_batch_size", 1) + + self.default_batch_size = config.get("default_batch_size", 100) From 40e806e0e49e788eda8c5aa82ba7b52e99b6926e Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Thu, 10 Feb 2022 10:09:03 -0800 Subject: [PATCH 02/19] add values gathered from config to background updater --- synapse/storage/background_updates.py | 39 +++++++++++++++++---------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index d64910aded33..1be8bd6f2cef 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -60,18 +60,19 @@ class _BackgroundUpdateHandler: class _BackgroundUpdateContextManager: - BACKGROUND_UPDATE_INTERVAL_MS = 1000 - BACKGROUND_UPDATE_DURATION_MS = 100 - - def __init__(self, sleep: bool, clock: Clock): + def __init__( + self, sleep: bool, clock: Clock, sleep_duration_ms: int, update_duration: int + ): self._sleep = sleep self._clock = clock + self._sleep_duration_ms = sleep_duration_ms + self._update_duration_ms = update_duration async def __aenter__(self) -> int: if self._sleep: - await self._clock.sleep(self.BACKGROUND_UPDATE_INTERVAL_MS / 1000) + await self._clock.sleep(self._sleep_duration_ms / 1000) - return self.BACKGROUND_UPDATE_DURATION_MS + return self._update_duration_ms async def __aexit__(self, *exc) -> None: pass @@ -131,9 +132,6 @@ class BackgroundUpdater: process and autotuning the batch size. """ - MINIMUM_BACKGROUND_BATCH_SIZE = 1 - DEFAULT_BACKGROUND_BATCH_SIZE = 100 - def __init__(self, hs: "HomeServer", database: "DatabasePool"): self._clock = hs.get_clock() self.db_pool = database @@ -158,6 +156,14 @@ def __init__(self, hs: "HomeServer", database: "DatabasePool"): # enable/disable background updates via the admin API. self.enabled = True + self.minimum_background_batch_size = hs.config.background_updates.min_batch_size + self.default_background_batch_size = ( + hs.config.background_updates.default_batch_size + ) + self.update_duration_ms = hs.config.background_updates.update_duration_ms + self.sleep_duration_ms = hs.config.background_updates.sleep_duration_ms + self.sleep_enabled = hs.config.background_updates.sleep_enabled + def register_update_controller_callbacks( self, on_update: ON_UPDATE_CALLBACK, @@ -214,7 +220,9 @@ def _get_context_manager_for_update( if self._on_update_callback is not None: return self._on_update_callback(update_name, database_name, oneshot) - return _BackgroundUpdateContextManager(sleep, self._clock) + return _BackgroundUpdateContextManager( + sleep, self._clock, self.sleep_duration_ms, self.update_duration_ms + ) async def _default_batch_size(self, update_name: str, database_name: str) -> int: """The batch size to use for the first iteration of a new background @@ -223,7 +231,7 @@ async def _default_batch_size(self, update_name: str, database_name: str) -> int if self._default_batch_size_callback is not None: return await self._default_batch_size_callback(update_name, database_name) - return self.DEFAULT_BACKGROUND_BATCH_SIZE + return self.default_background_batch_size async def _min_batch_size(self, update_name: str, database_name: str) -> int: """A lower bound on the batch size of a new background update. @@ -233,7 +241,7 @@ async def _min_batch_size(self, update_name: str, database_name: str) -> int: if self._min_batch_size_callback is not None: return await self._min_batch_size_callback(update_name, database_name) - return self.MINIMUM_BACKGROUND_BATCH_SIZE + return self.minimum_background_batch_size def get_current_update(self) -> Optional[BackgroundUpdatePerformance]: """Returns the current background update, if any.""" @@ -252,9 +260,12 @@ def start_doing_background_updates(self) -> None: if self.enabled: # if we start a new background update, not all updates are done. self._all_done = False - run_as_background_process("background_updates", self.run_background_updates) + sleep = self.sleep_enabled + run_as_background_process( + "background_updates", self.run_background_updates, sleep + ) - async def run_background_updates(self, sleep: bool = True) -> None: + async def run_background_updates(self, sleep: bool) -> None: if self._running or not self.enabled: return From 1948a44819c59e57278305e422ead0eb13d8762c Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Thu, 10 Feb 2022 10:10:17 -0800 Subject: [PATCH 03/19] refactor existing tests to use new values --- tests/rest/admin/test_background_updates.py | 10 ++++++---- tests/storage/test_background_update.py | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/rest/admin/test_background_updates.py b/tests/rest/admin/test_background_updates.py index 1e3fe9c62c00..9eac9a9f46eb 100644 --- a/tests/rest/admin/test_background_updates.py +++ b/tests/rest/admin/test_background_updates.py @@ -39,6 +39,7 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.store = hs.get_datastore() self.admin_user = self.register_user("admin", "pass", admin=True) self.admin_user_tok = self.login("admin", "pass") + self.updater = BackgroundUpdater(hs, self.store.db_pool) @parameterized.expand( [ @@ -139,6 +140,7 @@ def test_status_bg_update(self) -> None: self._register_bg_update() self.store.db_pool.updates.start_doing_background_updates() + self.reactor.pump([1.0, 1.0, 1.0]) channel = self.make_request( @@ -158,7 +160,7 @@ def test_status_bg_update(self) -> None: "average_items_per_ms": 0.001, "total_duration_ms": 1000.0, "total_item_count": ( - BackgroundUpdater.MINIMUM_BACKGROUND_BATCH_SIZE + self.updater.minimum_background_batch_size ), } }, @@ -213,7 +215,7 @@ def test_enabled(self) -> None: "average_items_per_ms": 0.001, "total_duration_ms": 1000.0, "total_item_count": ( - BackgroundUpdater.MINIMUM_BACKGROUND_BATCH_SIZE + self.updater.minimum_background_batch_size ), } }, @@ -242,7 +244,7 @@ def test_enabled(self) -> None: "average_items_per_ms": 0.001, "total_duration_ms": 1000.0, "total_item_count": ( - BackgroundUpdater.MINIMUM_BACKGROUND_BATCH_SIZE + self.updater.minimum_background_batch_size ), } }, @@ -281,7 +283,7 @@ def test_enabled(self) -> None: "average_items_per_ms": 0.001, "total_duration_ms": 2000.0, "total_item_count": ( - 2 * BackgroundUpdater.MINIMUM_BACKGROUND_BATCH_SIZE + 2 * self.updater.minimum_background_batch_size ), } }, diff --git a/tests/storage/test_background_update.py b/tests/storage/test_background_update.py index 6156dfac4e58..af3015b753f8 100644 --- a/tests/storage/test_background_update.py +++ b/tests/storage/test_background_update.py @@ -72,7 +72,7 @@ async def update(progress, count): # on the first call, we should get run with the default background update size self.update_handler.assert_called_once_with( - {"my_key": 1}, self.updates.MINIMUM_BACKGROUND_BATCH_SIZE + {"my_key": 1}, self.updates.minimum_background_batch_size ) # second step: complete the update From 29d1377868dc96ec8f4f555bd1729ef1ecfcfb3b Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Thu, 10 Feb 2022 10:10:50 -0800 Subject: [PATCH 04/19] make class available and appease mypy --- synapse/config/_base.pyi | 2 ++ synapse/config/homeserver.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/synapse/config/_base.pyi b/synapse/config/_base.pyi index 1eb5f5a68cf8..363d8b45545e 100644 --- a/synapse/config/_base.pyi +++ b/synapse/config/_base.pyi @@ -19,6 +19,7 @@ from synapse.config import ( api, appservice, auth, + background_updates, cache, captcha, cas, @@ -113,6 +114,7 @@ class RootConfig: caches: cache.CacheConfig federation: federation.FederationConfig retention: retention.RetentionConfig + background_updates: background_updates.BackgroundUpdateConfig config_classes: List[Type["Config"]] = ... def __init__(self) -> None: ... diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index 001605c265fb..a4ec70690802 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -16,6 +16,7 @@ from .api import ApiConfig from .appservice import AppServiceConfig from .auth import AuthConfig +from .background_updates import BackgroundUpdateConfig from .cache import CacheConfig from .captcha import CaptchaConfig from .cas import CasConfig @@ -99,4 +100,5 @@ class HomeServerConfig(RootConfig): WorkerConfig, RedisConfig, ExperimentalConfig, + BackgroundUpdateConfig, ] From 9efbe5fe283bc206c41afe4dc057303610252ec5 Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Fri, 11 Feb 2022 15:05:54 -0800 Subject: [PATCH 05/19] slightly better wording --- synapse/config/background_updates.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/config/background_updates.py b/synapse/config/background_updates.py index 0e84bf1428c4..2178604eb9e4 100644 --- a/synapse/config/background_updates.py +++ b/synapse/config/background_updates.py @@ -35,7 +35,7 @@ def generate_config_section(self, **kwargs) -> str: # Whether to sleep between updates. Defaults to True. Uncomment to change the default. # - #sleep: false + #sleep_enabled: false # If sleeping between updates, how long in milliseconds to sleep for. Defaults to 1000. Uncomment # and set a duration to change the default. @@ -56,7 +56,7 @@ def generate_config_section(self, **kwargs) -> str: def read_config(self, config, **kwargs) -> None: self.update_duration_ms = config.get("background_update_duration_ms", 100) - self.sleep_enabled = config.get("sleep", True) + self.sleep_enabled = config.get("sleep_enabled", True) self.sleep_duration_ms = config.get("sleep_duration_ms", 1000) From f2b4b4a3885a39bf28a6a2cc723fccb344325f79 Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Fri, 11 Feb 2022 15:06:49 -0800 Subject: [PATCH 06/19] add some tests --- tests/config/test_background_update.py | 53 ++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 tests/config/test_background_update.py diff --git a/tests/config/test_background_update.py b/tests/config/test_background_update.py new file mode 100644 index 000000000000..2303e27e27b2 --- /dev/null +++ b/tests/config/test_background_update.py @@ -0,0 +1,53 @@ +# Copyright 2022 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import yaml + +from synapse.storage.background_updates import BackgroundUpdater + +from tests.unittest import HomeserverTestCase, override_config + + +class RoomDirectoryConfigTestCase(HomeserverTestCase): + # Tests that the default values in the config are correctly loaded. Note that the default + # values are loaded when the corresponding config options are commented out, which is why there isn't + # a config specified here. + def test_default_configuration(self): + background_updater = BackgroundUpdater(self.hs, self.hs.get_datastore().db_pool) + + self.assertEqual(background_updater.minimum_background_batch_size, 1) + self.assertEqual(background_updater.default_background_batch_size, 100) + self.assertEqual(background_updater.sleep_enabled, True) + self.assertEqual(background_updater.sleep_duration_ms, 1000) + self.assertEqual(background_updater.update_duration_ms, 100) + + # Tests that non-default values for the config options are properly picked up and passed on. + @override_config( + yaml.safe_load( + """ + background_update_duration_ms: 1000 + sleep_enabled: false + sleep_duration_ms: 600 + min_batch_size: 5 + default_batch_size: 50 + """ + ) + ) + def test_custom_configuration(self): + background_updater = BackgroundUpdater(self.hs, self.hs.get_datastore().db_pool) + + self.assertEqual(background_updater.minimum_background_batch_size, 5) + self.assertEqual(background_updater.default_background_batch_size, 50) + self.assertEqual(background_updater.sleep_enabled, False) + self.assertEqual(background_updater.sleep_duration_ms, 600) + self.assertEqual(background_updater.update_duration_ms, 1000) From b3e0c9bc83a4c81f82d166d01e268f9464922f02 Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Fri, 11 Feb 2022 15:14:15 -0800 Subject: [PATCH 07/19] newsfragment --- changelog.d/11980.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/11980.misc diff --git a/changelog.d/11980.misc b/changelog.d/11980.misc new file mode 100644 index 000000000000..36e992e645a3 --- /dev/null +++ b/changelog.d/11980.misc @@ -0,0 +1 @@ +Add config settings for background update parameters. \ No newline at end of file From d66cbdfc28b90590c1a1be276931a109a915cca7 Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Fri, 11 Feb 2022 15:24:07 -0800 Subject: [PATCH 08/19] fix license date --- synapse/config/background_updates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/config/background_updates.py b/synapse/config/background_updates.py index 2178604eb9e4..fe39da329086 100644 --- a/synapse/config/background_updates.py +++ b/synapse/config/background_updates.py @@ -1,4 +1,4 @@ -# Copyright 2019-2021 Matrix.org Foundation C.I.C. +# Copyright 2022 Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 6e1aafb316946fdbf3a8d22517bc27597e122aca Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Fri, 11 Feb 2022 15:34:06 -0800 Subject: [PATCH 09/19] regenerate sample config --- docs/sample_config.yaml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 946cd281d265..d0029f9fa748 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -2724,3 +2724,36 @@ redis: # Optional password if configured on the Redis instance # #password: + + + + +## Background Updates ## + +# Background updates are database updates that are run in the background in batches. +# The duration, minimum batch size, default batch size, whether to sleep between batches and if so, how long to +# sleep can all be configured. This is helpful to speed up or slow down the updates. + +# How long in milliseconds to run a batch of background updates for. Defaults to 100. Uncomment and set +# a time to change the default. +# +#background_update_duration_ms: 500 + +# Whether to sleep between updates. Defaults to True. Uncomment to change the default. +# +#sleep_enabled: false + +# If sleeping between updates, how long in milliseconds to sleep for. Defaults to 1000. Uncomment +# and set a duration to change the default. +# +#sleep_duration_ms: 300 + +# Minimum size a batch of background updates can be. Must be greater than 0. Defaults to 1. Uncomment and +# set a size to change the default. +# +#min_batch_size: 10 + +# The batch size to use for the first iteration of a new background update. The default is 100. +# Uncomment and set a size to change the default. +# +#default_batch_size: 50 From d2fc2a35e385006f069d074b6b0bba38d7138ce2 Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Thu, 24 Feb 2022 11:38:53 -0800 Subject: [PATCH 10/19] add a test and some notes to myself --- tests/storage/test_background_update.py | 51 +++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/tests/storage/test_background_update.py b/tests/storage/test_background_update.py index af3015b753f8..ed226dc828ed 100644 --- a/tests/storage/test_background_update.py +++ b/tests/storage/test_background_update.py @@ -14,12 +14,14 @@ from unittest.mock import Mock +import yaml from twisted.internet.defer import Deferred, ensureDeferred from synapse.storage.background_updates import BackgroundUpdater from tests import unittest from tests.test_utils import make_awaitable, simple_async_mock +from tests.unittest import override_config class BackgroundUpdateTestCase(unittest.HomeserverTestCase): @@ -99,6 +101,55 @@ async def update(progress, count): self.assertTrue(result) self.assertFalse(self.update_handler.called) + # test that background update sleeps when asked and doesn't sleep when asked and uses proper duration + # test that background update is run with specified default batch size + # test that background update is run with specified duration + @override_config( + yaml.safe_load( + """ + min_batch_size: 5 + """ + ) + ) + def test_do_background_update_with_min_batch_set_by_config(self): + """ + Test that the background update is run with the min_batch_size set by the config + """ + # the time we claim it takes to update one item when running the update + duration_ms = 10 + + store = self.hs.get_datastore() + self.get_success( + store.db_pool.simple_insert( + "background_updates", + values={"update_name": "test_update", "progress_json": '{"my_key": 1}'}, + ) + ) + + # first step: make a bit of progress + async def update(progress, count): + await self.clock.sleep((count * duration_ms) / 1000) + progress = {"my_key": progress["my_key"] + 1} + await store.db_pool.runInteraction( + "update_progress", + self.updates._background_update_progress_txn, + "test_update", + progress, + ) + return count + + self.update_handler.side_effect = update + self.update_handler.reset_mock() + res = self.get_success( + self.updates.do_next_background_update(False), + by=0.01, + ) + self.assertFalse(res) + + # on the first call, we should get run with the minimum background update size specified in the config + self.update_handler.assert_called_once_with( + {"my_key": 1}, 5 + ) class BackgroundUpdateControllerTestCase(unittest.HomeserverTestCase): def prepare(self, reactor, clock, homeserver): From af7e0009bf8059ce5969a24eeaf3827e49de9ca9 Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Mon, 7 Mar 2022 14:34:34 -0800 Subject: [PATCH 11/19] add some tests + cleanup --- tests/config/test_background_update.py | 8 +- tests/rest/admin/test_background_updates.py | 6 +- tests/storage/test_background_update.py | 258 +++++++++++++++++++- 3 files changed, 255 insertions(+), 17 deletions(-) diff --git a/tests/config/test_background_update.py b/tests/config/test_background_update.py index 2303e27e27b2..b44c8c9a989a 100644 --- a/tests/config/test_background_update.py +++ b/tests/config/test_background_update.py @@ -23,7 +23,9 @@ class RoomDirectoryConfigTestCase(HomeserverTestCase): # values are loaded when the corresponding config options are commented out, which is why there isn't # a config specified here. def test_default_configuration(self): - background_updater = BackgroundUpdater(self.hs, self.hs.get_datastore().db_pool) + background_updater = BackgroundUpdater( + self.hs, self.hs.get_datastores().main.db_pool + ) self.assertEqual(background_updater.minimum_background_batch_size, 1) self.assertEqual(background_updater.default_background_batch_size, 100) @@ -44,7 +46,9 @@ def test_default_configuration(self): ) ) def test_custom_configuration(self): - background_updater = BackgroundUpdater(self.hs, self.hs.get_datastore().db_pool) + background_updater = BackgroundUpdater( + self.hs, self.hs.get_datastores().main.db_pool + ) self.assertEqual(background_updater.minimum_background_batch_size, 5) self.assertEqual(background_updater.default_background_batch_size, 50) diff --git a/tests/rest/admin/test_background_updates.py b/tests/rest/admin/test_background_updates.py index aa3759bfa045..c3153eb46413 100644 --- a/tests/rest/admin/test_background_updates.py +++ b/tests/rest/admin/test_background_updates.py @@ -160,7 +160,7 @@ def test_status_bg_update(self) -> None: "average_items_per_ms": 0.1, "total_duration_ms": 1000.0, "total_item_count": ( - BackgroundUpdater.DEFAULT_BACKGROUND_BATCH_SIZE + BackgroundUpdater.default_background_batch_size ), } }, @@ -215,7 +215,7 @@ def test_enabled(self) -> None: "average_items_per_ms": 0.1, "total_duration_ms": 1000.0, "total_item_count": ( - BackgroundUpdater.DEFAULT_BACKGROUND_BATCH_SIZE + BackgroundUpdater.default_background_batch_size ), } }, @@ -244,7 +244,7 @@ def test_enabled(self) -> None: "average_items_per_ms": 0.1, "total_duration_ms": 1000.0, "total_item_count": ( - BackgroundUpdater.DEFAULT_BACKGROUND_BATCH_SIZE + BackgroundUpdater.default_background_batch_size ), } }, diff --git a/tests/storage/test_background_update.py b/tests/storage/test_background_update.py index 72ff991e1f2a..7eb17f4826df 100644 --- a/tests/storage/test_background_update.py +++ b/tests/storage/test_background_update.py @@ -15,6 +15,7 @@ from unittest.mock import Mock import yaml + from twisted.internet.defer import Deferred, ensureDeferred from synapse.storage.background_updates import BackgroundUpdater @@ -74,7 +75,7 @@ async def update(progress, count): # on the first call, we should get run with the default background update size self.update_handler.assert_called_once_with( - {"my_key": 1}, self.updates.DEFAULT_BACKGROUND_BATCH_SIZE + {"my_key": 1}, self.updates.default_background_batch_size ) # second step: complete the update @@ -101,24 +102,21 @@ async def update(progress, count): self.assertTrue(result) self.assertFalse(self.update_handler.called) - # test that background update sleeps when asked and doesn't sleep when asked and uses proper duration - # test that background update is run with specified default batch size - # test that background update is run with specified duration @override_config( yaml.safe_load( """ - min_batch_size: 5 + default_batch_size: 20 """ ) ) - def test_do_background_update_with_min_batch_set_by_config(self): + def test_do_background_update_default_batch_set_by_config(self): """ - Test that the background update is run with the min_batch_size set by the config + Test that the background update is run with the default_batch_size set by the config """ # the time we claim it takes to update one item when running the update duration_ms = 10 - store = self.hs.get_datastore() + store = self.hs.get_datastores().main self.get_success( store.db_pool.simple_insert( "background_updates", @@ -126,7 +124,6 @@ def test_do_background_update_with_min_batch_set_by_config(self): ) ) - # first step: make a bit of progress async def update(progress, count): await self.clock.sleep((count * duration_ms) / 1000) progress = {"my_key": progress["my_key"] + 1} @@ -146,10 +143,247 @@ async def update(progress, count): ) self.assertFalse(res) - # on the first call, we should get run with the minimum background update size specified in the config - self.update_handler.assert_called_once_with( - {"my_key": 1}, 5 + # on the first call, we should get run with the default background update size specified in the config + self.update_handler.assert_called_once_with({"my_key": 1}, 20) + + def test_background_update_default_sleep_behavior(self): + """ + Test default background update behavior, which is to sleep + """ + + # the time we claim it takes to update one item when running the update + duration_ms = 10 + + store = self.hs.get_datastores().main + self.get_success( + store.db_pool.simple_insert( + "background_updates", + values={"update_name": "test_update", "progress_json": '{"my_key": 1}'}, + ) + ) + + async def update(progress, count): + await self.clock.sleep((count * duration_ms) / 1000) + progress = {"my_key": progress["my_key"] + 1} + await store.db_pool.runInteraction( + "update_progress", + self.updates._background_update_progress_txn, + "test_update", + progress, + ) + return count + + self.update_handler.side_effect = update + self.update_handler.reset_mock() + self.updates.start_doing_background_updates(), + + # 2: advance the reactor less than the default sleep duration (1000ms) + self.reactor.pump([0.5]) + # check that an update has not been run + self.update_handler.assert_not_called() + + # advance reactor past default sleep duration + self.reactor.pump([1]) + # check that update has been run + self.update_handler.assert_called() + + @override_config( + yaml.safe_load( + """ + sleep_duration_ms: 500 + """ + ) + ) + def test_background_update_sleep_set_in_config(self): + """ + Test that changing the sleep time in the config changes how long it sleeps + """ + + duration_ms = 10 + + store = self.hs.get_datastores().main + self.get_success( + store.db_pool.simple_insert( + "background_updates", + values={"update_name": "test_update", "progress_json": '{"my_key": 1}'}, + ) + ) + + async def update(progress, count): + await self.clock.sleep((count * duration_ms) / 1000) + progress = {"my_key": progress["my_key"] + 1} + await store.db_pool.runInteraction( + "update_progress", + self.updates._background_update_progress_txn, + "test_update", + progress, + ) + return count + + self.update_handler.side_effect = update + self.update_handler.reset_mock() + self.updates.start_doing_background_updates(), + + # 2: advance the reactor less than the configured sleep duration (500ms) + self.reactor.pump([0.45]) + # check that an update has not been run + self.update_handler.assert_not_called() + + # advance reactor past config sleep duration but less than default duration + self.reactor.pump([0.75]) + # check that update has been run + self.update_handler.assert_called() + + @override_config( + yaml.safe_load( + """ + sleep_enabled: false + """ + ) + ) + def test_disabling_background_update_sleep(self): + """ + Test that disabling sleep in the config results in bg update not sleeping + """ + # the time we claim it takes to update one item when running the update + duration_ms = 10 + + store = self.hs.get_datastores().main + self.get_success( + store.db_pool.simple_insert( + "background_updates", + values={"update_name": "test_update", "progress_json": '{"my_key": 1}'}, + ) + ) + + async def update(progress, count): + await self.clock.sleep((count * duration_ms) / 1000) + progress = {"my_key": progress["my_key"] + 1} + await store.db_pool.runInteraction( + "update_progress", + self.updates._background_update_progress_txn, + "test_update", + progress, + ) + return count + + self.update_handler.side_effect = update + self.update_handler.reset_mock() + self.updates.start_doing_background_updates(), + + # 2: advance the reactor very little + self.reactor.pump([0.025]) + # check that an update has run + self.update_handler.assert_called() + + @override_config( + yaml.safe_load( + """ + background_update_duration_ms: 500 + """ + ) + ) + def test_background_update_duration_set_in_config(self): + """ + Test that the desired duration set in the config is used in determining batch size + """ + # the time we claim it takes to update one item when running the update + duration_ms = 10 + + store = self.hs.get_datastores().main + self.get_success( + store.db_pool.simple_insert( + "background_updates", + values={"update_name": "test_update", "progress_json": '{"my_key": 1}'}, + ) + ) + + async def update(progress, count): + await self.clock.sleep((count * duration_ms) / 1000) + progress = {"my_key": progress["my_key"] + 1} + await store.db_pool.runInteraction( + "update_progress", + self.updates._background_update_progress_txn, + "test_update", + progress, + ) + return count + + self.update_handler.side_effect = update + self.update_handler.reset_mock() + res = self.get_success( + self.updates.do_next_background_update(False), + by=0.02, + ) + self.assertFalse(res) + + # the first update was run with the default batch size, this should be run with 500ms as the + # desired duration + async def update(progress, count): + self.assertEqual(progress, {"my_key": 2}) + self.assertAlmostEqual( + count, + 500 / duration_ms, + places=0, + ) + await self.updates._end_background_update("test_update") + return count + + self.update_handler.side_effect = update + self.get_success(self.updates.do_next_background_update(False)) + + @override_config( + yaml.safe_load( + """ + min_batch_size: 5 + """ ) + ) + def test_background_update_min_batch_set_in_config(self): + """ + Test that the minimum batch size set in the config is used + """ + # a very long-running individual update + duration_ms = 50 + + store = self.hs.get_datastores().main + self.get_success( + store.db_pool.simple_insert( + "background_updates", + values={"update_name": "test_update", "progress_json": '{"my_key": 1}'}, + ) + ) + + async def update(progress, count): + await self.clock.sleep((count * duration_ms) / 1000) + progress = {"my_key": progress["my_key"] + 1} + await store.db_pool.runInteraction( + "update_progress", + self.updates._background_update_progress_txn, + "test_update", + progress, + ) + return count + + self.update_handler.side_effect = update + self.update_handler.reset_mock() + res = self.get_success( + self.updates.do_next_background_update(False), + by=1, + ) + self.assertFalse(res) + + # the first update was run with the default batch size, this should be run with minimum batch size + # as the individual updates took a very long time + async def update(progress, count): + self.assertEqual(progress, {"my_key": 2}) + self.assertEqual(count, 5) + await self.updates._end_background_update("test_update") + return count + + self.update_handler.side_effect = update + self.get_success(self.updates.do_next_background_update(False)) + class BackgroundUpdateControllerTestCase(unittest.HomeserverTestCase): def prepare(self, reactor, clock, homeserver): From 91fea81f430855323eb0f0d6ab419d0d7ce20c1c Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Mon, 7 Mar 2022 15:09:40 -0800 Subject: [PATCH 12/19] fix refactored test --- tests/rest/admin/test_background_updates.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/rest/admin/test_background_updates.py b/tests/rest/admin/test_background_updates.py index c3153eb46413..6cf56b1e352f 100644 --- a/tests/rest/admin/test_background_updates.py +++ b/tests/rest/admin/test_background_updates.py @@ -136,7 +136,6 @@ def test_status_bg_update(self) -> None: """Test the status API works with a background update.""" # Create a new background update - self._register_bg_update() self.store.db_pool.updates.start_doing_background_updates() @@ -160,7 +159,7 @@ def test_status_bg_update(self) -> None: "average_items_per_ms": 0.1, "total_duration_ms": 1000.0, "total_item_count": ( - BackgroundUpdater.default_background_batch_size + self.updater.default_background_batch_size ), } }, @@ -215,7 +214,7 @@ def test_enabled(self) -> None: "average_items_per_ms": 0.1, "total_duration_ms": 1000.0, "total_item_count": ( - BackgroundUpdater.default_background_batch_size + self.updater.default_background_batch_size ), } }, @@ -244,7 +243,7 @@ def test_enabled(self) -> None: "average_items_per_ms": 0.1, "total_duration_ms": 1000.0, "total_item_count": ( - BackgroundUpdater.default_background_batch_size + self.updater.default_background_batch_size ), } }, From 686c73a8880654ba5c601a4af9b26ea08e3f7b8c Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Mon, 7 Mar 2022 16:01:10 -0800 Subject: [PATCH 13/19] refactor repetetive tests --- tests/storage/test_background_update.py | 131 ++++++------------------ 1 file changed, 31 insertions(+), 100 deletions(-) diff --git a/tests/storage/test_background_update.py b/tests/storage/test_background_update.py index 7eb17f4826df..cda5d15c687c 100644 --- a/tests/storage/test_background_update.py +++ b/tests/storage/test_background_update.py @@ -37,6 +37,19 @@ def prepare(self, reactor, clock, homeserver): self.updates.register_background_update_handler( "test_update", self.update_handler ) + self.store = self.hs.get_datastores().main + + async def update(self, progress, count): + duration_ms = 10 + await self.clock.sleep((count * duration_ms) / 1000) + progress = {"my_key": progress["my_key"] + 1} + await self.store.db_pool.runInteraction( + "update_progress", + self.updates._background_update_progress_txn, + "test_update", + progress, + ) + return count def test_do_background_update(self): # the time we claim it takes to update one item when running the update @@ -45,27 +58,14 @@ def test_do_background_update(self): # the target runtime for each bg update target_background_update_duration_ms = 100 - store = self.hs.get_datastores().main self.get_success( - store.db_pool.simple_insert( + self.store.db_pool.simple_insert( "background_updates", values={"update_name": "test_update", "progress_json": '{"my_key": 1}'}, ) ) - # first step: make a bit of progress - async def update(progress, count): - await self.clock.sleep((count * duration_ms) / 1000) - progress = {"my_key": progress["my_key"] + 1} - await store.db_pool.runInteraction( - "update_progress", - self.updates._background_update_progress_txn, - "test_update", - progress, - ) - return count - - self.update_handler.side_effect = update + self.update_handler.side_effect = self.update self.update_handler.reset_mock() res = self.get_success( self.updates.do_next_background_update(False), @@ -109,33 +109,19 @@ async def update(progress, count): """ ) ) - def test_do_background_update_default_batch_set_by_config(self): + def test_background_update_default_batch_set_by_config(self): """ Test that the background update is run with the default_batch_size set by the config """ - # the time we claim it takes to update one item when running the update - duration_ms = 10 - store = self.hs.get_datastores().main self.get_success( - store.db_pool.simple_insert( + self.store.db_pool.simple_insert( "background_updates", values={"update_name": "test_update", "progress_json": '{"my_key": 1}'}, ) ) - async def update(progress, count): - await self.clock.sleep((count * duration_ms) / 1000) - progress = {"my_key": progress["my_key"] + 1} - await store.db_pool.runInteraction( - "update_progress", - self.updates._background_update_progress_txn, - "test_update", - progress, - ) - return count - - self.update_handler.side_effect = update + self.update_handler.side_effect = self.update self.update_handler.reset_mock() res = self.get_success( self.updates.do_next_background_update(False), @@ -151,29 +137,14 @@ def test_background_update_default_sleep_behavior(self): Test default background update behavior, which is to sleep """ - # the time we claim it takes to update one item when running the update - duration_ms = 10 - - store = self.hs.get_datastores().main self.get_success( - store.db_pool.simple_insert( + self.store.db_pool.simple_insert( "background_updates", values={"update_name": "test_update", "progress_json": '{"my_key": 1}'}, ) ) - async def update(progress, count): - await self.clock.sleep((count * duration_ms) / 1000) - progress = {"my_key": progress["my_key"] + 1} - await store.db_pool.runInteraction( - "update_progress", - self.updates._background_update_progress_txn, - "test_update", - progress, - ) - return count - - self.update_handler.side_effect = update + self.update_handler.side_effect = self.update self.update_handler.reset_mock() self.updates.start_doing_background_updates(), @@ -199,28 +170,14 @@ def test_background_update_sleep_set_in_config(self): Test that changing the sleep time in the config changes how long it sleeps """ - duration_ms = 10 - - store = self.hs.get_datastores().main self.get_success( - store.db_pool.simple_insert( + self.store.db_pool.simple_insert( "background_updates", values={"update_name": "test_update", "progress_json": '{"my_key": 1}'}, ) ) - async def update(progress, count): - await self.clock.sleep((count * duration_ms) / 1000) - progress = {"my_key": progress["my_key"] + 1} - await store.db_pool.runInteraction( - "update_progress", - self.updates._background_update_progress_txn, - "test_update", - progress, - ) - return count - - self.update_handler.side_effect = update + self.update_handler.side_effect = self.update self.update_handler.reset_mock() self.updates.start_doing_background_updates(), @@ -245,29 +202,15 @@ def test_disabling_background_update_sleep(self): """ Test that disabling sleep in the config results in bg update not sleeping """ - # the time we claim it takes to update one item when running the update - duration_ms = 10 - store = self.hs.get_datastores().main self.get_success( - store.db_pool.simple_insert( + self.store.db_pool.simple_insert( "background_updates", values={"update_name": "test_update", "progress_json": '{"my_key": 1}'}, ) ) - async def update(progress, count): - await self.clock.sleep((count * duration_ms) / 1000) - progress = {"my_key": progress["my_key"] + 1} - await store.db_pool.runInteraction( - "update_progress", - self.updates._background_update_progress_txn, - "test_update", - progress, - ) - return count - - self.update_handler.side_effect = update + self.update_handler.side_effect = self.update self.update_handler.reset_mock() self.updates.start_doing_background_updates(), @@ -287,29 +230,17 @@ def test_background_update_duration_set_in_config(self): """ Test that the desired duration set in the config is used in determining batch size """ - # the time we claim it takes to update one item when running the update + # Duration of one background update item duration_ms = 10 - store = self.hs.get_datastores().main self.get_success( - store.db_pool.simple_insert( + self.store.db_pool.simple_insert( "background_updates", values={"update_name": "test_update", "progress_json": '{"my_key": 1}'}, ) ) - async def update(progress, count): - await self.clock.sleep((count * duration_ms) / 1000) - progress = {"my_key": progress["my_key"] + 1} - await store.db_pool.runInteraction( - "update_progress", - self.updates._background_update_progress_txn, - "test_update", - progress, - ) - return count - - self.update_handler.side_effect = update + self.update_handler.side_effect = self.update self.update_handler.reset_mock() res = self.get_success( self.updates.do_next_background_update(False), @@ -346,18 +277,18 @@ def test_background_update_min_batch_set_in_config(self): # a very long-running individual update duration_ms = 50 - store = self.hs.get_datastores().main self.get_success( - store.db_pool.simple_insert( + self.store.db_pool.simple_insert( "background_updates", values={"update_name": "test_update", "progress_json": '{"my_key": 1}'}, ) ) + # Run the update with the long-running update item async def update(progress, count): await self.clock.sleep((count * duration_ms) / 1000) progress = {"my_key": progress["my_key"] + 1} - await store.db_pool.runInteraction( + await self.store.db_pool.runInteraction( "update_progress", self.updates._background_update_progress_txn, "test_update", @@ -374,7 +305,7 @@ async def update(progress, count): self.assertFalse(res) # the first update was run with the default batch size, this should be run with minimum batch size - # as the individual updates took a very long time + # as the first items took a very long time async def update(progress, count): self.assertEqual(progress, {"my_key": 2}) self.assertEqual(count, 5) From 581764d684a6fbb58dc0b4ea2e82daf7b76a9160 Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Thu, 10 Mar 2022 08:36:40 -0800 Subject: [PATCH 14/19] group background update options under a key --- docs/sample_config.yaml | 39 ++++++++-------- synapse/config/background_updates.py | 61 +++++++++++++------------ tests/config/test_background_update.py | 11 +++-- tests/storage/test_background_update.py | 15 ++++-- 4 files changed, 69 insertions(+), 57 deletions(-) diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 2aa49cff167b..38be1a4b888b 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -2739,26 +2739,27 @@ redis: # The duration, minimum batch size, default batch size, whether to sleep between batches and if so, how long to # sleep can all be configured. This is helpful to speed up or slow down the updates. -# How long in milliseconds to run a batch of background updates for. Defaults to 100. Uncomment and set -# a time to change the default. -# -#background_update_duration_ms: 500 +background_updates: + # How long in milliseconds to run a batch of background updates for. Defaults to 100. Uncomment and set + # a time to change the default. + # + #background_update_duration_ms: 500 -# Whether to sleep between updates. Defaults to True. Uncomment to change the default. -# -#sleep_enabled: false + # Whether to sleep between updates. Defaults to True. Uncomment to change the default. + # + #sleep_enabled: false -# If sleeping between updates, how long in milliseconds to sleep for. Defaults to 1000. Uncomment -# and set a duration to change the default. -# -#sleep_duration_ms: 300 + # If sleeping between updates, how long in milliseconds to sleep for. Defaults to 1000. Uncomment + # and set a duration to change the default. + # + #sleep_duration_ms: 300 -# Minimum size a batch of background updates can be. Must be greater than 0. Defaults to 1. Uncomment and -# set a size to change the default. -# -#min_batch_size: 10 + # Minimum size a batch of background updates can be. Must be greater than 0. Defaults to 1. Uncomment and + # set a size to change the default. + # + #min_batch_size: 10 -# The batch size to use for the first iteration of a new background update. The default is 100. -# Uncomment and set a size to change the default. -# -#default_batch_size: 50 + # The batch size to use for the first iteration of a new background update. The default is 100. + # Uncomment and set a size to change the default. + # + #default_batch_size: 50 diff --git a/synapse/config/background_updates.py b/synapse/config/background_updates.py index fe39da329086..415a4c79f901 100644 --- a/synapse/config/background_updates.py +++ b/synapse/config/background_updates.py @@ -27,39 +27,44 @@ def generate_config_section(self, **kwargs) -> str: # Background updates are database updates that are run in the background in batches. # The duration, minimum batch size, default batch size, whether to sleep between batches and if so, how long to # sleep can all be configured. This is helpful to speed up or slow down the updates. - - # How long in milliseconds to run a batch of background updates for. Defaults to 100. Uncomment and set - # a time to change the default. - # - #background_update_duration_ms: 500 - - # Whether to sleep between updates. Defaults to True. Uncomment to change the default. - # - #sleep_enabled: false - - # If sleeping between updates, how long in milliseconds to sleep for. Defaults to 1000. Uncomment - # and set a duration to change the default. # - #sleep_duration_ms: 300 - - # Minimum size a batch of background updates can be. Must be greater than 0. Defaults to 1. Uncomment and - # set a size to change the default. - # - #min_batch_size: 10 - - # The batch size to use for the first iteration of a new background update. The default is 100. - # Uncomment and set a size to change the default. - # - #default_batch_size: 50 + background_updates: + # How long in milliseconds to run a batch of background updates for. Defaults to 100. Uncomment and set + # a time to change the default. + # + #background_update_duration_ms: 500 + + # Whether to sleep between updates. Defaults to True. Uncomment to change the default. + # + #sleep_enabled: false + + # If sleeping between updates, how long in milliseconds to sleep for. Defaults to 1000. Uncomment + # and set a duration to change the default. + # + #sleep_duration_ms: 300 + + # Minimum size a batch of background updates can be. Must be greater than 0. Defaults to 1. Uncomment and + # set a size to change the default. + # + #min_batch_size: 10 + + # The batch size to use for the first iteration of a new background update. The default is 100. + # Uncomment and set a size to change the default. + # + #default_batch_size: 50 """ def read_config(self, config, **kwargs) -> None: - self.update_duration_ms = config.get("background_update_duration_ms", 100) + bg_update_config = config.get("background_updates", {}) + + self.update_duration_ms = bg_update_config.get( + "background_update_duration_ms", 100 + ) - self.sleep_enabled = config.get("sleep_enabled", True) + self.sleep_enabled = bg_update_config.get("sleep_enabled", True) - self.sleep_duration_ms = config.get("sleep_duration_ms", 1000) + self.sleep_duration_ms = bg_update_config.get("sleep_duration_ms", 1000) - self.min_batch_size = config.get("min_batch_size", 1) + self.min_batch_size = bg_update_config.get("min_batch_size", 1) - self.default_batch_size = config.get("default_batch_size", 100) + self.default_batch_size = bg_update_config.get("default_batch_size", 100) diff --git a/tests/config/test_background_update.py b/tests/config/test_background_update.py index b44c8c9a989a..f44383160c05 100644 --- a/tests/config/test_background_update.py +++ b/tests/config/test_background_update.py @@ -37,11 +37,12 @@ def test_default_configuration(self): @override_config( yaml.safe_load( """ - background_update_duration_ms: 1000 - sleep_enabled: false - sleep_duration_ms: 600 - min_batch_size: 5 - default_batch_size: 50 + background_updates: + background_update_duration_ms: 1000 + sleep_enabled: false + sleep_duration_ms: 600 + min_batch_size: 5 + default_batch_size: 50 """ ) ) diff --git a/tests/storage/test_background_update.py b/tests/storage/test_background_update.py index cda5d15c687c..5cf18b690e48 100644 --- a/tests/storage/test_background_update.py +++ b/tests/storage/test_background_update.py @@ -105,7 +105,8 @@ async def update(progress, count): @override_config( yaml.safe_load( """ - default_batch_size: 20 + background_updates: + default_batch_size: 20 """ ) ) @@ -161,7 +162,8 @@ def test_background_update_default_sleep_behavior(self): @override_config( yaml.safe_load( """ - sleep_duration_ms: 500 + background_updates: + sleep_duration_ms: 500 """ ) ) @@ -194,7 +196,8 @@ def test_background_update_sleep_set_in_config(self): @override_config( yaml.safe_load( """ - sleep_enabled: false + background_updates: + sleep_enabled: false """ ) ) @@ -222,7 +225,8 @@ def test_disabling_background_update_sleep(self): @override_config( yaml.safe_load( """ - background_update_duration_ms: 500 + background_updates: + background_update_duration_ms: 500 """ ) ) @@ -266,7 +270,8 @@ async def update(progress, count): @override_config( yaml.safe_load( """ - min_batch_size: 5 + background_updates: + min_batch_size: 5 """ ) ) From 5947111ed6bc47a3f528ec84537372fa561222b0 Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Thu, 10 Mar 2022 08:40:21 -0800 Subject: [PATCH 15/19] regenerate sample config --- docs/sample_config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 38be1a4b888b..dd4e86b0b9ad 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -2738,8 +2738,8 @@ redis: # Background updates are database updates that are run in the background in batches. # The duration, minimum batch size, default batch size, whether to sleep between batches and if so, how long to # sleep can all be configured. This is helpful to speed up or slow down the updates. - -background_updates: +# +background_updates: # How long in milliseconds to run a batch of background updates for. Defaults to 100. Uncomment and set # a time to change the default. # From 66bc0a023b73dcb6033462ac0c0515785d7e3fcc Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Thu, 10 Mar 2022 09:25:44 -0800 Subject: [PATCH 16/19] fix get returning None --- synapse/config/background_updates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/config/background_updates.py b/synapse/config/background_updates.py index 415a4c79f901..b7f177cd2a15 100644 --- a/synapse/config/background_updates.py +++ b/synapse/config/background_updates.py @@ -55,7 +55,7 @@ def generate_config_section(self, **kwargs) -> str: """ def read_config(self, config, **kwargs) -> None: - bg_update_config = config.get("background_updates", {}) + bg_update_config = config.get("background_updates") or {} self.update_duration_ms = bg_update_config.get( "background_update_duration_ms", 100 From b98dec5ea6dd3e4b19bdc8b9842fee7354d5d5d6 Mon Sep 17 00:00:00 2001 From: Shay Date: Fri, 11 Mar 2022 10:04:43 -0800 Subject: [PATCH 17/19] Update synapse/config/background_updates.py Co-authored-by: Erik Johnston --- synapse/config/background_updates.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/config/background_updates.py b/synapse/config/background_updates.py index b7f177cd2a15..f6cdeacc4b19 100644 --- a/synapse/config/background_updates.py +++ b/synapse/config/background_updates.py @@ -20,8 +20,6 @@ class BackgroundUpdateConfig(Config): def generate_config_section(self, **kwargs) -> str: return """\ - - ## Background Updates ## # Background updates are database updates that are run in the background in batches. From c8913865701345f09bafaae8cac0c7e16c86fa35 Mon Sep 17 00:00:00 2001 From: Shay Date: Fri, 11 Mar 2022 10:04:58 -0800 Subject: [PATCH 18/19] Update tests/config/test_background_update.py Co-authored-by: Erik Johnston --- tests/config/test_background_update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/config/test_background_update.py b/tests/config/test_background_update.py index f44383160c05..0c32c1ca299e 100644 --- a/tests/config/test_background_update.py +++ b/tests/config/test_background_update.py @@ -18,7 +18,7 @@ from tests.unittest import HomeserverTestCase, override_config -class RoomDirectoryConfigTestCase(HomeserverTestCase): +class BackgroundUpdateConfigTestCase(HomeserverTestCase): # Tests that the default values in the config are correctly loaded. Note that the default # values are loaded when the corresponding config options are commented out, which is why there isn't # a config specified here. From bcb04ad62263ca7cc1ebccc5f21504315c48b8fb Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Fri, 11 Mar 2022 10:10:55 -0800 Subject: [PATCH 19/19] regnerate sample config --- docs/sample_config.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index dd4e86b0b9ad..605ac2b7f8ce 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -2731,8 +2731,6 @@ redis: #password: - - ## Background Updates ## # Background updates are database updates that are run in the background in batches.