From be29ccb6d03dbd1151d9220a7b260bb3975dff89 Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Wed, 1 Mar 2023 14:48:50 -0800 Subject: [PATCH 01/11] add function to delete backup keys when deactiving account --- synapse/handlers/deactivate_account.py | 3 +++ .../storage/databases/main/e2e_room_keys.py | 26 +++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index d24f64938219..1fe9ff888a6e 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -174,6 +174,9 @@ async def deactivate_account( # Remove account data (including ignored users and push rules). await self.store.purge_account_data_for_user(user_id) + # Delete any server-side backup keys + await self.store.bulk_delete_backup_keys_and_versions_for_user(user_id) + # Let modules know the user has been deactivated. await self._third_party_rules.on_user_deactivation_status_changed( user_id, diff --git a/synapse/storage/databases/main/e2e_room_keys.py b/synapse/storage/databases/main/e2e_room_keys.py index 6240f9a75ed3..49a8b261b5fe 100644 --- a/synapse/storage/databases/main/e2e_room_keys.py +++ b/synapse/storage/databases/main/e2e_room_keys.py @@ -550,3 +550,29 @@ def _delete_e2e_room_keys_version_txn(txn: LoggingTransaction) -> None: await self.db_pool.runInteraction( "delete_e2e_room_keys_version", _delete_e2e_room_keys_version_txn ) + + async def bulk_delete_backup_keys_and_versions_for_user(self, user_id: str) -> None: + """ + Bulk deletes all backup room keys and versions for a given user. + + Args: + user_id: the user whose backup keys and versions we're deleting + """ + + def _delete_all_e2e_room_keys_and_versions_txn(txn: LoggingTransaction) -> None: + self.db_pool.simple_delete_txn( + txn, + table="e2e_room_keys", + keyvalues={"user_id": user_id}, + ) + + self.db_pool.simple_delete_txn( + txn, + table="e2e_room_keys_versions", + keyvalues={"user_id": user_id}, + ) + + await self.db_pool.runInteraction( + "delete_all_e2e_room_keys_and_versions", + _delete_all_e2e_room_keys_and_versions_txn, + ) From fa6c88e8d5d0133c74031e6ca943583a80bc5f25 Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Wed, 1 Mar 2023 14:48:54 -0800 Subject: [PATCH 02/11] add test --- tests/rest/client/test_account.py | 66 +++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/tests/rest/client/test_account.py b/tests/rest/client/test_account.py index e2ee1a1766cd..2f05de306e7a 100644 --- a/tests/rest/client/test_account.py +++ b/tests/rest/client/test_account.py @@ -476,6 +476,72 @@ def test_pending_invites(self) -> None: self.assertEqual(len(memberships), 1, memberships) self.assertEqual(memberships[0].room_id, room_id, memberships) + def test_deactivate_account_deletes_server_side_backup_keys(self) -> None: + key_handler = self.hs.get_e2e_room_keys_handler() + room_keys = { + "rooms": { + "!abc:matrix.org": { + "sessions": { + "c0ff33": { + "first_message_index": 1, + "forwarded_count": 1, + "is_verified": False, + "session_data": "SSBBTSBBIEZJU0gK", + } + } + } + } + } + + user_id = self.register_user("missPiggy", "test") + tok = self.login("missPiggy", "test") + + # add some backup keys/versions + version = self.get_success( + key_handler.create_version( + user_id, + { + "algorithm": "m.megolm_backup.v1", + "auth_data": "first_version_auth_data", + }, + ) + ) + + self.get_success(key_handler.upload_room_keys(user_id, version, room_keys)) + + version2 = self.get_success( + key_handler.create_version( + user_id, + { + "algorithm": "m.megolm_backup.v1", + "auth_data": "second_version_auth_data", + }, + ) + ) + + self.get_success(key_handler.upload_room_keys(user_id, version2, room_keys)) + + self.deactivate(user_id, tok) + store = self.hs.get_datastores().main + + # Check that the user has been marked as deactivated. + self.assertTrue(self.get_success(store.get_user_deactivated_status(user_id))) + + # Check that there are no entries in 'e2e_room_keys` and `e2e_room_keys_versions` + res = self.get_success( + self.hs.get_datastores().main.db_pool.simple_select_list( + "e2e_room_keys", {"user_id": user_id}, "*", "simple_select" + ) + ) + self.assertEqual(len(res), 0) + + res2 = self.get_success( + self.hs.get_datastores().main.db_pool.simple_select_list( + "e2e_room_keys_versions", {"user_id": user_id}, "*", "simple_select" + ) + ) + self.assertEqual(len(res2), 0) + def deactivate(self, user_id: str, tok: str) -> None: request_data = { "auth": { From 39fd7889ea388ba25d5cdfbae03c70eed0d9ac66 Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Wed, 1 Mar 2023 14:49:01 -0800 Subject: [PATCH 03/11] newsfragment --- changelog.d/15181.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/15181.bugfix diff --git a/changelog.d/15181.bugfix b/changelog.d/15181.bugfix new file mode 100644 index 000000000000..191bb6f611a7 --- /dev/null +++ b/changelog.d/15181.bugfix @@ -0,0 +1 @@ +Delete server-side backup keys when deactivating an account. \ No newline at end of file From 7be3ca6fd6a6e77f4027770cc0333dc29c613a92 Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Wed, 15 Mar 2023 13:51:43 -0700 Subject: [PATCH 04/11] add background update to delete backup keys for previously deactivated users --- .../storage/databases/main/e2e_room_keys.py | 88 +++++++++++++++++- ..._e2e_backup_keys_for_deactivated_users.sql | 17 ++++ tests/rest/client/test_account.py | 92 +++++++++++++++++++ 3 files changed, 194 insertions(+), 3 deletions(-) create mode 100644 synapse/storage/schema/main/delta/73/26_delete_e2e_backup_keys_for_deactivated_users.sql diff --git a/synapse/storage/databases/main/e2e_room_keys.py b/synapse/storage/databases/main/e2e_room_keys.py index 49a8b261b5fe..f29e074b72da 100644 --- a/synapse/storage/databases/main/e2e_room_keys.py +++ b/synapse/storage/databases/main/e2e_room_keys.py @@ -13,17 +13,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Dict, Iterable, Mapping, Optional, Tuple, cast +from typing import TYPE_CHECKING, Dict, Iterable, Mapping, Optional, Tuple, cast from typing_extensions import Literal, TypedDict from synapse.api.errors import StoreError from synapse.logging.opentracing import log_kv, trace from synapse.storage._base import SQLBaseStore, db_to_json -from synapse.storage.database import LoggingTransaction +from synapse.storage.database import ( + DatabasePool, + LoggingDatabaseConnection, + LoggingTransaction, +) from synapse.types import JsonDict, JsonSerializable, StreamKeyType from synapse.util import json_encoder +if TYPE_CHECKING: + from synapse.server import HomeServer + class RoomKey(TypedDict): """`KeyBackupData` in the Matrix spec. @@ -37,7 +44,82 @@ class RoomKey(TypedDict): session_data: JsonSerializable -class EndToEndRoomKeyStore(SQLBaseStore): +class EndToEndRoomKeyBackgroundStore(SQLBaseStore): + def __init__( + self, + database: DatabasePool, + db_conn: LoggingDatabaseConnection, + hs: "HomeServer", + ): + super().__init__(database, db_conn, hs) + + self.db_pool.updates.register_background_update_handler( + "delete_e2e_backup_keys_for_deactivated_users", + self._delete_e2e_backup_keys_for_deactivated_users, + ) + + def _delete_keys_txn(self, txn: LoggingTransaction, user_id: str) -> None: + self.db_pool.simple_delete_txn( + txn, + table="e2e_room_keys", + keyvalues={"user_id": user_id}, + ) + + self.db_pool.simple_delete_txn( + txn, + table="e2e_room_keys_versions", + keyvalues={"user_id": user_id}, + ) + + async def _delete_e2e_backup_keys_for_deactivated_users( + self, progress: JsonDict, batch_size: int + ) -> int: + """ + Retroactively purges account data for users that have already been deactivated. + Gets run as a background update caused by a schema delta. + """ + + last_user: str = progress.get("last_user", "") + + def _delete_backup_keys_for_deactivated_users_txn( + txn: LoggingTransaction, + ) -> int: + sql = """ + SELECT name FROM users + WHERE deactivated = ? and name > ? + ORDER BY name ASC + LIMIT ? + """ + + txn.execute(sql, (1, last_user, batch_size)) + users = [row[0] for row in txn] + + for user in users: + self._delete_keys_txn(txn, user) + + if users: + self.db_pool.updates._background_update_progress_txn( + txn, + "delete_e2e_backup_keys_for_deactivated_users", + {"last_user": users[-1]}, + ) + + return len(users) + + number_deleted = await self.db_pool.runInteraction( + "_delete_backup_keys_for_deactivated_users", + _delete_backup_keys_for_deactivated_users_txn, + ) + + if number_deleted < batch_size: + await self.db_pool.updates._end_background_update( + "delete_e2e_backup_keys_for_deactivated_users" + ) + + return number_deleted + + +class EndToEndRoomKeyStore(EndToEndRoomKeyBackgroundStore): """The store for end to end room key backups. See https://spec.matrix.org/v1.1/client-server-api/#server-side-key-backups diff --git a/synapse/storage/schema/main/delta/73/26_delete_e2e_backup_keys_for_deactivated_users.sql b/synapse/storage/schema/main/delta/73/26_delete_e2e_backup_keys_for_deactivated_users.sql new file mode 100644 index 000000000000..3181001a47b6 --- /dev/null +++ b/synapse/storage/schema/main/delta/73/26_delete_e2e_backup_keys_for_deactivated_users.sql @@ -0,0 +1,17 @@ +/* Copyright 2023 The Matrix.org Foundation C.I.C + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (7324, 'delete_e2e_backup_keys_for_deactivated_users', '{}'); \ No newline at end of file diff --git a/tests/rest/client/test_account.py b/tests/rest/client/test_account.py index 2f05de306e7a..f56236a14a01 100644 --- a/tests/rest/client/test_account.py +++ b/tests/rest/client/test_account.py @@ -542,6 +542,98 @@ def test_deactivate_account_deletes_server_side_backup_keys(self) -> None: ) self.assertEqual(len(res2), 0) + def test_background_update_deletes_deactivated_users_server_side_backup_keys( + self, + ) -> None: + + key_handler = self.hs.get_e2e_room_keys_handler() + room_keys = { + "rooms": { + "!abc:matrix.org": { + "sessions": { + "c0ff33": { + "first_message_index": 1, + "forwarded_count": 1, + "is_verified": False, + "session_data": "SSBBTSBBIEZJU0gK", + } + } + } + } + } + self.store = self.hs.get_datastores().main + + # create a bunch of users and add keys for them + users = [] + for i in range(0, 20): + user_id = self.register_user("missPiggy" + str(i), "test") + users.append((user_id,)) + + # add some backup keys/versions + version = self.get_success( + key_handler.create_version( + user_id, + { + "algorithm": "m.megolm_backup.v1", + "auth_data": str(i) + "_version_auth_data", + }, + ) + ) + + self.get_success(key_handler.upload_room_keys(user_id, version, room_keys)) + + version2 = self.get_success( + key_handler.create_version( + user_id, + { + "algorithm": "m.megolm_backup.v1", + "auth_data": str(i) + "_version_auth_data", + }, + ) + ) + + self.get_success(key_handler.upload_room_keys(user_id, version2, room_keys)) + + # deactivate most of the users by editing DB + self.get_success( + self.store.db_pool.simple_update_many( + table="users", + key_names=("name",), + key_values=users[0:18], + value_names=("deactivated",), + value_values=[(1,) for i in range(1, 19)], + desc="", + ) + ) + + # run background update + self.get_success( + self.store.db_pool.simple_insert( + "background_updates", + { + "update_name": "delete_e2e_backup_keys_for_deactivated_users", + "progress_json": "{}", + }, + ) + ) + self.store.db_pool.updates._all_done = False + self.wait_for_background_updates() + + # check that keys are deleted for the deactivated users but not the others + res = self.get_success( + self.hs.get_datastores().main.db_pool.simple_select_list( + "e2e_room_keys", None, ("user_id",), "simple_select" + ) + ) + self.assertEqual(len(res), 4) + + res2 = self.get_success( + self.hs.get_datastores().main.db_pool.simple_select_list( + "e2e_room_keys_versions", None, ("user_id",), "simple_select" + ) + ) + self.assertEqual(len(res2), 4) + def deactivate(self, user_id: str, tok: str) -> None: request_data = { "auth": { From f15dec9ccc9c9a718b1c56544dafb3fea1a45bbc Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Wed, 15 Mar 2023 14:46:42 -0700 Subject: [PATCH 05/11] update schema delta file --- .../01_delete_e2e_backup_keys_for_deactivated_users.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename synapse/storage/schema/main/delta/{73/26_delete_e2e_backup_keys_for_deactivated_users.sql => 74/01_delete_e2e_backup_keys_for_deactivated_users.sql} (100%) diff --git a/synapse/storage/schema/main/delta/73/26_delete_e2e_backup_keys_for_deactivated_users.sql b/synapse/storage/schema/main/delta/74/01_delete_e2e_backup_keys_for_deactivated_users.sql similarity index 100% rename from synapse/storage/schema/main/delta/73/26_delete_e2e_backup_keys_for_deactivated_users.sql rename to synapse/storage/schema/main/delta/74/01_delete_e2e_backup_keys_for_deactivated_users.sql From b2a07ce3308f05732fe59a88d31417c8f016101e Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Wed, 15 Mar 2023 14:51:52 -0700 Subject: [PATCH 06/11] black --- tests/rest/client/test_account.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/rest/client/test_account.py b/tests/rest/client/test_account.py index 26ff5310fd61..fe81b6c726d8 100644 --- a/tests/rest/client/test_account.py +++ b/tests/rest/client/test_account.py @@ -543,7 +543,6 @@ def test_deactivate_account_deletes_server_side_backup_keys(self) -> None: def test_background_update_deletes_deactivated_users_server_side_backup_keys( self, ) -> None: - key_handler = self.hs.get_e2e_room_keys_handler() room_keys = { "rooms": { From 5856dbd68b04d5a1172538b7189b7cf343d61a60 Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Wed, 15 Mar 2023 20:51:27 -0700 Subject: [PATCH 07/11] add e2e_room_keys to script? --- synapse/_scripts/synapse_port_db.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/_scripts/synapse_port_db.py b/synapse/_scripts/synapse_port_db.py index 2c9cbf8b275b..0fbc5371361d 100755 --- a/synapse/_scripts/synapse_port_db.py +++ b/synapse/_scripts/synapse_port_db.py @@ -58,6 +58,7 @@ from synapse.storage.databases.main.client_ips import ClientIpBackgroundUpdateStore from synapse.storage.databases.main.deviceinbox import DeviceInboxBackgroundUpdateStore from synapse.storage.databases.main.devices import DeviceBackgroundUpdateStore +from synapse.storage.databases.main.e2e_room_keys import EndToEndRoomKeyBackgroundStore from synapse.storage.databases.main.end_to_end_keys import EndToEndKeyBackgroundStore from synapse.storage.databases.main.event_push_actions import EventPushActionsStore from synapse.storage.databases.main.events_bg_updates import ( @@ -221,6 +222,7 @@ class Store( MainStateBackgroundUpdateStore, UserDirectoryBackgroundUpdateStore, EndToEndKeyBackgroundStore, + EndToEndRoomKeyBackgroundStore, StatsStore, AccountDataWorkerStore, PushRuleStore, From a902c772b6ac572e7deaa730f35b062cef7bc4ca Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Mon, 20 Mar 2023 10:08:52 -0700 Subject: [PATCH 08/11] rename background update and update ordering --- ....sql => 02_delete_e2e_backup_keys_for_deactivated_users.sql} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename synapse/storage/schema/main/delta/74/{01_delete_e2e_backup_keys_for_deactivated_users.sql => 02_delete_e2e_backup_keys_for_deactivated_users.sql} (91%) diff --git a/synapse/storage/schema/main/delta/74/01_delete_e2e_backup_keys_for_deactivated_users.sql b/synapse/storage/schema/main/delta/74/02_delete_e2e_backup_keys_for_deactivated_users.sql similarity index 91% rename from synapse/storage/schema/main/delta/74/01_delete_e2e_backup_keys_for_deactivated_users.sql rename to synapse/storage/schema/main/delta/74/02_delete_e2e_backup_keys_for_deactivated_users.sql index 3181001a47b6..fbde79eace84 100644 --- a/synapse/storage/schema/main/delta/74/01_delete_e2e_backup_keys_for_deactivated_users.sql +++ b/synapse/storage/schema/main/delta/74/02_delete_e2e_backup_keys_for_deactivated_users.sql @@ -14,4 +14,4 @@ */ INSERT INTO background_updates (ordering, update_name, progress_json) VALUES - (7324, 'delete_e2e_backup_keys_for_deactivated_users', '{}'); \ No newline at end of file + (7402, 'delete_e2e_backup_keys_for_deactivated_users', '{}'); \ No newline at end of file From b265bd33af9e901d07524c9a9133b8fa5d5fe156 Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Tue, 28 Mar 2023 14:20:48 -0700 Subject: [PATCH 09/11] fix schema drift --- ....sql => 03_delete_e2e_backup_keys_for_deactivated_users.sql} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename synapse/storage/schema/main/delta/74/{02_delete_e2e_backup_keys_for_deactivated_users.sql => 03_delete_e2e_backup_keys_for_deactivated_users.sql} (91%) diff --git a/synapse/storage/schema/main/delta/74/02_delete_e2e_backup_keys_for_deactivated_users.sql b/synapse/storage/schema/main/delta/74/03_delete_e2e_backup_keys_for_deactivated_users.sql similarity index 91% rename from synapse/storage/schema/main/delta/74/02_delete_e2e_backup_keys_for_deactivated_users.sql rename to synapse/storage/schema/main/delta/74/03_delete_e2e_backup_keys_for_deactivated_users.sql index fbde79eace84..8d713140feec 100644 --- a/synapse/storage/schema/main/delta/74/02_delete_e2e_backup_keys_for_deactivated_users.sql +++ b/synapse/storage/schema/main/delta/74/03_delete_e2e_backup_keys_for_deactivated_users.sql @@ -14,4 +14,4 @@ */ INSERT INTO background_updates (ordering, update_name, progress_json) VALUES - (7402, 'delete_e2e_backup_keys_for_deactivated_users', '{}'); \ No newline at end of file + (7403, 'delete_e2e_backup_keys_for_deactivated_users', '{}'); \ No newline at end of file From 86f4c053a13c0196226548d82f0eee1442d2e404 Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Tue, 28 Mar 2023 14:26:31 -0700 Subject: [PATCH 10/11] fix even more drift --- .../01_delete_e2e_backup_keys_for_deactivated_users.sql} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename synapse/storage/schema/main/delta/{74/03_delete_e2e_backup_keys_for_deactivated_users.sql => 75/01_delete_e2e_backup_keys_for_deactivated_users.sql} (91%) diff --git a/synapse/storage/schema/main/delta/74/03_delete_e2e_backup_keys_for_deactivated_users.sql b/synapse/storage/schema/main/delta/75/01_delete_e2e_backup_keys_for_deactivated_users.sql similarity index 91% rename from synapse/storage/schema/main/delta/74/03_delete_e2e_backup_keys_for_deactivated_users.sql rename to synapse/storage/schema/main/delta/75/01_delete_e2e_backup_keys_for_deactivated_users.sql index 8d713140feec..38630cb49cd6 100644 --- a/synapse/storage/schema/main/delta/74/03_delete_e2e_backup_keys_for_deactivated_users.sql +++ b/synapse/storage/schema/main/delta/75/01_delete_e2e_backup_keys_for_deactivated_users.sql @@ -14,4 +14,4 @@ */ INSERT INTO background_updates (ordering, update_name, progress_json) VALUES - (7403, 'delete_e2e_backup_keys_for_deactivated_users', '{}'); \ No newline at end of file + (7501, 'delete_e2e_backup_keys_for_deactivated_users', '{}'); \ No newline at end of file From 59f75a0891a3eadd8daa5f07f119a0dc86fb8fcf Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Tue, 4 Apr 2023 12:11:55 -0700 Subject: [PATCH 11/11] update schema --- .../04_delete_e2e_backup_keys_for_deactivated_users.sql} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename synapse/storage/schema/main/delta/{75/01_delete_e2e_backup_keys_for_deactivated_users.sql => 74/04_delete_e2e_backup_keys_for_deactivated_users.sql} (91%) diff --git a/synapse/storage/schema/main/delta/75/01_delete_e2e_backup_keys_for_deactivated_users.sql b/synapse/storage/schema/main/delta/74/04_delete_e2e_backup_keys_for_deactivated_users.sql similarity index 91% rename from synapse/storage/schema/main/delta/75/01_delete_e2e_backup_keys_for_deactivated_users.sql rename to synapse/storage/schema/main/delta/74/04_delete_e2e_backup_keys_for_deactivated_users.sql index 38630cb49cd6..a194f4cecef3 100644 --- a/synapse/storage/schema/main/delta/75/01_delete_e2e_backup_keys_for_deactivated_users.sql +++ b/synapse/storage/schema/main/delta/74/04_delete_e2e_backup_keys_for_deactivated_users.sql @@ -14,4 +14,4 @@ */ INSERT INTO background_updates (ordering, update_name, progress_json) VALUES - (7501, 'delete_e2e_backup_keys_for_deactivated_users', '{}'); \ No newline at end of file + (7404, 'delete_e2e_backup_keys_for_deactivated_users', '{}'); \ No newline at end of file