From a369474aab355c056953021e9d89cb4a10bb2640 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Thu, 12 Mar 2020 15:59:24 +0000 Subject: [PATCH 1/4] Populate the room version from state events See `rooms_version_column_3.sql.postgres` for details about why we need to do that. --- .../57/rooms_version_column_3.sql.postgres | 38 +++++++++++++++++++ .../57/rooms_version_column_3.sql.sqlite | 22 +++++++++++ 2 files changed, 60 insertions(+) create mode 100644 synapse/storage/data_stores/main/schema/delta/57/rooms_version_column_3.sql.postgres create mode 100644 synapse/storage/data_stores/main/schema/delta/57/rooms_version_column_3.sql.sqlite diff --git a/synapse/storage/data_stores/main/schema/delta/57/rooms_version_column_3.sql.postgres b/synapse/storage/data_stores/main/schema/delta/57/rooms_version_column_3.sql.postgres new file mode 100644 index 000000000000..31be4d8e73ed --- /dev/null +++ b/synapse/storage/data_stores/main/schema/delta/57/rooms_version_column_3.sql.postgres @@ -0,0 +1,38 @@ +/* Copyright 2020 The Matrix.org Foundation C.I.C. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- When we first added the room_version column to the rooms table, it was populated from +-- the current_state_events table. However, there was an issue causing a background +-- update to clean up the current_state_events table for rooms where the server is no +-- longer participating, before that column could be populated. Therefore, some rooms had +-- a NULL room_version. + +-- The rooms_version_column_2.sql.* delta files were introduced to make the populating +-- synchronous instead of running it in a background update, which fixed this issue. +-- However, all of the instances of Synapse installed or updated in the meantime got +-- their rooms table corrupted with NULL room_versions. + +-- This query fishes out the room versions from the create event using the state_events +-- table instead of the current_state_events one, as the former still have all of the +-- create events. + +UPDATE rooms SET room_version=( + SELECT COALESCE(json::json->'content'->>'room_version','1') + FROM state_events se INNER JOIN event_json ej USING (event_id) + WHERE se.room_id=rooms.room_id AND se.type='m.room.create' AND se.state_key='' +) WHERE rooms.room_version IS NULL; + +-- see also rooms_version_column_3.sql.sqlite which has a copy of the above query, using +-- sqlite syntax for the json extraction. diff --git a/synapse/storage/data_stores/main/schema/delta/57/rooms_version_column_3.sql.sqlite b/synapse/storage/data_stores/main/schema/delta/57/rooms_version_column_3.sql.sqlite new file mode 100644 index 000000000000..d13911a64e0b --- /dev/null +++ b/synapse/storage/data_stores/main/schema/delta/57/rooms_version_column_3.sql.sqlite @@ -0,0 +1,22 @@ +/* Copyright 2020 The Matrix.org Foundation C.I.C. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- see rooms_version_column_3.sql.postgres for details of what's going on here. + +UPDATE rooms SET room_version=( + SELECT COALESCE(json_extract(ej.json, '$.content.room_version'), '1') + FROM state_events se INNER JOIN event_json ej USING (event_id) + WHERE se.room_id=rooms.room_id AND se.type='m.room.create' AND se.state_key='' +) WHERE rooms.room_version IS NULL; From 5fba53d78e7d37ef7ddb2eef73ac144451a6c70a Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Thu, 12 Mar 2020 16:48:11 +0000 Subject: [PATCH 2/4] Only consider one create event per room --- .../main/schema/delta/57/rooms_version_column_3.sql.postgres | 1 + .../main/schema/delta/57/rooms_version_column_3.sql.sqlite | 1 + 2 files changed, 2 insertions(+) diff --git a/synapse/storage/data_stores/main/schema/delta/57/rooms_version_column_3.sql.postgres b/synapse/storage/data_stores/main/schema/delta/57/rooms_version_column_3.sql.postgres index 31be4d8e73ed..92aaadde0d99 100644 --- a/synapse/storage/data_stores/main/schema/delta/57/rooms_version_column_3.sql.postgres +++ b/synapse/storage/data_stores/main/schema/delta/57/rooms_version_column_3.sql.postgres @@ -32,6 +32,7 @@ UPDATE rooms SET room_version=( SELECT COALESCE(json::json->'content'->>'room_version','1') FROM state_events se INNER JOIN event_json ej USING (event_id) WHERE se.room_id=rooms.room_id AND se.type='m.room.create' AND se.state_key='' + LIMIT 1 ) WHERE rooms.room_version IS NULL; -- see also rooms_version_column_3.sql.sqlite which has a copy of the above query, using diff --git a/synapse/storage/data_stores/main/schema/delta/57/rooms_version_column_3.sql.sqlite b/synapse/storage/data_stores/main/schema/delta/57/rooms_version_column_3.sql.sqlite index d13911a64e0b..e19dab97cbf6 100644 --- a/synapse/storage/data_stores/main/schema/delta/57/rooms_version_column_3.sql.sqlite +++ b/synapse/storage/data_stores/main/schema/delta/57/rooms_version_column_3.sql.sqlite @@ -19,4 +19,5 @@ UPDATE rooms SET room_version=( SELECT COALESCE(json_extract(ej.json, '$.content.room_version'), '1') FROM state_events se INNER JOIN event_json ej USING (event_id) WHERE se.room_id=rooms.room_id AND se.type='m.room.create' AND se.state_key='' + LIMIT 1 ) WHERE rooms.room_version IS NULL; From 7275b77ac3cae0b54daa835376ea9441fd6da44a Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Thu, 12 Mar 2020 17:38:52 +0000 Subject: [PATCH 3/4] Changelog --- changelog.d/7070.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/7070.bugfix diff --git a/changelog.d/7070.bugfix b/changelog.d/7070.bugfix new file mode 100644 index 000000000000..24ac24d89112 --- /dev/null +++ b/changelog.d/7070.bugfix @@ -0,0 +1 @@ +Fix a bug causing `/sync` returning with 404 errors about missing events and unknown rooms. From bc99fb0ff0fd30593c3bcaadbe38ce07a3102dff Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 16 Mar 2020 23:30:25 +0100 Subject: [PATCH 4/4] Update changelog.d/7070.bugfix --- changelog.d/7070.bugfix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/7070.bugfix b/changelog.d/7070.bugfix index 24ac24d89112..9031927546fe 100644 --- a/changelog.d/7070.bugfix +++ b/changelog.d/7070.bugfix @@ -1 +1 @@ -Fix a bug causing `/sync` returning with 404 errors about missing events and unknown rooms. +Repair a data-corruption issue which was introduced in Synapse 1.10, and fixed in Synapse 1.11, and which could cause `/sync` to return with 404 errors about missing events and unknown rooms.