From 7d365c5b9a08d1690107e72a8276ce6796b5b523 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Thu, 9 Jun 2022 11:30:34 -0400 Subject: [PATCH 1/8] Fix-up some comments. --- contrib/graph/graph.py | 20 ++++++++++++-------- contrib/graph/graph2.py | 6 +++++- contrib/graph/graph3.py | 24 ++++++++++++++---------- 3 files changed, 31 insertions(+), 19 deletions(-) diff --git a/contrib/graph/graph.py b/contrib/graph/graph.py index fdbac087bdab..a047b9ecdfd3 100644 --- a/contrib/graph/graph.py +++ b/contrib/graph/graph.py @@ -1,11 +1,3 @@ -import argparse -import cgi -import datetime -import json - -import pydot -import urllib2 - # Copyright 2014-2016 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,12 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. +import argparse +import cgi +import datetime +import json + +import pydot +import urllib2 + def make_name(pdu_id, origin): return "%s@%s" % (pdu_id, origin) def make_graph(pdus, room, filename_prefix): + """ + Generate a dot and SVG file for a graph of events in the room based on the + topological ordering by querying a homeserver. + """ pdu_map = {} node_map = {} diff --git a/contrib/graph/graph2.py b/contrib/graph/graph2.py index 0980231e4a01..b1fe430cb582 100644 --- a/contrib/graph/graph2.py +++ b/contrib/graph/graph2.py @@ -26,6 +26,10 @@ def make_graph(db_name, room_id, file_prefix, limit): + """ + Generate a dot and SVG file for a graph of events in the room based on the + topological ordering by reading from a Synapse SQLite database. + """ conn = sqlite3.connect(db_name) sql = ( @@ -126,7 +130,7 @@ def make_graph(db_name, room_id, file_prefix, limit): if __name__ == "__main__": parser = argparse.ArgumentParser( description="Generate a PDU graph for a given room by talking " - "to the given homeserver to get the list of PDUs. \n" + "to the given a Synapse SQLite file to get the list of PDUs. \n" "Requires pydot." ) parser.add_argument( diff --git a/contrib/graph/graph3.py b/contrib/graph/graph3.py index dd0c19368b9e..bba1fb38caae 100644 --- a/contrib/graph/graph3.py +++ b/contrib/graph/graph3.py @@ -1,13 +1,3 @@ -import argparse -import cgi -import datetime - -import pydot -import simplejson as json - -from synapse.events import FrozenEvent -from synapse.util.frozenutils import unfreeze - # Copyright 2016 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,8 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. +import argparse +import cgi +import datetime + +import pydot +import simplejson as json + +from synapse.events import FrozenEvent +from synapse.util.frozenutils import unfreeze + def make_graph(file_name, room_id, file_prefix, limit): + """ + Generate a dot and SVG file for a graph of events in the room based on the + topological ordering by reading line-delimited JSON from a file. + """ print("Reading lines") with open(file_name) as f: lines = f.readlines() From 6d61179e4802cabddfa7b4a7345a7630d7b3793e Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Thu, 9 Jun 2022 11:48:24 -0400 Subject: [PATCH 2/8] Add type-hints. --- contrib/graph/graph.py | 7 ++++--- contrib/graph/graph2.py | 2 +- contrib/graph/graph3.py | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/contrib/graph/graph.py b/contrib/graph/graph.py index a047b9ecdfd3..a6700a2f3850 100644 --- a/contrib/graph/graph.py +++ b/contrib/graph/graph.py @@ -16,16 +16,17 @@ import cgi import datetime import json +from typing import List import pydot import urllib2 -def make_name(pdu_id, origin): +def make_name(pdu_id: str, origin: str) -> str: return "%s@%s" % (pdu_id, origin) -def make_graph(pdus, room, filename_prefix): +def make_graph(pdus: List[dict], room: str, filename_prefix: str) -> None: """ Generate a dot and SVG file for a graph of events in the room based on the topological ordering by querying a homeserver. @@ -115,7 +116,7 @@ def make_graph(pdus, room, filename_prefix): graph.write_svg("%s.svg" % filename_prefix, prog="dot") -def get_pdus(host, room): +def get_pdus(host: str, room: str) -> List[dict]: transaction = json.loads( urllib2.urlopen( "http://%s/_matrix/federation/v1/context/%s/" % (host, room) diff --git a/contrib/graph/graph2.py b/contrib/graph/graph2.py index b1fe430cb582..8f65ac9d908e 100644 --- a/contrib/graph/graph2.py +++ b/contrib/graph/graph2.py @@ -25,7 +25,7 @@ from synapse.util.frozenutils import unfreeze -def make_graph(db_name, room_id, file_prefix, limit): +def make_graph(db_name: str, room_id: str, file_prefix: str, limit: int) -> None: """ Generate a dot and SVG file for a graph of events in the room based on the topological ordering by reading from a Synapse SQLite database. diff --git a/contrib/graph/graph3.py b/contrib/graph/graph3.py index bba1fb38caae..01da83bb5268 100644 --- a/contrib/graph/graph3.py +++ b/contrib/graph/graph3.py @@ -23,7 +23,7 @@ from synapse.util.frozenutils import unfreeze -def make_graph(file_name, room_id, file_prefix, limit): +def make_graph(file_name: str, room_id: str, file_prefix: str, limit: int) -> None: """ Generate a dot and SVG file for a graph of events in the room based on the topological ordering by reading line-delimited JSON from a file. From 4296564447c93f139060771a949defc0340f0411 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Thu, 9 Jun 2022 11:22:42 -0400 Subject: [PATCH 3/8] Update graphs code for Python 3. --- contrib/graph/graph.py | 4 ++-- contrib/graph/graph2.py | 4 ++-- contrib/graph/graph3.py | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/contrib/graph/graph.py b/contrib/graph/graph.py index a6700a2f3850..ee5d3b48b2d1 100644 --- a/contrib/graph/graph.py +++ b/contrib/graph/graph.py @@ -16,10 +16,10 @@ import cgi import datetime import json +import urllib.request from typing import List import pydot -import urllib2 def make_name(pdu_id: str, origin: str) -> str: @@ -118,7 +118,7 @@ def make_graph(pdus: List[dict], room: str, filename_prefix: str) -> None: def get_pdus(host: str, room: str) -> List[dict]: transaction = json.loads( - urllib2.urlopen( + urllib.request.urlopen( "http://%s/_matrix/federation/v1/context/%s/" % (host, room) ).read() ) diff --git a/contrib/graph/graph2.py b/contrib/graph/graph2.py index 8f65ac9d908e..30eb682df61f 100644 --- a/contrib/graph/graph2.py +++ b/contrib/graph/graph2.py @@ -14,8 +14,8 @@ import argparse -import cgi import datetime +import html import json import sqlite3 @@ -88,7 +88,7 @@ def make_graph(db_name: str, room_id: str, file_prefix: str, limit: int) -> None "name": event.event_id, "type": event.type, "state_key": event.get("state_key", None), - "content": cgi.escape(content, quote=True), + "content": html.escape(content, quote=True), "time": t, "depth": event.depth, "state_group": state_group, diff --git a/contrib/graph/graph3.py b/contrib/graph/graph3.py index 01da83bb5268..6bd2ec1a5385 100644 --- a/contrib/graph/graph3.py +++ b/contrib/graph/graph3.py @@ -13,11 +13,11 @@ # limitations under the License. import argparse -import cgi import datetime +import html +import json import pydot -import simplejson as json from synapse.events import FrozenEvent from synapse.util.frozenutils import unfreeze @@ -70,8 +70,8 @@ def make_graph(file_name: str, room_id: str, file_prefix: str, limit: int) -> No content.append( "%s: %s," % ( - cgi.escape(key, quote=True).encode("ascii", "xmlcharrefreplace"), - cgi.escape(value, quote=True).encode("ascii", "xmlcharrefreplace"), + html.escape(key, quote=True).encode("ascii", "xmlcharrefreplace"), + html.escape(value, quote=True).encode("ascii", "xmlcharrefreplace"), ) ) From 0ec181985110724c98c010b5ab23692d6e94e8a7 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Thu, 9 Jun 2022 11:23:24 -0400 Subject: [PATCH 4/8] Update graphs code for changes in Synapse internals. --- contrib/graph/graph2.py | 16 ++++++++++++---- contrib/graph/graph3.py | 12 +++++++++--- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/contrib/graph/graph2.py b/contrib/graph/graph2.py index 30eb682df61f..7ca8a787ba79 100644 --- a/contrib/graph/graph2.py +++ b/contrib/graph/graph2.py @@ -21,7 +21,8 @@ import pydot -from synapse.events import FrozenEvent +from synapse.api.room_versions import KNOWN_ROOM_VERSIONS +from synapse.events import make_event_from_dict from synapse.util.frozenutils import unfreeze @@ -32,8 +33,12 @@ def make_graph(db_name: str, room_id: str, file_prefix: str, limit: int) -> None """ conn = sqlite3.connect(db_name) + sql = "SELECT room_version FROM rooms WHERE room_id = ?" + c = conn.execute(sql, (room_id,)) + room_version = KNOWN_ROOM_VERSIONS[c.fetchone()[0]] + sql = ( - "SELECT json FROM event_json as j " + "SELECT json, internal_metadata FROM event_json as j " "INNER JOIN events as e ON e.event_id = j.event_id " "WHERE j.room_id = ?" ) @@ -47,7 +52,10 @@ def make_graph(db_name: str, room_id: str, file_prefix: str, limit: int) -> None c = conn.execute(sql, args) - events = [FrozenEvent(json.loads(e[0])) for e in c.fetchall()] + events = [ + make_event_from_dict(json.loads(e[0]), room_version, json.loads(e[1])) + for e in c.fetchall() + ] events.sort(key=lambda e: e.depth) @@ -100,7 +108,7 @@ def make_graph(db_name: str, room_id: str, file_prefix: str, limit: int) -> None graph.add_node(node) for event in events: - for prev_id, _ in event.prev_events: + for prev_id in event.prev_event_ids(): try: end_node = node_map[prev_id] except Exception: diff --git a/contrib/graph/graph3.py b/contrib/graph/graph3.py index 6bd2ec1a5385..4171e045be35 100644 --- a/contrib/graph/graph3.py +++ b/contrib/graph/graph3.py @@ -19,7 +19,8 @@ import pydot -from synapse.events import FrozenEvent +from synapse.api.room_versions import KNOWN_ROOM_VERSIONS +from synapse.events import make_event_from_dict from synapse.util.frozenutils import unfreeze @@ -34,7 +35,12 @@ def make_graph(file_name: str, room_id: str, file_prefix: str, limit: int) -> No print("Read lines") - events = [FrozenEvent(json.loads(line)) for line in lines] + # Figure out the room version, assume the first line is the create event. + room_version = KNOWN_ROOM_VERSIONS[ + json.loads(lines[0]).get("content", {}).get("room_version") + ] + + events = [make_event_from_dict(json.loads(line), room_version) for line in lines] print("Loaded events.") @@ -105,7 +111,7 @@ def make_graph(file_name: str, room_id: str, file_prefix: str, limit: int) -> No print("Created Nodes") for event in events: - for prev_id, _ in event.prev_events: + for prev_id in event.prev_event_ids(): try: end_node = node_map[prev_id] except Exception: From 137678114a525c4cbee5c4917a987fb1f19e83a1 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Thu, 9 Jun 2022 12:00:49 -0400 Subject: [PATCH 5/8] Remove unused parameters. --- contrib/graph/graph.py | 4 ++-- contrib/graph/graph3.py | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/contrib/graph/graph.py b/contrib/graph/graph.py index ee5d3b48b2d1..3887eb89d5e7 100644 --- a/contrib/graph/graph.py +++ b/contrib/graph/graph.py @@ -26,7 +26,7 @@ def make_name(pdu_id: str, origin: str) -> str: return "%s@%s" % (pdu_id, origin) -def make_graph(pdus: List[dict], room: str, filename_prefix: str) -> None: +def make_graph(pdus: List[dict], filename_prefix: str) -> None: """ Generate a dot and SVG file for a graph of events in the room based on the topological ordering by querying a homeserver. @@ -146,4 +146,4 @@ def get_pdus(host: str, room: str) -> List[dict]: pdus = get_pdus(host, room) - make_graph(pdus, room, prefix) + make_graph(pdus, prefix) diff --git a/contrib/graph/graph3.py b/contrib/graph/graph3.py index 4171e045be35..7fddba7fd695 100644 --- a/contrib/graph/graph3.py +++ b/contrib/graph/graph3.py @@ -24,7 +24,7 @@ from synapse.util.frozenutils import unfreeze -def make_graph(file_name: str, room_id: str, file_prefix: str, limit: int) -> None: +def make_graph(file_name: str, file_prefix: str, limit: int) -> None: """ Generate a dot and SVG file for a graph of events in the room based on the topological ordering by reading line-delimited JSON from a file. @@ -149,8 +149,7 @@ def make_graph(file_name: str, room_id: str, file_prefix: str, limit: int) -> No ) parser.add_argument("-l", "--limit", help="Only retrieve the last N events.") parser.add_argument("event_file") - parser.add_argument("room") args = parser.parse_args() - make_graph(args.event_file, args.room, args.prefix, args.limit) + make_graph(args.event_file, args.prefix, args.limit) From ea7f6036baa66dae2d46e178f5de0f7c8be699ce Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Thu, 9 Jun 2022 12:01:17 -0400 Subject: [PATCH 6/8] Run pyupgrade. --- contrib/graph/graph.py | 4 ++-- contrib/graph/graph2.py | 4 ++-- contrib/graph/graph3.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/contrib/graph/graph.py b/contrib/graph/graph.py index 3887eb89d5e7..3c4f47dbd25e 100644 --- a/contrib/graph/graph.py +++ b/contrib/graph/graph.py @@ -23,7 +23,7 @@ def make_name(pdu_id: str, origin: str) -> str: - return "%s@%s" % (pdu_id, origin) + return f"{pdu_id}@{origin}" def make_graph(pdus: List[dict], filename_prefix: str) -> None: @@ -119,7 +119,7 @@ def make_graph(pdus: List[dict], filename_prefix: str) -> None: def get_pdus(host: str, room: str) -> List[dict]: transaction = json.loads( urllib.request.urlopen( - "http://%s/_matrix/federation/v1/context/%s/" % (host, room) + f"http://{host}/_matrix/federation/v1/context/{room}/" ).read() ) diff --git a/contrib/graph/graph2.py b/contrib/graph/graph2.py index 7ca8a787ba79..57a4fa5e3716 100644 --- a/contrib/graph/graph2.py +++ b/contrib/graph/graph2.py @@ -112,7 +112,7 @@ def make_graph(db_name: str, room_id: str, file_prefix: str, limit: int) -> None try: end_node = node_map[prev_id] except Exception: - end_node = pydot.Node(name=prev_id, label="<%s>" % (prev_id,)) + end_node = pydot.Node(name=prev_id, label=f"<{prev_id}>") node_map[prev_id] = end_node graph.add_node(end_node) @@ -124,7 +124,7 @@ def make_graph(db_name: str, room_id: str, file_prefix: str, limit: int) -> None if len(event_ids) <= 1: continue - cluster = pydot.Cluster(str(group), label="" % (str(group),)) + cluster = pydot.Cluster(str(group), label=f"") for event_id in event_ids: cluster.add_node(node_map[event_id]) diff --git a/contrib/graph/graph3.py b/contrib/graph/graph3.py index 7fddba7fd695..a28a1594c7fe 100644 --- a/contrib/graph/graph3.py +++ b/contrib/graph/graph3.py @@ -115,7 +115,7 @@ def make_graph(file_name: str, file_prefix: str, limit: int) -> None: try: end_node = node_map[prev_id] except Exception: - end_node = pydot.Node(name=prev_id, label="<%s>" % (prev_id,)) + end_node = pydot.Node(name=prev_id, label=f"<{prev_id}>") node_map[prev_id] = end_node graph.add_node(end_node) From e11941e430421ec127ffbf4536f2da7ea91b282e Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Thu, 9 Jun 2022 12:03:08 -0400 Subject: [PATCH 7/8] Newsfragment --- changelog.d/13013.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/13013.misc diff --git a/changelog.d/13013.misc b/changelog.d/13013.misc new file mode 100644 index 000000000000..903c6a3c8ab7 --- /dev/null +++ b/changelog.d/13013.misc @@ -0,0 +1 @@ +Modernize the `contrib/graph/` scripts. From 94072fd7d7951514ced8c4c3627c24dff5b39449 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Fri, 10 Jun 2022 07:10:18 -0400 Subject: [PATCH 8/8] Clarifications from review. Co-authored-by: Shay --- contrib/graph/graph2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/graph/graph2.py b/contrib/graph/graph2.py index 57a4fa5e3716..b46094ce0a50 100644 --- a/contrib/graph/graph2.py +++ b/contrib/graph/graph2.py @@ -138,7 +138,7 @@ def make_graph(db_name: str, room_id: str, file_prefix: str, limit: int) -> None if __name__ == "__main__": parser = argparse.ArgumentParser( description="Generate a PDU graph for a given room by talking " - "to the given a Synapse SQLite file to get the list of PDUs. \n" + "to the given Synapse SQLite file to get the list of PDUs. \n" "Requires pydot." ) parser.add_argument(