From 2b80ae9a920b99fa97a319fbb475a898cac67851 Mon Sep 17 00:00:00 2001 From: schroedtert Date: Tue, 12 Mar 2024 11:56:40 +0100 Subject: [PATCH] Update to new database version 2 (#1338) * Update to new database version 2 New version handles changing geometries during the simulation. * update visualiser --------- Co-authored-by: Ozaq --- python_modules/jupedsim/jupedsim/geometry.py | 14 +++ python_modules/jupedsim/jupedsim/recording.py | 18 ++- .../jupedsim/jupedsim/sqlite_serialization.py | 106 ++++++++++++++++-- 3 files changed, 124 insertions(+), 14 deletions(-) diff --git a/python_modules/jupedsim/jupedsim/geometry.py b/python_modules/jupedsim/jupedsim/geometry.py index 580ae0b60..166257628 100644 --- a/python_modules/jupedsim/jupedsim/geometry.py +++ b/python_modules/jupedsim/jupedsim/geometry.py @@ -1,6 +1,8 @@ # Copyright © 2012-2024 Forschungszentrum Jülich GmbH # SPDX-License-Identifier: LGPL-3.0-or-later +import shapely + import jupedsim.native as py_jps @@ -32,3 +34,15 @@ def holes(self) -> list[list[tuple[float, float]]]: A list of polygons forming holes inside the boundary. """ return self._obj.holes() + + def as_wkt(self) -> str: + """_summary_ + + Returns: + String: _description_ + """ + poly = shapely.Polygon(self.boundary(), holes=self.holes()) + return shapely.to_wkt( + poly, + rounding_precision=-1, + ) diff --git a/python_modules/jupedsim/jupedsim/recording.py b/python_modules/jupedsim/jupedsim/recording.py index 75e12b811..28417c8f0 100644 --- a/python_modules/jupedsim/jupedsim/recording.py +++ b/python_modules/jupedsim/jupedsim/recording.py @@ -6,6 +6,7 @@ import shapely from jupedsim.internal.aabb import AABB +from jupedsim.sqlite_serialization import update_database_to_latest_version @dataclass @@ -26,13 +27,14 @@ class RecordingFrame: class Recording: - __supported_database_version = 1 + __supported_database_version = 2 """Provides access to a simulation recording in a sqlite database""" def __init__(self, db_connection_str: str, uri=False) -> None: self.db = sqlite3.connect( db_connection_str, uri=uri, isolation_level=None ) + update_database_to_latest_version(self.db) self._check_version_compatible() def frame(self, index: int) -> RecordingFrame: @@ -66,8 +68,16 @@ def geometry(self) -> shapely.GeometryCollection: """ cur = self.db.cursor() res = cur.execute("SELECT wkt FROM geometry") - wkt_str = res.fetchone()[0] - return shapely.from_wkt(wkt_str) + geometries = [shapely.from_wkt(s) for s in res.fetchall()] + return shapely.union_all(geometries) + + def geometry_id_for_frame(self, frame_id) -> int: + cur = self.db.cursor() + res = cur.execute( + "SELECT geometry_hash from frame_data WHERE frame == ?", + (frame_id,), + ) + return res.fetchone()[0] def bounds(self) -> AABB: """Get bounds of the position data contained in this recording.""" @@ -91,7 +101,7 @@ def num_frames(self) -> int: """ cur = self.db.cursor() - res = cur.execute("SELECT MAX(frame) FROM trajectory_data") + res = cur.execute("SELECT count(*) FROM frame_data") return res.fetchone()[0] @property diff --git a/python_modules/jupedsim/jupedsim/sqlite_serialization.py b/python_modules/jupedsim/jupedsim/sqlite_serialization.py index 155d50e57..e37d97d4f 100644 --- a/python_modules/jupedsim/jupedsim/sqlite_serialization.py +++ b/python_modules/jupedsim/jupedsim/sqlite_serialization.py @@ -1,14 +1,30 @@ # Copyright © 2012-2024 Forschungszentrum Jülich GmbH # SPDX-License-Identifier: LGPL-3.0-or-later +import itertools import sqlite3 from pathlib import Path - -import shapely +from typing import Final from jupedsim.serialization import TrajectoryWriter from jupedsim.simulation import Simulation +DATABASE_VERSION: Final = 2 + + +def get_database_version(connection: sqlite3.Connection) -> int: + cur = connection.cursor() + return int( + cur.execute( + "SELECT value FROM metadata WHERE key = ?", ("version",) + ).fetchone()[0] + ) + + +def uses_latest_database_version(connection: sqlite3.Connection) -> bool: + version = get_database_version(connection) + return version == DATABASE_VERSION + class SqliteTrajectoryWriter(TrajectoryWriter): """Write trajectory data into a sqlite db""" @@ -40,11 +56,7 @@ def begin_writing(self, simulation: Simulation) -> None: such as framerate etc... """ fps = 1 / simulation.delta_time() / self._every_nth_frame - geometry = simulation.get_geometry() - geo = shapely.to_wkt( - shapely.Polygon(geometry.boundary(), holes=geometry.holes()), - rounding_precision=-1, - ) + geo = simulation.get_geometry().as_wkt() cur = self._con.cursor() try: @@ -65,11 +77,25 @@ def begin_writing(self, simulation: Simulation) -> None: ) cur.executemany( "INSERT INTO metadata VALUES(?, ?)", - (("version", "1"), ("fps", fps)), + (("version", DATABASE_VERSION), ("fps", fps)), ) cur.execute("DROP TABLE IF EXISTS geometry") - cur.execute("CREATE TABLE geometry(wkt TEXT NOT NULL)") - cur.execute("INSERT INTO geometry VALUES(?)", (geo,)) + cur.execute( + "CREATE TABLE geometry(" + " hash INTEGER NOT NULL, " + " wkt TEXT NOT NULL)" + ) + cur.execute( + "INSERT INTO geometry VALUES(?, ?)", + (hash(geo), geo), + ) + cur.execute("DROP TABLE IF EXISTS frame_data") + cur.execute( + "CREATE TABLE frame_data(" + " frame INTEGER NOT NULL," + " geometry_hash INTEGER NOT NULL)" + ) + cur.execute( "CREATE INDEX frame_id_idx ON trajectory_data(frame, id)" ) @@ -128,6 +154,10 @@ def write_iteration_state(self, simulation: Simulation) -> None: ("ymax", str(max(ymax, float(old_ymax)))), ], ) + cur.execute( + "INSERT INTO frame_data VALUES(?, ?)", + (frame, hash(simulation.get_geometry().as_wkt())), + ) cur.execute("COMMIT") except sqlite3.Error as e: cur.execute("ROLLBACK") @@ -160,3 +190,59 @@ def _y_min(self, cur): def _y_max(self, cur): return self._value_or_default(cur, "ymax", float("-inf")) + + +def update_database_to_latest_version(connection: sqlite3.Connection): + version = get_database_version(connection) + + if version == 1: + convert_database_v1_to_v2(connection) + version = 2 + + # if version == 2: + # convert_database_v2_to_v3 + # version = 3 + # ... for future versions + + +def convert_database_v1_to_v2(connection: sqlite3.Connection): + cur = connection.cursor() + + try: + cur.execute("BEGIN") + + version = get_database_version(connection) + if version != 1: + raise RuntimeError( + f"Internal Error: When converting from database version 1 to 2, encountered database version {version}." + ) + + cur.execute( + "UPDATE metadata SET value = ? WHERE key = ?", (2, "version") + ) + + cur.execute( + "CREATE TABLE frame_data(" + " frame INTEGER NOT NULL," + " geometry_hash INTEGER NOT NULL)" + ) + + res = cur.execute("SELECT wkt FROM geometry") + wkt_str = res.fetchone()[0] + wkt_hash = hash(wkt_str) + + cur.execute("ALTER TABLE geometry ADD hash INTEGER NOT NULL DEFAULT 0") + cur.execute("UPDATE geometry SET hash = ?", (wkt_hash,)) + + res = cur.execute("SELECT max(frame) FROM trajectory_data") + frame_limit = res.fetchone()[0] + 1 + + cur.executemany( + "INSERT INTO frame_data VALUES(?, ?)", + zip(range(frame_limit), itertools.repeat(wkt_hash)), + ) + cur.execute("COMMIT") + cur.execute("VACUUM") + except sqlite3.Error as e: + cur.execute("ROLLBACK") + raise TrajectoryWriter.Exception(f"Error writing to database: {e}")