Update to new database version 2 (PedestrianDynamics#1338)

* Update to new database version 2

New version handles changing geometries during the simulation.

* update visualiser

---------

Co-authored-by: Ozaq <[email protected]>

python_modules/jupedsim/jupedsim/geometry.py

@@ -1,6 +1,8 @@
 # Copyright © 2012-2024 Forschungszentrum Jülich GmbH
 # SPDX-License-Identifier: LGPL-3.0-or-later
 
+import shapely
+
 import jupedsim.native as py_jps
 
 
@@ -32,3 +34,15 @@ def holes(self) -> list[list[tuple[float, float]]]:
             A list of polygons forming holes inside the boundary.
         """
         return self._obj.holes()
+
+    def as_wkt(self) -> str:
+        """_summary_
+
+        Returns:
+            String: _description_
+        """
+        poly = shapely.Polygon(self.boundary(), holes=self.holes())
+        return shapely.to_wkt(
+            poly,
+            rounding_precision=-1,
+        )

python_modules/jupedsim/jupedsim/recording.py

@@ -6,6 +6,7 @@
 import shapely
 
 from jupedsim.internal.aabb import AABB
+from jupedsim.sqlite_serialization import update_database_to_latest_version
 
 
 @dataclass
@@ -26,13 +27,14 @@ class RecordingFrame:
 
 
 class Recording:
-    __supported_database_version = 1
+    __supported_database_version = 2
     """Provides access to a simulation recording in a sqlite database"""
 
     def __init__(self, db_connection_str: str, uri=False) -> None:
         self.db = sqlite3.connect(
             db_connection_str, uri=uri, isolation_level=None
         )
+        update_database_to_latest_version(self.db)
         self._check_version_compatible()
 
     def frame(self, index: int) -> RecordingFrame:
@@ -66,8 +68,16 @@ def geometry(self) -> shapely.GeometryCollection:
         """
         cur = self.db.cursor()
         res = cur.execute("SELECT wkt FROM geometry")
-        wkt_str = res.fetchone()[0]
-        return shapely.from_wkt(wkt_str)
+        geometries = [shapely.from_wkt(s) for s in res.fetchall()]
+        return shapely.union_all(geometries)
+
+    def geometry_id_for_frame(self, frame_id) -> int:
+        cur = self.db.cursor()
+        res = cur.execute(
+            "SELECT geometry_hash from frame_data WHERE frame == ?",
+            (frame_id,),
+        )
+        return res.fetchone()[0]
 
     def bounds(self) -> AABB:
         """Get bounds of the position data contained in this recording."""
@@ -91,7 +101,7 @@ def num_frames(self) -> int:
 
         """
         cur = self.db.cursor()
-        res = cur.execute("SELECT MAX(frame) FROM trajectory_data")
+        res = cur.execute("SELECT count(*) FROM frame_data")
         return res.fetchone()[0]
 
     @property

python_modules/jupedsim/jupedsim/sqlite_serialization.py

@@ -1,14 +1,30 @@
 # Copyright © 2012-2024 Forschungszentrum Jülich GmbH
 # SPDX-License-Identifier: LGPL-3.0-or-later
 
+import itertools
 import sqlite3
 from pathlib import Path
-
-import shapely
+from typing import Final
 
 from jupedsim.serialization import TrajectoryWriter
 from jupedsim.simulation import Simulation
 
+DATABASE_VERSION: Final = 2
+
+
+def get_database_version(connection: sqlite3.Connection) -> int:
+    cur = connection.cursor()
+    return int(
+        cur.execute(
+            "SELECT value FROM metadata WHERE key = ?", ("version",)
+        ).fetchone()[0]
+    )
+
+
+def uses_latest_database_version(connection: sqlite3.Connection) -> bool:
+    version = get_database_version(connection)
+    return version == DATABASE_VERSION
+
 
 class SqliteTrajectoryWriter(TrajectoryWriter):
     """Write trajectory data into a sqlite db"""
@@ -40,11 +56,7 @@ def begin_writing(self, simulation: Simulation) -> None:
         such as framerate etc...
         """
         fps = 1 / simulation.delta_time() / self._every_nth_frame
-        geometry = simulation.get_geometry()
-        geo = shapely.to_wkt(
-            shapely.Polygon(geometry.boundary(), holes=geometry.holes()),
-            rounding_precision=-1,
-        )
+        geo = simulation.get_geometry().as_wkt()
 
         cur = self._con.cursor()
         try:
@@ -65,11 +77,25 @@ def begin_writing(self, simulation: Simulation) -> None:
             )
             cur.executemany(
                 "INSERT INTO metadata VALUES(?, ?)",
-                (("version", "1"), ("fps", fps)),
+                (("version", DATABASE_VERSION), ("fps", fps)),
             )
             cur.execute("DROP TABLE IF EXISTS geometry")
-            cur.execute("CREATE TABLE geometry(wkt TEXT NOT NULL)")
-            cur.execute("INSERT INTO geometry VALUES(?)", (geo,))
+            cur.execute(
+                "CREATE TABLE geometry("
+                "   hash INTEGER NOT NULL, "
+                "   wkt TEXT NOT NULL)"
+            )
+            cur.execute(
+                "INSERT INTO geometry VALUES(?, ?)",
+                (hash(geo), geo),
+            )
+            cur.execute("DROP TABLE IF EXISTS frame_data")
+            cur.execute(
+                "CREATE TABLE frame_data("
+                "   frame INTEGER NOT NULL,"
+                "   geometry_hash INTEGER NOT NULL)"
+            )
+
             cur.execute(
                 "CREATE INDEX frame_id_idx ON trajectory_data(frame, id)"
             )
@@ -128,6 +154,10 @@ def write_iteration_state(self, simulation: Simulation) -> None:
                     ("ymax", str(max(ymax, float(old_ymax)))),
                 ],
             )
+            cur.execute(
+                "INSERT INTO frame_data VALUES(?, ?)",
+                (frame, hash(simulation.get_geometry().as_wkt())),
+            )
             cur.execute("COMMIT")
         except sqlite3.Error as e:
             cur.execute("ROLLBACK")
@@ -160,3 +190,59 @@ def _y_min(self, cur):
 
     def _y_max(self, cur):
         return self._value_or_default(cur, "ymax", float("-inf"))
+
+
+def update_database_to_latest_version(connection: sqlite3.Connection):
+    version = get_database_version(connection)
+
+    if version == 1:
+        convert_database_v1_to_v2(connection)
+        version = 2
+
+    # if version == 2:
+    #     convert_database_v2_to_v3
+    #     version = 3
+    # ... for future versions
+
+
+def convert_database_v1_to_v2(connection: sqlite3.Connection):
+    cur = connection.cursor()
+
+    try:
+        cur.execute("BEGIN")
+
+        version = get_database_version(connection)
+        if version != 1:
+            raise RuntimeError(
+                f"Internal Error: When converting from database version 1 to 2, encountered database version {version}."
+            )
+
+        cur.execute(
+            "UPDATE metadata SET value = ? WHERE key = ?", (2, "version")
+        )
+
+        cur.execute(
+            "CREATE TABLE frame_data("
+            "   frame INTEGER NOT NULL,"
+            "   geometry_hash INTEGER NOT NULL)"
+        )
+
+        res = cur.execute("SELECT wkt FROM geometry")
+        wkt_str = res.fetchone()[0]
+        wkt_hash = hash(wkt_str)
+
+        cur.execute("ALTER TABLE geometry ADD hash INTEGER NOT NULL DEFAULT 0")
+        cur.execute("UPDATE geometry SET hash = ?", (wkt_hash,))
+
+        res = cur.execute("SELECT max(frame) FROM trajectory_data")
+        frame_limit = res.fetchone()[0] + 1
+
+        cur.executemany(
+            "INSERT INTO frame_data VALUES(?, ?)",
+            zip(range(frame_limit), itertools.repeat(wkt_hash)),
+        )
+        cur.execute("COMMIT")
+        cur.execute("VACUUM")
+    except sqlite3.Error as e:
+        cur.execute("ROLLBACK")
+        raise TrajectoryWriter.Exception(f"Error writing to database: {e}")