Migration of data from demo db to dummy db on dev server

bin/migration/db_utils.py

@@ -0,0 +1,24 @@
+import psycopg2 
+
+class DatabaseManager:
+    def __init__(self, database, user, password, host, port):
+        self.connection = psycopg2.connect(
+            database=database,
+            user=user,
+            password=password,
+            host=host,
+            port=port
+        )
+        self.cursor = self.connection
+
+    def execute_query(self, query, params=None):
+        self.cursor.execute(query, params)
+        return self.cursor.fetchall()  
+
+    def close_connection(self):
+        self.cursor.close()
+        self.connection.close()
+
+
+
+

bin/migration/main_script.py

@@ -0,0 +1,100 @@
+import os
+
+from db_utils import DatabaseManager
+
+from tables.rooms import RoomManager
+from tables.users import UserManager
+from tables.roles import RoleManager
+from tables.courts import CourtManager
+from tables.courtrooms import CourtRoomManager
+from tables.regions import RegionManager
+from tables.courtregions import CourtRegionManager
+from tables.portalaccess import PortalAccessManager
+from tables.appaccess import AppAccessManager
+from tables.cases import CaseManager
+from tables.bookings import BookingManager
+from tables.participants import ParticipantManager
+from tables.bookingparticipants import BookingParticipantManager
+from tables.capturesessions import CaptureSessionManager
+from tables.recordings import RecordingManager
+
+import time
+
+start_time = time.time()
+
+# get passwords from env variables
+source_db_password = os.environ.get('SOURCE_DB_PASSWORD')
+destination_db_password = os.environ.get('DESTINATION_DB_PASSWORD')
+
+# database connections
+source_db = DatabaseManager(
+    database="pre-pdb-demo",
+    user="psqladmin",
+    password=source_db_password,
+    host="pre-db-demo.postgres.database.azure.com",
+    port="5432",
+)
+
+destination_db = DatabaseManager(
+    database="dev-pre-copy",
+    user="psqladmin",
+    password=destination_db_password,
+    host="pre-db-dev.postgres.database.azure.com",
+    port="5432",
+)
+
+# managers for different tables
+room_manager = RoomManager(source_db.connection.cursor())
+user_manager = UserManager(source_db.connection.cursor())
+role_manager = RoleManager(source_db.connection.cursor())
+court_manager = CourtManager(source_db.connection.cursor())
+courtroom_manager = CourtRoomManager()
+region_manager = RegionManager()
+court_region_manager = CourtRegionManager()
+portal_access_manager = PortalAccessManager(source_db.connection.cursor())
+app_access_manager = AppAccessManager(source_db.connection.cursor())
+case_manager = CaseManager(source_db.connection.cursor())
+booking_manager = BookingManager(source_db.connection.cursor())
+participant_manager = ParticipantManager(source_db.connection.cursor())
+booking_participant_manager = BookingParticipantManager()
+capture_session_manager = CaptureSessionManager(source_db.connection.cursor())
+recording_manager = RecordingManager(source_db.connection.cursor())
+
+def migrate_manager_data(manager, destination_cursor):
+    if hasattr(manager, 'get_data') and callable(getattr(manager, 'get_data')):
+        source_data = manager.get_data()
+        manager.migrate_data(destination_cursor, source_data)
+    else:
+        manager.migrate_data(destination_cursor)
+
+
+def main():
+    destination_db_cursor = destination_db.connection.cursor()
+
+    migrate_manager_data(room_manager, destination_db_cursor) # 20 / 20 migrated
+    migrate_manager_data(user_manager, destination_db_cursor) # 263 / 263 migrated
+    migrate_manager_data(role_manager, destination_db_cursor) # Levels 1 - 4 & superuser
+    migrate_manager_data(court_manager, destination_db_cursor) # 9 / 9 migrated and an added 'default court'
+    migrate_manager_data(courtroom_manager, destination_db_cursor) # 20 PRE rooms
+    migrate_manager_data(region_manager, destination_db_cursor) # 10 regions - not in current setup
+    migrate_manager_data(court_region_manager, destination_db_cursor) # 10 court region associations - not in current setup
+    migrate_manager_data(portal_access_manager, destination_db_cursor) # 52 users with Level 3 access migrated (58 users with no role set - not migrated)
+    migrate_manager_data(app_access_manager, destination_db_cursor) # 153 users migrated (58 users with no role set - not migrated)
+    migrate_manager_data(case_manager, destination_db_cursor) # 475 / 484 cases migrated (9 cases not migrated do not have a caseref)
+    migrate_manager_data(booking_manager, destination_db_cursor) # 475 / 484 cases migrated (9 cases not migrated do not have a caseref)
+    migrate_manager_data(participant_manager, destination_db_cursor) # 465 / 1747 partipants migrated - failed import ids in log
+    migrate_manager_data(booking_participant_manager, destination_db_cursor)
+    migrate_manager_data(capture_session_manager, destination_db_cursor)
+    migrate_manager_data(recording_manager, destination_db_cursor)
+
+    participant_manager.log_failed_imports()
+
+    source_db.close_connection()
+    destination_db.close_connection()
+
+    end_time = time.time()
+    execution_time = end_time - start_time
+    print(f"Execution time: {execution_time} seconds")
+
+if __name__ == "__main__":
+    main()

bin/migration/tables/appaccess.py

@@ -0,0 +1,90 @@
+from .helpers import check_existing_record, parse_to_timestamp, audit_entry_creation
+from datetime import datetime
+import uuid
+
+class AppAccessManager:
+    def __init__(self, source_cursor):
+        self.source_cursor = source_cursor
+<<<<<<< HEAD
+=======
+        self.failed_imports = set()
+>>>>>>> 90b5173 (converted enum types to uppercase, added in exceptions and functions for failed imports on some tables and added in scripts to count records)
+
+    def get_data(self):
+        self.source_cursor.execute("SELECT * FROM public.users WHERE prerole != 'Level 3'")
+        return self.source_cursor.fetchall()
+
+    def migrate_data(self, destination_cursor, source_data):
+        batch_app_users_data = []
+<<<<<<< HEAD
+=======
+        id = None
+>>>>>>> 90b5173 (converted enum types to uppercase, added in exceptions and functions for failed imports on some tables and added in scripts to count records)
+
+        for user in source_data:
+            user_id = user[0]
+
+            destination_cursor.execute("SELECT id FROM public.courts WHERE name = 'Default Court'")
+            default_court_id = destination_cursor.fetchone()[0]
+
+            if not check_existing_record(destination_cursor,'app_access', 'user_id', user_id):
+                id=str(uuid.uuid4())
+                court_id = default_court_id  
+
+                destination_cursor.execute("SELECT id FROM public.roles WHERE name = %s", (user[3],))
+                role_id = destination_cursor.fetchone()
+
+                last_access = datetime.now() # ?
+                active = True # ?
+                created_at = parse_to_timestamp(user[15])
+                modified_at =parse_to_timestamp(user[17])
+                created_by = user[14]
+
+                batch_app_users_data.append((
+                    id, user_id, court_id, role_id, last_access, active, created_at, modified_at
+                ))
+
+                audit_entry_creation(
+                    destination_cursor,
+                    table_name='app_access',
+                    record_id=id,
+                    record=user_id,
+                    created_at=created_at,
+                    created_by=created_by,
+                )
+
+<<<<<<< HEAD
+        if batch_app_users_data:
+            destination_cursor.executemany(
+                """
+                INSERT INTO public.app_access
+                    (id, user_id, court_id, role_id, last_access, active, created_at, modified_at)
+                VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
+                """,
+                batch_app_users_data,
+            )
+            destination_cursor.connection.commit()
+=======
+            else:
+                self.failed_imports.add(('app_access',id)) 
+
+        try: 
+            if batch_app_users_data:
+                destination_cursor.executemany(
+                    """
+                    INSERT INTO public.app_access
+                        (id, user_id, court_id, role_id, last_access, active, created_at, modified_at)
+                    VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
+                    """,
+                    batch_app_users_data,
+                )
+                destination_cursor.connection.commit()
+        except Exception as e:  
+            self.failed_imports.add(('app_access',id)) 
+
+
+    def log_failed_imports(self, filename='failed_imports_log.txt'):
+        with open(filename, 'w') as file:
+            for table_name, failed_id in self.failed_imports:
+                file.write(f"Table: {table_name}, ID: {failed_id}\n")
+>>>>>>> 90b5173 (converted enum types to uppercase, added in exceptions and functions for failed imports on some tables and added in scripts to count records)

bin/migration/tables/bookingparticipants.py

@@ -0,0 +1,38 @@
+from .helpers import audit_entry_creation
+import uuid
+
+class BookingParticipantManager:
+
+    def migrate_data(self, destination_cursor):
+        # Fetch case_id's for participants
+        destination_cursor.execute("""
+            SELECT p.id AS participant_id, b.id AS booking_id
+            FROM public.participants p
+            JOIN public.bookings b ON p.case_id = b.case_id
+            WHERE NOT EXISTS (
+                SELECT 1
+                FROM public.booking_participant bp
+                WHERE bp.participant_id = p.id AND bp.booking_id = b.id
+            )
+        """)
+        booking_participant_query = destination_cursor.fetchall()
+
+        for row in booking_participant_query:
+            participant_id, booking_id = row
+            id = str(uuid.uuid4())
+
+            destination_cursor.execute(
+                """
+                INSERT INTO public.booking_participant (id, participant_id, booking_id)
+                VALUES (%s, %s, %s)
+                """,
+                (id, participant_id, booking_id),  
+            )
+
+            audit_entry_creation(
+                destination_cursor,
+                table_name="booking_participant",
+                record_id=id,
+                record=booking_id,
+            )
+

bin/migration/tables/bookings.py

@@ -0,0 +1,93 @@
+from .helpers import check_existing_record, parse_to_timestamp, audit_entry_creation
+from datetime import datetime
+
+
+class BookingManager:
+    def __init__(self, source_cursor):
+        self.source_cursor = source_cursor
+<<<<<<< HEAD
+=======
+        self.failed_imports = set()
+>>>>>>> 90b5173 (converted enum types to uppercase, added in exceptions and functions for failed imports on some tables and added in scripts to count records)
+
+    def get_data(self):
+        self.source_cursor.execute("SELECT * FROM public.cases")
+        return self.source_cursor.fetchall()
+
+    def migrate_data(self, destination_cursor, source_data):
+        for booking in source_data:
+            id = booking[0]
+
+            if not check_existing_record(destination_cursor,'bookings', 'id', id):
+                destination_cursor.execute(
+                    "SELECT * FROM public.temp_cases WHERE reference = %s", (booking[1],)
+                )
+                case_details = destination_cursor.fetchone()
+                case_id = case_details[1] if case_details else None
+<<<<<<< HEAD
+
+                if case_id:
+                    # Check if case_id exists in the cases table
+                    if check_existing_record(destination_cursor,'cases','id', case_id):
+                        scheduled_for = (datetime.today()) 
+                        created_at = parse_to_timestamp(case_details[1])
+                        modified_at = parse_to_timestamp(case_details[3])
+                        created_by = case_details[2]
+
+                        destination_cursor.execute(
+                            """
+                            INSERT INTO public.bookings 
+                                (id, case_id, scheduled_for, created_at, modified_at)
+                            VALUES (%s, %s, %s, %s, %s )
+                            """,
+                            (id, case_id, scheduled_for, created_at, modified_at),
+                        )
+
+                        audit_entry_creation(
+                            destination_cursor,
+                            table_name="bookings",
+                            record_id=id,
+                            record=case_id,
+                            created_at=created_at,
+                            created_by=created_by,
+                        )
+
+=======
+
+                try:
+                    if case_id:
+                        # Check if case_id exists in the cases table
+                        if check_existing_record(destination_cursor,'cases','id', case_id):
+                            scheduled_for = (datetime.today()) 
+                            created_at = parse_to_timestamp(case_details[1])
+                            modified_at = parse_to_timestamp(case_details[3])
+                            created_by = case_details[2]
+
+                            destination_cursor.execute(
+                                """
+                                INSERT INTO public.bookings 
+                                    (id, case_id, scheduled_for, created_at, modified_at)
+                                VALUES (%s, %s, %s, %s, %s )
+                                """,
+                                (id, case_id, scheduled_for, created_at, modified_at),
+                            )
+
+                            audit_entry_creation(
+                                destination_cursor,
+                                table_name="bookings",
+                                record_id=id,
+                                record=case_id,
+                                created_at=created_at,
+                                created_by=created_by,
+                            )
+                except Exception as e:  
+                        self.failed_imports.add(('bookings', id))
+            else:
+                self.failed_imports.add(('bookings', id))
+
+    def log_failed_imports(self, filename='failed_imports_log.txt'):
+        with open(filename, 'w') as file:
+            for table_name, failed_id in self.failed_imports:
+                file.write(f"Table: {table_name}, ID: {failed_id}\n")
+>>>>>>> 90b5173 (converted enum types to uppercase, added in exceptions and functions for failed imports on some tables and added in scripts to count records)
+