rebased after changes

bin/migration/main_script.py

@@ -18,6 +18,14 @@
 from tables.capturesessions import CaptureSessionManager
 from tables.recordings import RecordingManager
 
+<<<<<<< HEAD
+=======
+import time
+
+start_time = time.time()
+
+
+>>>>>>> 90b5173 (converted enum types to uppercase, added in exceptions and functions for failed imports on some tables and added in scripts to count records)
 # get passwords from env variables
 source_db_password = os.environ.get('SOURCE_DB_PASSWORD')
 destination_db_password = os.environ.get('DESTINATION_DB_PASSWORD')
@@ -26,23 +34,35 @@
 source_db = DatabaseManager(
     database="pre-pdb-demo",
     user="psqladmin",
+<<<<<<< HEAD
     password=source_db_password,
+=======
+    password="0q1)V04AXX5_zbGjDsRi",
+>>>>>>> 90b5173 (converted enum types to uppercase, added in exceptions and functions for failed imports on some tables and added in scripts to count records)
     host="pre-db-demo.postgres.database.azure.com",
     port="5432",
 )
 
 destination_db = DatabaseManager(
     database="dev-pre-copy",
     user="psqladmin",
+<<<<<<< HEAD
     password=destination_db_password,
+=======
+    password="UP27EUYsdjZMSF03Q4HB",
+>>>>>>> 90b5173 (converted enum types to uppercase, added in exceptions and functions for failed imports on some tables and added in scripts to count records)
     host="pre-db-dev.postgres.database.azure.com",
     port="5432",
 )
 
 # managers for different tables
 room_manager = RoomManager(source_db.connection.cursor())
 user_manager = UserManager(source_db.connection.cursor())
+<<<<<<< HEAD
 role_manager = RoleManager(source_db.connection.cursor())
+=======
+role_manager = RoleManager()
+>>>>>>> 90b5173 (converted enum types to uppercase, added in exceptions and functions for failed imports on some tables and added in scripts to count records)
 court_manager = CourtManager(source_db.connection.cursor())
 courtroom_manager = CourtRoomManager()
 region_manager = RegionManager()
@@ -67,6 +87,7 @@ def migrate_manager_data(manager, destination_cursor):
 def main():
     destination_db_cursor = destination_db.connection.cursor()
 
+<<<<<<< HEAD
     migrate_manager_data(room_manager, destination_db_cursor)
     migrate_manager_data(user_manager, destination_db_cursor)
     migrate_manager_data(role_manager, destination_db_cursor)
@@ -79,12 +100,39 @@ def main():
     migrate_manager_data(case_manager, destination_db_cursor)
     migrate_manager_data(booking_manager, destination_db_cursor)
     migrate_manager_data(participant_manager, destination_db_cursor)
+=======
+    migrate_manager_data(room_manager, destination_db_cursor) # 20 / 20 migrated
+    migrate_manager_data(user_manager, destination_db_cursor) # 263 / 263 migrated
+    migrate_manager_data(role_manager, destination_db_cursor) # Levels 1 - 4 & superuser
+    migrate_manager_data(court_manager, destination_db_cursor) # 9 / 9 migrated and an added 'default court' 
+    migrate_manager_data(courtroom_manager, destination_db_cursor) # 20 PRE rooms
+    migrate_manager_data(region_manager, destination_db_cursor) # 10 regions - not in current setup
+    migrate_manager_data(court_region_manager, destination_db_cursor) # 10 court region associations - not in current setup
+    migrate_manager_data(portal_access_manager, destination_db_cursor) # 52 users with Level 3 access migrated (58 users with no role set - not migrated)
+    migrate_manager_data(app_access_manager, destination_db_cursor) # 153 users migrated (58 users with no role set - not migrated)
+    migrate_manager_data(case_manager, destination_db_cursor) # 475 / 484 cases migrated (9 cases not migrated do not have a caseref)
+    migrate_manager_data(booking_manager, destination_db_cursor) # 475 / 484 cases migrated (9 cases not migrated do not have a caseref)
+    migrate_manager_data(participant_manager, destination_db_cursor) # 465 / 1747 partipants migrated - failed import ids in log
+>>>>>>> 90b5173 (converted enum types to uppercase, added in exceptions and functions for failed imports on some tables and added in scripts to count records)
     migrate_manager_data(booking_participant_manager, destination_db_cursor)
     migrate_manager_data(capture_session_manager, destination_db_cursor)
     migrate_manager_data(recording_manager, destination_db_cursor)
 
+<<<<<<< HEAD
     source_db.close_connection()
     destination_db.close_connection()
 
+=======
+    participant_manager.log_failed_imports()
+
+    source_db.close_connection()
+    destination_db.close_connection()
+
+    end_time = time.time()
+    execution_time = end_time - start_time
+    print(f"Execution time: {execution_time} seconds")
+
+
+>>>>>>> 90b5173 (converted enum types to uppercase, added in exceptions and functions for failed imports on some tables and added in scripts to count records)
 if __name__ == "__main__":
     main()

bin/migration/tables/appaccess.py

@@ -5,13 +5,21 @@
 class AppAccessManager:
     def __init__(self, source_cursor):
         self.source_cursor = source_cursor
+<<<<<<< HEAD
+=======
+        self.failed_imports = set()
+>>>>>>> 90b5173 (converted enum types to uppercase, added in exceptions and functions for failed imports on some tables and added in scripts to count records)
 
     def get_data(self):
         self.source_cursor.execute("SELECT * FROM public.users WHERE prerole != 'Level 3'")
         return self.source_cursor.fetchall()
 
     def migrate_data(self, destination_cursor, source_data):
         batch_app_users_data = []
+<<<<<<< HEAD
+=======
+        id = None
+>>>>>>> 90b5173 (converted enum types to uppercase, added in exceptions and functions for failed imports on some tables and added in scripts to count records)
 
         for user in source_data:
             user_id = user[0]
@@ -45,6 +53,7 @@ def migrate_data(self, destination_cursor, source_data):
                     created_by=created_by,
                 )
 
+<<<<<<< HEAD
         if batch_app_users_data:
             destination_cursor.executemany(
                 """
@@ -55,3 +64,27 @@ def migrate_data(self, destination_cursor, source_data):
                 batch_app_users_data,
             )
             destination_cursor.connection.commit()
+=======
+            else:
+                self.failed_imports.add(('app_access',id)) 
+
+        try: 
+            if batch_app_users_data:
+                destination_cursor.executemany(
+                    """
+                    INSERT INTO public.app_access
+                        (id, user_id, court_id, role_id, last_access, active, created_at, modified_at)
+                    VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
+                    """,
+                    batch_app_users_data,
+                )
+                destination_cursor.connection.commit()
+        except Exception as e:  
+            self.failed_imports.add(('app_access',id)) 
+
+
+    def log_failed_imports(self, filename='failed_imports_log.txt'):
+        with open(filename, 'w') as file:
+            for table_name, failed_id in self.failed_imports:
+                file.write(f"Table: {table_name}, ID: {failed_id}\n")
+>>>>>>> 90b5173 (converted enum types to uppercase, added in exceptions and functions for failed imports on some tables and added in scripts to count records)

bin/migration/tables/bookings.py

@@ -5,6 +5,10 @@
 class BookingManager:
     def __init__(self, source_cursor):
         self.source_cursor = source_cursor
+<<<<<<< HEAD
+=======
+        self.failed_imports = set()
+>>>>>>> 90b5173 (converted enum types to uppercase, added in exceptions and functions for failed imports on some tables and added in scripts to count records)
 
     def get_data(self):
         self.source_cursor.execute("SELECT * FROM public.cases")
@@ -20,6 +24,7 @@ def migrate_data(self, destination_cursor, source_data):
                 )
                 case_details = destination_cursor.fetchone()
                 case_id = case_details[1] if case_details else None
+<<<<<<< HEAD
 
                 if case_id:
                     # Check if case_id exists in the cases table
@@ -47,4 +52,42 @@ def migrate_data(self, destination_cursor, source_data):
                             created_by=created_by,
                         )
 
+=======
+
+                try:
+                    if case_id:
+                        # Check if case_id exists in the cases table
+                        if check_existing_record(destination_cursor,'cases','id', case_id):
+                            scheduled_for = (datetime.today()) 
+                            created_at = parse_to_timestamp(case_details[1])
+                            modified_at = parse_to_timestamp(case_details[3])
+                            created_by = case_details[2]
+
+                            destination_cursor.execute(
+                                """
+                                INSERT INTO public.bookings 
+                                    (id, case_id, scheduled_for, created_at, modified_at)
+                                VALUES (%s, %s, %s, %s, %s )
+                                """,
+                                (id, case_id, scheduled_for, created_at, modified_at),
+                            )
+
+                            audit_entry_creation(
+                                destination_cursor,
+                                table_name="bookings",
+                                record_id=id,
+                                record=case_id,
+                                created_at=created_at,
+                                created_by=created_by,
+                            )
+                except Exception as e:  
+                        self.failed_imports.add(('bookings', id))
+            else:
+                self.failed_imports.add(('bookings', id))
+
+    def log_failed_imports(self, filename='failed_imports_log.txt'):
+        with open(filename, 'w') as file:
+            for table_name, failed_id in self.failed_imports:
+                file.write(f"Table: {table_name}, ID: {failed_id}\n")
+>>>>>>> 90b5173 (converted enum types to uppercase, added in exceptions and functions for failed imports on some tables and added in scripts to count records)
 

bin/migration/tables/capturesessions.py

@@ -4,6 +4,10 @@
 class CaptureSessionManager:
     def __init__(self, source_cursor):
         self.source_cursor = source_cursor
+<<<<<<< HEAD
+=======
+        self.failed_imports = set()
+>>>>>>> 90b5173 (converted enum types to uppercase, added in exceptions and functions for failed imports on some tables and added in scripts to count records)
 
     def get_data(self):
         self.source_cursor.execute("SELECT DISTINCT ON (parentrecuid) * FROM public.recordings")
@@ -49,7 +53,11 @@ def migrate_data(self, destination_cursor, source_data):
             booking_id = temp_recording[2]
 
             if check_existing_record(destination_cursor,'bookings', 'id', booking_id) and not check_existing_record(destination_cursor,'capture_sessions','id', id):
+<<<<<<< HEAD
                 origin = 'pre'
+=======
+                origin = 'PRE'
+>>>>>>> 90b5173 (converted enum types to uppercase, added in exceptions and functions for failed imports on some tables and added in scripts to count records)
                 ingest_address = recording[8] 
                 live_output_url = recording[20]
                 # started_at =  ?
@@ -58,6 +66,7 @@ def migrate_data(self, destination_cursor, source_data):
                 # finished_by_user_id = ?
                 # status = ?
 
+<<<<<<< HEAD
                 destination_cursor.execute(
                     """
                     INSERT INTO public.capture_sessions ( id, booking_id, origin, ingest_address, live_output_url)
@@ -73,3 +82,29 @@ def migrate_data(self, destination_cursor, source_data):
                     record=booking_id,
                 )
 
+=======
+                try:
+                    destination_cursor.execute(
+                        """
+                        INSERT INTO public.capture_sessions ( id, booking_id, origin, ingest_address, live_output_url)
+                        VALUES (%s, %s, %s,%s,%s)
+                        """,
+                        ( id, booking_id, origin, ingest_address, live_output_url),  
+                    )
+
+                    audit_entry_creation(
+                        destination_cursor,
+                        table_name="capture_sessions",
+                        record_id=id,
+                        record=booking_id,
+                    )
+                except Exception as e:  
+                        self.failed_imports.add(('participants', id))
+            else:
+                self.failed_imports.add(('capture_sessions', id))
+
+    def log_failed_imports(self, filename='failed_imports_log.txt'):
+        with open(filename, 'w') as file:
+            for table_name, failed_id in self.failed_imports:
+                file.write(f"Table: {table_name}, ID: {failed_id}\n")
+>>>>>>> 90b5173 (converted enum types to uppercase, added in exceptions and functions for failed imports on some tables and added in scripts to count records)

bin/migration/tables/courts.py

@@ -13,6 +13,7 @@ def get_data(self):
     def migrate_data(self, destination_cursor, source_courts_data):
         # Courts data -  https://cjscommonplatform-my.sharepoint.com/:x:/r/personal/lawrie_baber-scovell2_hmcts_net/_layouts/15/Doc.aspx?sourcedoc=%7B07C83A7F-EF01-4C78-9B02-AEDD443D15A1%7D&file=Courts%20PRE%20NRO.xlsx&wdOrigin=TEAMS-WEB.undefined_ns.rwc&action=default&mobileredirect=true
         court_types = {
+<<<<<<< HEAD
             'Reading Crown Court': ('crown','449','UKJ-South East (England)'),
             'Nottingham Crown Court': ('crown','444','UKF-East Midlands (England)'),
             'Mold Crown Court': ('crown','438','UKL-Wales'),
@@ -22,17 +23,36 @@ def migrate_data(self, destination_cursor, source_courts_data):
             'Kingston-upon-Thames Crown Court': ('crown','427','UKI-London'),
             'Durham Crown Court': ('crown','422','UKC-North East (England)'),
             'Birmingham': ('crown','404','UKG-West Midlands (England)')
+=======
+            'Reading Crown Court': ('CROWN','449','UKJ-South East (England)'),
+            'Nottingham Crown Court': ('CROWN','444','UKF-East Midlands (England)'),
+            'Mold Crown Court': ('CROWN','438','UKL-Wales'),
+            'Liverpool Crown Court': ('CROWN','433','UKD-North West (England)'),
+            'Leeds Youth Court': ('MAGISTRATE','429','UKE-Yorkshire and The Humber'),
+            'Leeds Crown Court': ('CROWN','429','UKE-Yorkshire and The Humber'),
+            'Kingston-upon-Thames Crown Court': ('CROWN','427','UKI-London'),
+            'Durham Crown Court': ('CROWN','422','UKC-North East (England)'),
+            'Birmingham': ('CROWN','404','UKG-West Midlands (England)')
+>>>>>>> 90b5173 (converted enum types to uppercase, added in exceptions and functions for failed imports on some tables and added in scripts to count records)
         }
         batch_courts_data = []
 
         for court in source_courts_data:
             id = court[0]
+<<<<<<< HEAD
             court_info = court_types.get(court[1], ('crown', 'Unknown', 'Unknown'))
+=======
+            court_info = court_types.get(court[1], ('CROWN', 'Unknown', 'Unknown'))
+>>>>>>> 90b5173 (converted enum types to uppercase, added in exceptions and functions for failed imports on some tables and added in scripts to count records)
             court_type, location_code, _ = court_info
             name = court[1]
 
             if not check_existing_record(destination_cursor,'courts', 'id', id ):
+<<<<<<< HEAD
                 batch_courts_data.append((id, court_type, name, location_code))
+=======
+                batch_courts_data.append((id, court_type.upper(), name, location_code))
+>>>>>>> 90b5173 (converted enum types to uppercase, added in exceptions and functions for failed imports on some tables and added in scripts to count records)
 
         if batch_courts_data:
             destination_cursor.executemany(
@@ -56,7 +76,11 @@ def migrate_data(self, destination_cursor, source_courts_data):
                 default_court_id =str(uuid.uuid4())
                 destination_cursor.execute(
                     'INSERT INTO public.courts (id, court_type, name, location_code) VALUES (%s, %s, %s, %s)',
+<<<<<<< HEAD
                     (default_court_id, 'crown', 'Default Court', 'default'),
+=======
+                    (default_court_id, 'CROWN', 'Default Court', 'default'),
+>>>>>>> 90b5173 (converted enum types to uppercase, added in exceptions and functions for failed imports on some tables and added in scripts to count records)
                 )
                 destination_cursor.connection.commit()
 

bin/migration/tables/helpers.py

@@ -44,8 +44,13 @@ def audit_entry_creation(db_connection, table_name, record_id, record, created_a
         "id": str(uuid.uuid4()),
         "table_name": table_name,
         "table_record_id": record_id,
+<<<<<<< HEAD
         "source": "auto",
         "type": "create",
+=======
+        "source": "AUTO",
+        "type": "CREATE",
+>>>>>>> 90b5173 (converted enum types to uppercase, added in exceptions and functions for failed imports on some tables and added in scripts to count records)
         "category": "data_migration",
         "activity": f"{table_name}_record_creation",
         "functional_area": "data_processing",