Skip to content

Commit

Permalink
Configuration Management: Code refactoring
Browse files Browse the repository at this point in the history 
This patch is more of code refactoring for the Configuration Management -
Backup and Restore modules. We've added consitent logging machinery to
enable logging output to both the STDOUT and File, the log location is
set from the system Options module.
  • Loading branch information
@pgathogo
pgathogo committed Mar 21, 2024
1 parent fd97c50 commit 0bc2115
Show file tree
Hide file tree
Showing 10 changed files with 359 additions and 97 deletions.
8 changes: 7 additions & 1 deletion stdm/settings/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@
CURRENT_PROFILE,
RegistryConfig,
ENTITY_BROWSER_RECORD_LIMIT,
ENTITY_SORT_ORDER
ENTITY_SORT_ORDER,
LOG_MODE

)


Expand Down Expand Up @@ -70,6 +72,10 @@ def save_entity_browser_record_limit(limit: int):
reg_config = RegistryConfig()
reg_config.write({ENTITY_BROWSER_RECORD_LIMIT: limit})

def save_log_mode(log_mode: str):
reg_config = RegistryConfig()
reg_config.write({LOG_MODE: log_mode})


def get_entity_sort_details(group_name: str, entity_name: str) -> str:
reg_config = RegistryConfig()
Expand Down
8 changes: 7 additions & 1 deletion stdm/settings/registryconfig.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
ENTITY_BROWSER_RECORD_LIMIT = 'EntityBrowserRecordLimit'
ENTITY_SORT_ORDER = 'EntitySortOrder'
RUN_TEMPLATE_CONVERTER = 'RunTemplateConverter'

LOG_MODE = 'LogMode'

def registry_value(key_name: str):
"""
Expand Down Expand Up @@ -122,6 +122,12 @@ def debug_logging():

return True

def logging_mode():
log_mode = registry_value(LOG_MODE)
if log_mode is None:
return "STDOUT"
return log_mode


def set_debug_logging(state: bool):
"""
Expand Down
130 changes: 83 additions & 47 deletions stdm/ui/backup_restore_handler.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@
QObject,
pyqtSignal,
QDir,
QFile
QFile,
QDateTime
)

from stdm.utils.logging_handlers import (
Expand Down Expand Up @@ -49,8 +50,9 @@
class BackupRestoreHandler(QObject):
update_status = pyqtSignal(str, int)

def __init__(self):
def __init__(self, log_mode: str='STDOUT'):
QObject.__init__(self, None)
self._log_mode = log_mode
self._backup_info_file = ""
self._backup_info = None
self._configuration = None
Expand All @@ -63,7 +65,7 @@ def __init__(self):
self._backup_folder = ""
self._restore_steps = 1

self.logger = MessageLogger()
self._logger = self._make_logger()

def _init_handler(self, backup_info: dict):
self._backup_info = backup_info
Expand All @@ -77,6 +79,16 @@ def _init_handler(self, backup_info: dict):
self._backup_type = backup_type[self.configuration['compressed']]
self._backup_folder = os.path.dirname(self._backup_info_file)

def _make_logger(self) ->MessageLogger:
if self._log_mode == 'FILE':
dtime = QDateTime.currentDateTime().toString('ddMMyyyy_HH.mm')
restore_log_file ='/.stdm/logs/config_restore_{}.log'.format(dtime)
FileHandler.set_filepath(restore_log_file)
return MessageLogger(handler=FileHandler)

if self._log_mode == 'STDOUT':
return MessageLogger(handler=StdOutHandler)

@property
def backup_info_file(self):
return self._backup_info_file
Expand Down Expand Up @@ -124,6 +136,13 @@ def backup_type(self) ->str:
def backup_folder(self) ->str:
return self._backup_folder

def _log_error(self, msg: str):
self._logger.log_error(msg)
self._show_progress(msg)

def _log_info(self, msg: str):
self._logger.log_info(msg)
self._show_progress(msg)

def _read_templates(self, templates: list) ->dict:
data = []
Expand All @@ -147,99 +166,115 @@ def read_backup_info_file(self, json_file: str) ->bool:
def _valid_backup_info(self, backup_info: dict) ->bool:
if len(backup_info) == 0:
msg = f"Invalid backup info file"
self.logger.log_error(msg)
self._log_error(msg)
return False

self._init_handler(backup_info)

# find configuration file (.stc)
config_stc_filepath = f'{self._backup_folder}/{self._config_filename}'
if not os.path.exists(config_stc_filepath):
self.logger.log_error(f"File missing: `{self._config_filename}` ")
err_msg = f"File: `{self._config_filename}` ... Missing. "
self._log_error(err_msg)
return False

self._log_info('Configuration file... Found.')

# Check if DB backup file exists
db_backup_filepath = f'{self._backup_folder}/{self.db_backup_file}'
if not os.path.exists(db_backup_filepath):
self.logger.log_error(f"File missing: `{self.db_backup_file}`")
err_msg = f"File: `{self.db_backup_file}` ... Missing."
self._log_error(err_msg)
return False

self._log_info('DB Backup file... Found.')

# Check if template files exists
found_templates = 0
for profile in self._profiles:
profile['missing_templates'] = []
for template in profile['templates']:
temp_file_path = f"{self._backup_folder}/{template}{'.sdt'}"
if not os.path.exists(temp_file_path):
profile['missing_templates'].append(template)
else:
found_templates += 1

for template in profile['missing_templates']:
self.logger.log_error(f"File missing: `{template}`")
err_msg = f"File: `{template}` ... Missing."
self._log_error(err_msg)

templates_msg = f'[{found_templates}... template found.]'
self._log_info(templates_msg)

return True

def restore_backup(self, username: str, password: str, db_name: str) ->bool:
msg_title = 'Backup Restore'

self.send_message('Authenticating user...')
auth_msg = f'Authenticating user `{username}` ...'
self._log_info(auth_msg)

status, msg, conn_params = self._authenticate_user(username, password)
if not status:
error = f"Authentication failed! Incorrect password.\n\n {msg}"
error = f"Authentication failed! Probably incorrect password.\n\n {msg}"
msg = QApplication.translate(msg_title, error)
self.logger.log_error(error)
self._log_error(error)
return msg, False
else:
msg = f'Authentication successful.'
self.logger.log_info(msg)
msg = f'Authentication... Successful.'
self._log_info(msg)

self.db_name = db_name
db_backup_filepath = f'{self.backup_folder}{"/"}{self.db_backup_file}'

self.send_message('Check if database exists...')
db_msg = f'Check if database: `{db_name}` exists...'
self._log_info(db_msg)

if self._find_database(db_name):
error =f"Database `{db_name}` already exists! Restore aborted."
error =f"Database: `{db_name}` already exists! Restore aborted."
msg = QApplication.translate(msg_title, error)
self.logger.log_error(error)
self._log_error(error)
return msg, False
else:
msg = f'Database `{db_name}` not found.'
self.logger.log_info(msg)
msg = f'Database: `{db_name}`... Not found.'
self._log_info(msg)

# # STEP 0: Create DB
self.send_message('Creating database...')
self._log_info('Creating database...')
if not self._create_database(conn_params, username, password,
db_name):
error = f'Failed to create database! Restore aborted.'
msg = QApplication.translate(msg_title, error)
self.logger.log_error(error)
self._log_error(error)
return msg, False
else:
msg = f'Database `{db_name}` created.'
self.logger.log_info(msg)
msg = f'Database: `{db_name}`... Created.'
self._log_info(msg)

sleep(1)

# STEP 1: Restore database
self.send_message('Restoring database...')
self._log_info('Restoring database...')
if not self._restore_database(conn_params, username, password,
db_name, db_backup_filepath):
error = f"Failed to restore database!"
msg =QApplication.translate(msg_title, error)
self._log_error(error)
return msg, False
else:
msg = f'Database restored successfully.'
self.logger.log_info(msg)
self._log_info(msg)

# STEP 2a: Copy configuration.stc to the 'configurations' folder
self.send_message('Copying configruation file...')
self._log_info('Copying configruation file...')
restore_location = f'{QDir.homePath()}/.stdm/configurations/{self.db_name}'
dir = QDir(restore_location)
if not dir.exists():
dir.mkpath(restore_location)
msg = f'Folder `{restore_location}` created.'
msg = f'Folder: `{restore_location}`... Created.'
else:
msg = f'Folder `{restore_location}` already exists. Folder creation skipped.'
self.logger.log_info(msg)
msg = f'Folder: `{restore_location}`... already exists. Folder creation skipped.'
self._log_info(msg)

src_config_filepath = f'{self.backup_folder}/{self.config_filename}'
dest_config_filepath = f'{restore_location}/{self.config_filename}'
Expand All @@ -248,37 +283,37 @@ def restore_backup(self, username: str, password: str, db_name: str) ->bool:
if not QFile.copy(src_config_filepath, dest_config_filepath):
error = f'Failed to copy configuration file. Restore aborted.'
msg = QApplication.translate(msg_title, error)
self.logger.log_error(error)
self._log_error(error)
return msg, False
else:
msg = f'Configuration file copied successfully.'
self.logger.log_info(msg)
self._log_info(msg)
else:
msg = f'Configuration file already exists. Copy skipped.'
self.logger.log_info(msg)
self._log_info(msg)

# STEP 2b: Copy log file to configurations folder
src_logfile_filepath = f'{self.backup_info_file}'
dest_logfile_filepath = f'{restore_location}/{os.path.basename(self.backup_info_file)}'

self.send_message('Write log file to configurations folder...')
self._log_info('Write log file to configurations folder...')
if not QFile.exists(dest_logfile_filepath):
# Update DB name. Restore might happend in a different database.
self._backup_info['configuration']['database']['name'] = self._db_name

self._create_backup_log(self._backup_info, dest_logfile_filepath)
msg = f'Log file {[dest_logfile_filepath]} created successfully.'
self.logger.log_info(msg)
msg = f'Log file: `{[dest_logfile_filepath]}`... Created successfully.'
self._log_info(msg)
else:
msg = f'Log file already exists. Copy skipped.'
self.logger.log_info(msg)
self._log_info(msg)

# STEP 3: Copy templates
reg_config = RegistryConfig()
templates_path = reg_config.read([COMPOSER_TEMPLATE])
dest_path = templates_path[COMPOSER_TEMPLATE]

self.send_message('Copying templates...')
self._log_info('Copying templates...')
for template in self.templates:
src_temp_filepath = f'{self.backup_folder}{"/"}{template}{".sdt"}'
dest_temp_filepath = f'{dest_path}{"/"}{template}{".sdt"}'
Expand All @@ -287,20 +322,21 @@ def restore_backup(self, username: str, password: str, db_name: str) ->bool:

if len(self.templates) > 0:
msg = f'Templates copied successfully.'
self.logger.log_info(msg)
self._log_info(msg)
else:
msg = f'No templates found.'
self.logger.log_info(msg)
self._log_info(msg)

self._log_info('Restore process... Done.')

self.send_message('Restore process done.')
msg = f'Restore process completed. Backup restored successfully.'
self.logger.log_info(msg)
self._log_info(msg)

return msg, True

def _find_database(self, database: str) ->bool:
msg = f'Finding database `{database}`...'
self.logger.log_info(msg)
msg = f'Finding database: `{database}`...'
self._log_info(msg)

sql = f"SELECT 1 AS found FROM pg_database WHERE datname = '{database}'"
result = _execute(sql)
Expand All @@ -313,8 +349,8 @@ def _create_backup_log(self, log: dict, log_file: str):
json.dump(log, lf, indent=4)

def _authenticate_user(self, user_name: str, password: str) ->tuple[bool, str, DatabaseConnection]:
msg = f'Authenticating user `{user_name}`...'
self.logger.log_info(msg)
msg = f'Authenticating user: `{user_name}`...'
self._log_info(msg)

reg_config = RegistryConfig()
settings = reg_config.read(['Host', 'Database', 'Port'])
Expand Down Expand Up @@ -372,8 +408,8 @@ def _restore_database(self, db_conn_params: DatabaseConnection, user: str,
def _create_database(self, db_conn_params: DatabaseConnection, user: str,
password: str, db_name: str) ->bool:

msg = f'Creating database `{db_name}`...'
self.logger.log_info(msg)
msg = f'Creating database: `{db_name}`...'
self._log_info(msg)

base_pg_folder = self._get_pg_base_folder()
if base_pg_folder == "":
Expand Down Expand Up @@ -408,7 +444,7 @@ def _create_database(self, db_conn_params: DatabaseConnection, user: str,

return True

def send_message(self, msg: str):
def _show_progress(self, msg: str):
self.update_status.emit(msg, self._restore_steps)
self._restore_steps = self._restore_steps + 1
QApplication.processEvents()
Expand Down
Loading

0 comments on commit 0bc2115

Please sign in to comment.