From 5d4032e018edc7c06297cc8035c2391e90bfebb6 Mon Sep 17 00:00:00 2001 From: Johnny Miller Date: Tue, 7 Nov 2023 09:00:45 +0800 Subject: [PATCH] feat($database): support database init when startup --- python_boilerplate/__main__.py | 6 ++-- python_boilerplate/data_migration/__init__.py | 0 .../data_migration/data_migration.py | 8 +++++ .../data_migration/video_game/__init__.py | 0 .../video_game/data_initialization_dml.py | 36 +++++++++++++++++++ python_boilerplate/demo/pandas_usage.py | 2 -- python_boilerplate/message/email.py | 2 +- .../repository/model/video_game.py | 24 +++++++++++++ tests/message/test_email.py | 6 ++-- 9 files changed, 76 insertions(+), 8 deletions(-) create mode 100644 python_boilerplate/data_migration/__init__.py create mode 100644 python_boilerplate/data_migration/data_migration.py create mode 100644 python_boilerplate/data_migration/video_game/__init__.py create mode 100644 python_boilerplate/data_migration/video_game/data_initialization_dml.py create mode 100644 python_boilerplate/repository/model/video_game.py diff --git a/python_boilerplate/__main__.py b/python_boilerplate/__main__.py index 8b828cc..aad4653 100644 --- a/python_boilerplate/__main__.py +++ b/python_boilerplate/__main__.py @@ -32,8 +32,9 @@ from python_boilerplate.configuration.thread_pool_configuration import ( configure as configure_thread_pool, ) -from python_boilerplate.message.email import __init__ +from python_boilerplate.data_migration.data_migration import init_db_data from python_boilerplate.message.email import cleanup as email_cleanup +from python_boilerplate.message.email import init_smtp from python_boilerplate.repository.model.startup_log import StartupLog from python_boilerplate.repository.startup_log_repository import ( retain_startup_log, @@ -62,7 +63,8 @@ def startup() -> None: configure_apscheduler() # Initialization - __init__() + init_smtp() + init_db_data() # Saving startup log # Cannot save startup log in parallel, because the ThreadPoolExecutor won't be able to start another future diff --git a/python_boilerplate/data_migration/__init__.py b/python_boilerplate/data_migration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python_boilerplate/data_migration/data_migration.py b/python_boilerplate/data_migration/data_migration.py new file mode 100644 index 0000000..df44d85 --- /dev/null +++ b/python_boilerplate/data_migration/data_migration.py @@ -0,0 +1,8 @@ +from python_boilerplate.data_migration.video_game.data_initialization_dml import ( + execute as execute_video_game, +) + + +def init_db_data() -> None: + # Executing db init + execute_video_game() diff --git a/python_boilerplate/data_migration/video_game/__init__.py b/python_boilerplate/data_migration/video_game/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python_boilerplate/data_migration/video_game/data_initialization_dml.py b/python_boilerplate/data_migration/video_game/data_initialization_dml.py new file mode 100644 index 0000000..eb6a560 --- /dev/null +++ b/python_boilerplate/data_migration/video_game/data_initialization_dml.py @@ -0,0 +1,36 @@ +from loguru import logger + +from python_boilerplate.demo.pandas_usage import video_games +from python_boilerplate.repository.model.video_game import VideoGame + + +def execute() -> None: + count = VideoGame.select().count() + if count == 1212: + logger.info(f"Count of video_game: {count}") + return + video_game_list = [] + for index, row in video_games.iterrows(): + video_game_list.append( + { + "title": row["Title"], + "handheld": 1 if row["Features.Handheld?"] else 0, + "max_players": row["Features.Max Players"], + "multiplatform": 1 if row["Features.Multiplatform?"] else 0, + "online": 1 if row["Features.Online?"] else 0, + "genres": row["Metadata.Genres"], + "licensed": 1 if row["Metadata.Licensed?"] else 0, + "publishers": row["Metadata.Publishers"], + "sequel": 1 if row["Metadata.Sequel?"] else 0, + "review_score": row["Metrics.Review Score"], + "sales": row["Metrics.Sales"], + "used_price": row["Metrics.Used Price"], + "console": row["Release.Console"], + "rating": row["Release.Rating"], + "re_release": 1 if row["Release.Re-release?"] else 0, + "year": row["Release.Year"], + } + ) + logger.info(f"Length of video_game_list: {len(video_game_list)}") + VideoGame.insert_many(rows=video_game_list).execute() + logger.info(f"Done inserting {VideoGame.select().count()} {VideoGame}") diff --git a/python_boilerplate/demo/pandas_usage.py b/python_boilerplate/demo/pandas_usage.py index b58503d..8022604 100644 --- a/python_boilerplate/demo/pandas_usage.py +++ b/python_boilerplate/demo/pandas_usage.py @@ -7,7 +7,6 @@ from loguru import logger from pandas import DataFrame, DatetimeIndex, Series -from python_boilerplate.__main__ import startup from python_boilerplate.common.asynchronization import async_function from python_boilerplate.common.common_function import get_data_dir, get_resources_dir from python_boilerplate.common.profiling import cpu_profile, elapsed_time, mem_profile @@ -125,5 +124,4 @@ def data_generation() -> None: if __name__ == "__main__": - startup() look_for_sony_published_games() diff --git a/python_boilerplate/message/email.py b/python_boilerplate/message/email.py index f5f0adc..4f92e74 100644 --- a/python_boilerplate/message/email.py +++ b/python_boilerplate/message/email.py @@ -31,7 +31,7 @@ _smtp.login(_sender, _password) -def __init__() -> None: +def init_smtp() -> None: """ Initializes the email module. """ diff --git a/python_boilerplate/repository/model/video_game.py b/python_boilerplate/repository/model/video_game.py new file mode 100644 index 0000000..bf787ac --- /dev/null +++ b/python_boilerplate/repository/model/video_game.py @@ -0,0 +1,24 @@ +from peewee import CharField, DecimalField, IntegerField + +from python_boilerplate.common.orm import peewee_table +from python_boilerplate.repository.model.base_model import BaseModel + + +@peewee_table +class VideoGame(BaseModel): + title = CharField(max_length=200, null=False, index=True) + handheld = IntegerField(null=True) + max_players = IntegerField(null=True) + multiplatform = IntegerField(null=True) + online = IntegerField(null=True) + genres = CharField(max_length=200, null=True) + licensed = IntegerField(null=True) + publishers = CharField(max_length=200, null=True) + sequel = IntegerField(null=True) + review_score = CharField(max_length=200, null=True) + sales = DecimalField(null=True) + used_price = DecimalField(null=True) + console = CharField(max_length=200, null=True) + rating = CharField(max_length=10, null=True) + re_release = IntegerField(null=True) + year = CharField(max_length=10, null=True) diff --git a/tests/message/test_email.py b/tests/message/test_email.py index 533c2e7..d8d1d67 100644 --- a/tests/message/test_email.py +++ b/tests/message/test_email.py @@ -1,11 +1,11 @@ -from python_boilerplate.message.email import __init__, cleanup +from python_boilerplate.message.email import cleanup, init_smtp def test_init() -> None: try: - __init__() + init_smtp() except Exception as ex: - assert False, f"{__init__} raised an exception {ex}" + assert False, f"{init_smtp} raised an exception {ex}" def test_cleanup() -> None: