Merge branch 'master' into 135-document-spritesheetpy

db_viewer.py

@@ -0,0 +1,32 @@
+import sqlite3
+import pandas as pd
+
+def read_table(table: str):
+    with sqlite3.connect('byte_server.db') as db:
+        return pd.read_sql_query(f'SELECT * FROM {table}', db)
+
+def tournaments():
+    return read_table('tournament')
+
+def teams():
+    return read_table('team')
+
+def runs():
+    return read_table('run')
+
+def turns():
+    return read_table('turn')
+
+def submissions():
+    return read_table('submission')
+
+def universities():
+    return read_table('university')
+
+def team_types():
+    return read_table('team_type')
+
+def submission_run_infos():
+    return read_table('submission_run_info')
+
+

requirements.txt

@@ -13,4 +13,5 @@ pydantic~=2.3.0
 fastapi[all]~=0.103.1
 schedule~=1.2.1
 requests~=2.31.0
-urllib3~=2.0.4
+urllib3~=2.0.4
+pandas~=2.1.4

server/client_runner.py

@@ -18,6 +18,7 @@
 from server.models.submission import Submission
 from datetime import datetime
 from queue import Queue
+from sqlalchemy.exc import IntegrityError
 
 import schedule
 
@@ -68,7 +69,7 @@ def __init__(self):
 
         self.version: str = self.get_version_number()
 
-        self.best_run_for_client = {}
+        self.best_run_for_client: dict = {}
         self.runner_temp_dir: str = os.path.join(os.getcwd(), 'server', 'runner_temp')
         self.seed_path: str = os.path.join(self.runner_temp_dir, 'seeds')
 
@@ -117,6 +118,8 @@ def external_runner(self) -> None:
         self.total_number_of_games_for_one_client = self.count_number_of_game_appearances(games)
         self.tournament = self.insert_new_tournament()
 
+        self.delete_turns()
+
         if not os.path.exists(self.runner_temp_dir):
             os.mkdir(self.runner_temp_dir)
 
@@ -147,58 +150,65 @@ def external_runner(self) -> None:
         print('Job completed\n')
 
     def internal_runner(self, submission_tuple, index) -> None:
-        max_score: int = -1
+        score_for_each_submission: dict[int, int] = {}
         results = dict()
-        try:
-            # Run game
-            # Create a folder for this client and seed
-            end_path = os.path.join(self.runner_temp_dir, str(index))
-            if not os.path.exists(end_path):
-                os.mkdir(end_path)
-
-            shutil.copy('launcher.pyz', end_path)
-
-            # Write the clients into the folder
-            for index_2, submission in enumerate(submission_tuple):
-                # runner will run -fn argument, which makes the file name the file name
-                # So we can grab the submission_id out of the results later
-                with open(os.path.join(end_path, f'client_{index_2}_{submission.submission_id}.py'), 'x') as f:
-                    f.write(str(submission.file_txt, 'utf-8'))
-
-            # Determine what seed this run needs based on it's serial index
-            seed_index = index // self.number_of_unique_games
-            logging.info(f'running run {index} for game ({submission_tuple[0].submission_id}, '
-                            f'{submission_tuple[1].submission_id}) using seed index {seed_index}')
-
-            # Copy the seed into the run folder
-            if os.path.exists(os.path.join(self.seed_path, str(seed_index), 'logs', 'game_map.json')):
-                os.mkdir(os.path.join(end_path, 'logs'))
-                shutil.copyfile(os.path.join(self.seed_path, str(seed_index), 'logs', 'game_map.json'),
-                                os.path.join(end_path, 'logs', 'game_map.json'))
 
+        # Run game
+        # Create a folder for this client and seed
+        end_path = os.path.join(self.runner_temp_dir, str(index))
+        if not os.path.exists(end_path):
+            os.mkdir(end_path)
+
+        shutil.copy('launcher.pyz', end_path)
+
+        # Write the clients into the folder
+        for index_2, submission in enumerate(submission_tuple):
+            # runner will run -fn argument, which makes the file name the file name
+            # So we can grab the submission_id out of the results later
+            with open(os.path.join(end_path, f'client_{index_2}_{submission.submission_id}.py'), 'x') as f:
+                f.write(str(submission.file_txt, 'utf-8'))
+
+        # Determine what seed this run needs based on it's serial index
+        seed_index = index // self.number_of_unique_games
+        logging.info(f'running run {index} for game ({submission_tuple[0].submission_id}, '
+                        f'{submission_tuple[1].submission_id}) using seed index {seed_index}')
+
+        # Copy the seed into the run folder
+        if os.path.exists(os.path.join(self.seed_path, str(seed_index), 'logs', 'game_map.json')):
+            os.mkdir(os.path.join(end_path, 'logs'))
+            shutil.copyfile(os.path.join(self.seed_path, str(seed_index), 'logs', 'game_map.json'),
+                            os.path.join(end_path, 'logs', 'game_map.json'))
+
+        try:
             res = self.run_runner(end_path, os.path.join(os.getcwd(), 'server', 'runners', 'runner'))
 
             if os.path.exists(os.path.join(end_path, 'logs', 'results.json')):
                 with open(os.path.join(end_path, 'logs', 'results.json'), 'r') as f:
                     results: dict = json.load(f)
 
         finally:
-            player_sub_ids = [x["file_name"].split("_")[-1] for x in results['players']]
+            player_sub_ids: list[int] = [int(x["file_name"].split("_")[-1]) for x in results['players']]
             run_id: int = self.insert_run(
                 self.tournament.tournament_id,
                 self.index_to_seed_id[seed_index],
                 results)
             for i, result in enumerate(results["players"]):
                 self.insert_submission_run_info(player_sub_ids[i], run_id, result["error"], i,
                                                 result["avatar"]["score"])
+                score_for_each_submission[player_sub_ids[i]] = result["avatar"]["score"]
+
+            # don't store logs with non-eligible teams
+            if any([not submission.team.team_type.eligible for submission in submission_tuple]):
+                return
 
             # Update information in best run dict
             for submission in submission_tuple:
-                if max_score > self.best_run_for_client.get(submission.submission_id, {'score': -2})["score"]:
+                if (score_for_each_submission[submission.submission_id] >
+                        self.best_run_for_client.get(submission.submission_id, {'score': -2})['score']):
                     self.best_run_for_client[submission.submission_id] = {}
                     self.best_run_for_client[submission.submission_id]["log_path"] = os.path.join(end_path, 'logs')
                     self.best_run_for_client[submission.submission_id]["run_id"] = run_id
-                    self.best_run_for_client[submission.submission_id]["score"] = max_score
+                    self.best_run_for_client[submission.submission_id]["score"] = score_for_each_submission[submission.submission_id]
 
     def run_runner(self, end_path, runner) -> bytes:
         """
@@ -296,7 +306,7 @@ def read_best_logs_and_insert(self) -> None:
                 if file in ['game_map.json', 'results.json', 'turn_logs.json']:
                     continue
                 with open(os.path.join(path, file)) as fl:
-                    turn_logs.append(TurnBase(turn_id=0, turn_number=int(file[-9:-5]), run_id=self.best_run_for_client[
+                    turn_logs.append(TurnBase(turn_number=int(file[-9:-5]), run_id=self.best_run_for_client[
                         submission_id]["run_id"], turn_data=bytes(fl.read(), 'utf-8')))
 
             self.insert_logs(turn_logs)
@@ -305,8 +315,13 @@ def insert_logs(self, logs: list[TurnBase]) -> None:
         """
         Inserts logs
         """
-        with DB() as db:
-            crud_turn.create_all(db, logs)
+        try:
+            with DB() as db:
+                crud_turn.create_all(db, logs)
+
+        # do nothing if fails to insert due to records already existing
+        except IntegrityError:
+            ...
 
     def close_server(self) -> None:
         if self.tournament != -1 and not self.tournament.is_finished:
@@ -326,6 +341,10 @@ def delete_runner_temp(self) -> None:
             except PermissionError:
                 continue
 
+    def delete_turns(self) -> None:
+        with DB() as db:
+            crud_turn.delete_all(db)
+
     def return_team_parings(self, submissions: list[Submission]) -> list[tuple[Submission, Submission]]:
         fixtures = list(itertools.permutations(submissions, 2))
         self.number_of_unique_games = len(fixtures)

server/crud/crud_submission.py

@@ -5,6 +5,7 @@
 from sqlalchemy.orm import Session, joinedload
 
 from server.models.submission import Submission
+from server.models.team import Team
 from server.schemas.submission.submission_w_team import SubmissionWTeam
 
 
@@ -132,6 +133,9 @@ def delete(db: Session, id: int, submission: SubmissionWTeam) -> None:
 def get_latest_submission_for_each_team(db: Session) -> list[Type[Submission]]:
     return (db.query(Submission)
             .filter(Submission.submission_id.in_(
-        db.query(func.max(Submission.submission_id))
-        .group_by(Submission.team_uuid)))
+                    db.query(
+                        func.max(Submission.submission_id))
+                    .group_by(Submission.team_uuid)))
+            .options(joinedload(Submission.team)
+                     .joinedload(Team.team_type))
             .all())

server/crud/crud_turn.py

@@ -1,4 +1,5 @@
-from sqlalchemy.orm import Session, joinedload
+from sqlalchemy import and_
+from sqlalchemy.orm import Session, joinedload, Query
 
 from server.models.turn import Turn
 from server.schemas.turn.turn_schema import TurnBase
@@ -13,7 +14,7 @@ def create(db: Session, turn: TurnBase) -> Turn:
     :param turn:
     :return:
     """
-    db_turn: Turn = Turn(**turn.model_dump(exclude={'turn_id'}))
+    db_turn: Turn = Turn(**turn.model_dump())
     db.add(db_turn)
     db.commit()
     db.refresh(db_turn)
@@ -28,21 +29,24 @@ def create_all(db: Session, turns: [TurnBase]) -> None:
 
 
 # read the most recent turn
-def read(db: Session, id: int, eager: bool = False) -> Turn | None:
+def read(db: Session, turn_number: int, run_id: int, eager: bool = False) -> Turn | None:
     """
     This gets information from the Turn table and returns it. Eager loading will determine whether to only return the
     entry in the Turn table or to return it with more information from the tables that it's related to.
     :param db:
-    :param id:
+    :param turn_number:
+    :param run_id:
     :param eager:
     :return:
     """
     return (db.query(Turn)
-            .filter(Turn.turn_id == id)
+            .filter(and_(Turn.turn_number == turn_number,
+                         Turn.run_id == run_id))
             .first() if not eager
             else db.query(Turn)
             .options(joinedload(Turn.run))
-            .filter(Turn.turn_id == id)
+            .filter(and_(Turn.turn_number == turn_number,
+                         Turn.run_id == run_id))
             .first())
 
 
@@ -82,17 +86,19 @@ def read_all_W_filter(db: Session, eager: bool = False, **kwargs) -> [Turn]:
 
 
 # update a turn
-def update(db: Session, id: int, turn: TurnBase) -> Turn | None:
+def update(db: Session, turn_number: int, run_id: int, turn: TurnBase) -> Turn | None:
     """
     This method takes a Turn object and updates the specified Turn in the database with it. If there is nothing to
     update, returns None.
     :param db:
-    :param id:
+    :param turn_number:
+    :param run_id:
     :param turn:
     :return:
     """
     db_turn: Turn | None = (db.query(Turn)
-                            .filter(Turn.turn_id == id)
+                            .filter(and_(Turn.turn_number == turn_number,
+                                         Turn.run_id == run_id))
                             .one_or_none())
     if db_turn is None:
         return
@@ -106,19 +112,32 @@ def update(db: Session, id: int, turn: TurnBase) -> Turn | None:
 
 
 # delete a turn
-def delete(db: Session, id: int, turn_table: TurnBase) -> None:
+def delete(db: Session, turn_number: int, run_id: int) -> None:
     """
     Deletes the specified Turn entity from the database.
     :param db:
-    :param id:
-    :param turn_table:
+    :param turn_number:
+    :param run_id:
     :return: None
     """
     db_turn: Turn | None = (db.query(Turn)
-                            .filter(Turn.turn_id == id)
+                            .filter(and_(Turn.turn_number == turn_number,
+                                         Turn.run_id == run_id))
                             .one_or_none())
     if db_turn is None:
         return
 
     db.delete(db_turn)
     db.commit()
+
+
+def delete_all(db: Session) -> None:
+    """
+    Deletes all turn records from the database.
+    :param db:
+    :return: None
+    """
+    db_turns: Query = db.query(Turn)
+
+    db_turns.delete()
+    db.commit()

server/models/turn.py

@@ -17,9 +17,8 @@ class Turn(Base):
     """
 
     __tablename__: str = 'turn'
-    turn_id: Mapped[int] = mapped_column(Integer(), primary_key=True)
-    turn_number: Mapped[int] = mapped_column(Integer())
-    run_id: Mapped[int] = mapped_column(Integer(), ForeignKey('run.run_id', ondelete='CASCADE'))
+    turn_number: Mapped[int] = mapped_column(Integer(), nullable=False, primary_key=True)
+    run_id: Mapped[int] = mapped_column(Integer(), ForeignKey('run.run_id', ondelete='CASCADE'), primary_key=True)
     turn_data: Mapped[str] = mapped_column(LargeBinary(), nullable=False)
 
     run: Mapped['Run'] = relationship(back_populates='turns', passive_deletes=True)

server/runner_utils.py

@@ -4,5 +4,5 @@
 def worker_main(jobqueue: Queue):
     while not jobqueue.empty():
         job_func = jobqueue.get()
-        job_func[0](job_func[1:])
+        job_func[0](*job_func[1:])
         jobqueue.task_done()

server/schemas/turn/turn_base.py

@@ -5,7 +5,6 @@ class TurnBase(BaseModel):
     """
     All variables that represent columns in the Turn table and their data type.
     """
-    turn_id: int
     turn_number: int
     run_id: int
     turn_data: bytes

server/visualizer_runner.py

@@ -42,12 +42,14 @@ def __init__(self):
 
         self.logs_path: str = os.path.join('server', 'vis_temp')
 
+        self.refresh_vis_temp_folder()
+
         (schedule.every(Config().SLEEP_TIME_SECONDS_BETWEEN_VIS).seconds
          .until(Config().END_DATETIME)
          .do(self.internal_runner))
 
         (schedule.every().day.at(str(Config().END_DATETIME.split()[-1]))
-         .do(self.delete_vis_temp))
+         .do(self.delete_vis_temp_folder))
 
         try:
             while 1:
@@ -71,16 +73,20 @@ def internal_runner(self) -> None:
             print(f'Tournament id: {self.tournament_id}')
         self.visualizer_loop()
 
-    # Delete visual logs path at end of competition
-    def delete_vis_temp(self) -> None:
+    # Refresh visual logs path between tournaments
+    def refresh_vis_temp_folder(self) -> None:
         if not os.path.exists(self.logs_path):
             os.mkdir(self.logs_path)
         else:
             shutil.rmtree(self.logs_path)
             os.mkdir(self.logs_path)
 
+    def delete_vis_temp_folder(self) -> None:
+        if os.path.exists(self.logs_path):
+            shutil.rmtree(self.logs_path)
+
     def get_latest_log_files(self, tournament: Tournament) -> None:
-        self.delete_vis_temp()
+        self.refresh_vis_temp_folder()
 
         print("Getting latest log files")
         run: Run