Training packs (#360)

* Training packs :D

* Training pack interface

* Remove import

* Better display (and creation)

* Allow modification of BaseURL

* Update requirements.txt

* Catch exceptions properly

* Order by match date when selecting replays

* Show create button when none exist

* Fix style

* Fix style

* Remove local link

* Fix rotations

* Add some conditions to shots

* Preview shots in replay viewer

* Fix UI a bit

* Add date picker

* Move replay viewer to top when half screen

* Remove non-standard playlists

* Move to from celery tasks and filter by map/playlist

* Add header and game id

* Add modal to create pack

* Adjust ball speed to better match replays

* Remove kickoffs

* Add beta restriction

* Fix JS style

* Fix Python test

* Fixes

* Remove logging

* Add Crypto to test

* Query fixes

* Add comment with license

* Move things around a bit

* Move more stuff around

* Remove extra commented stuff

* Misc fixes

* Add task id and metric to find task time

* Fix for importerror

* Add compact

* changed coverage ignore

This should ignore files that we do not care about coverage for

* ignore parsing files

* Few fixes

* Raise exception when user has no replays for this

* Add logger

* Catch error with config

* Add training packs test

* Fix pack test

* Fix test (to account for new mag)

.coveragerc

@@ -13,4 +13,5 @@ omit =
     loader.py
     imports_test.py
     helpers/*
-    redis/*
+    redis/*
+    backend/tasks/training_packs/parsing/*

backend/blueprints/spa_api/service_layers/replay/replay_positions.py

@@ -19,12 +19,19 @@ def __init__(self, id_: str,
     @staticmethod
     def create_from_id(id_: str, query_params=None) -> 'ReplayPositions':
         filter_frames = None
+        filter_frame_start = None
+        filter_frame_count = None
         if query_params is not None and 'frame' in query_params:
             filter_frames = query_params['frame']
+        if query_params is not None and 'frame_start' in query_params and 'frame_count' in query_params:
+            filter_frame_start = query_params['frame_start']
+            filter_frame_count = query_params['frame_count']
 
         data_frame = FileManager.get_pandas(id_)
         if filter_frames is not None:
             data_frame = ReplayPositions.filter_frames(filter_frames, data_frame)
+        if filter_frame_start is not None and filter_frame_count is not None:
+            data_frame = ReplayPositions.filter_frames_range(filter_frame_start, filter_frame_count, data_frame)
         protobuf_game = FileManager.get_proto(id_)
 
         cs = ['pos_x', 'pos_y', 'pos_z']
@@ -62,3 +69,7 @@ def process_player_df(game) -> List[ReplayPlayer]:
     @staticmethod
     def filter_frames(frames, data_frame):
         return data_frame.loc[data_frame.index.isin(frames)]
+
+    @staticmethod
+    def filter_frames_range(fmin, count, data_frame):
+        return data_frame.loc[(data_frame.index >= fmin) & (data_frame.index < fmin + count)]

backend/blueprints/spa_api/spa_api.py

@@ -11,6 +11,7 @@
 
 from carball.analysis.utils.proto_manager import ProtobufManager
 from flask import jsonify, Blueprint, current_app, request, send_from_directory, Response
+from sqlalchemy import desc
 from werkzeug.utils import secure_filename, redirect
 
 from backend.blueprints.spa_api.service_layers.homepage.patreon import PatreonProgress
@@ -25,6 +26,7 @@
     player_id, heatmap_query_params
 from backend.database.startup import lazy_get_redis
 from backend.tasks.add_replay import create_replay_task, parsed_replay_processing
+from backend.tasks.celery_tasks import create_training_pack
 from backend.utils.logging import ErrorLogger
 from backend.utils.global_functions import get_current_user_id
 from backend.blueprints.spa_api.service_layers.replay.visualizations import Visualizations
@@ -45,16 +47,18 @@
     REPLAY_BUCKET = config.REPLAY_BUCKET
     PROTO_BUCKET = config.PROTO_BUCKET
     PARSED_BUCKET = config.PARSED_BUCKET
+    TRAINING_PACK_BUCKET = config.TRAINING_PACK_BUCKET
 except:
     print('Not uploading to buckets')
     REPLAY_BUCKET = ''
     PROTO_BUCKET = ''
     PARSED_BUCKET = ''
+    TRAINING_PACK_BUCKET = ''
 
 from backend.blueprints.spa_api.service_layers.stat import get_explanations
 from backend.blueprints.spa_api.service_layers.utils import with_session
 from backend.blueprints.steam import get_vanity_to_steam_id_or_random_response, steam_id_to_profile
-from backend.database.objects import Game, GameVisibilitySetting
+from backend.database.objects import Game, GameVisibilitySetting, TrainingPack
 from backend.database.wrapper.chart.chart_data import convert_to_csv
 from backend.tasks import celery_tasks
 from backend.blueprints.spa_api.errors.errors import CalculatedError, NotYetImplemented, PlayerNotFound, \
@@ -76,6 +80,12 @@
 from backend.blueprints.spa_api.utils.decorators import require_user, with_query_params
 from backend.blueprints.spa_api.utils.query_params_handler import QueryParam, get_query_params
 
+try:
+    from backend.tasks.training_packs.task import TrainingPackCreation
+except (ModuleNotFoundError, ImportError):
+    TrainingPackCreation = None
+    print("Missing config or AES Key and CRC, not creating training packs")
+
 logger = logging.getLogger(__name__)
 
 bp = Blueprint('api', __name__, url_prefix='/api/')
@@ -270,8 +280,12 @@ def api_get_replay_basic_team_stats_download(id_):
 
 
 @bp.route('replay/<id_>/positions')
-@with_query_params(accepted_query_params=[QueryParam(name='frame', type_=int, optional=True, is_list=True),
-                                          QueryParam(name='as_proto', type_=bool, optional=True)])
+@with_query_params(accepted_query_params=[
+    QueryParam(name='frame', type_=int, optional=True, is_list=True),
+    QueryParam(name='frame_start', type_=int, optional=True),
+    QueryParam(name='frame_count', type_=int, optional=True),
+    QueryParam(name='as_proto', type_=bool, optional=True)
+])
 def api_get_replay_positions(id_, query_params=None):
     positions = ReplayPositions.create_from_id(id_, query_params=query_params)
     if query_params is None or 'as_proto' not in query_params:
@@ -514,6 +528,31 @@ def api_handle_error(error: CalculatedError):
     return response
 
 
+@bp.route('/training/create')
+@require_user
+@with_query_params(accepted_query_params=[
+    QueryParam(name="date_start", type_=str, optional=True),
+    QueryParam(name="date_end", type_=str, optional=True)
+])
+def api_create_trainingpack(query_params=None):
+    date_start = None
+    date_end = None
+    if 'date_start' in query_params:
+        date_start = query_params['date_start']
+    if 'date_end' in query_params:
+        date_end = query_params['date_end']
+    task = create_training_pack.delay(get_current_user_id(), 10, date_start, date_end)
+    return better_jsonify({'status': 'Success', 'id': task.id})
+
+
+@bp.route('/training/list')
+@require_user
+@with_session
+def api_find_trainingpack(session=None):
+    player = get_current_user_id()
+    return better_jsonify(TrainingPackCreation.list_packs(player, session))
+
+
 # Homepage
 
 @bp.route('/home/twitch')

backend/database/objects.py

@@ -2,8 +2,8 @@
 import datetime
 import enum
 
-from sqlalchemy import Column, Integer, String, Boolean, Float, ForeignKey, DateTime, Enum, Table, UniqueConstraint, \
-    and_, ForeignKeyConstraint
+from sqlalchemy import Column, Integer, String, Boolean, Float, ForeignKey, DateTime, Enum, UniqueConstraint, \
+    ForeignKeyConstraint, JSON
 from sqlalchemy.dialects import postgresql
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import relationship, validates
@@ -381,3 +381,12 @@ class GameVisibility(DBObjectBase):
     player = Column(String(40), ForeignKey('players.platformid'))
     visibility = Column(Enum(GameVisibilitySetting))
     release_date = Column(DateTime, default=datetime.datetime.max)
+
+
+class TrainingPack(DBObjectBase):
+    __tablename__ = "training_packs"
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    guid = Column(String(40))
+    player = Column(String(40), ForeignKey('players.platformid'), index=True)
+    shots = Column(JSON)
+    creation_date = Column(DateTime, default=datetime.datetime.utcnow)

backend/initial_setup.py

@@ -35,6 +35,11 @@
     users = []
     prod = False
 
+try:
+    from config import BASE_URL
+except ImportError:
+    BASE_URL = 'https://calculated.gg'
+
 
 class CalculatedServer:
 
@@ -111,7 +116,7 @@ def set_up_app_config(app: Flask):
         app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
         app.config['MAX_CONTENT_LENGTH'] = 512 * 1024 * 1024
         app.config['TEMPLATES_AUTO_RELOAD'] = True
-        app.config['BASE_URL'] = 'https://calculated.gg'
+        app.config['BASE_URL'] = 'https://calculated.gg' if BASE_URL is None else BASE_URL
         app.config['REPLAY_DIR'] = os.path.join(BASE_FOLDER, 'data', 'rlreplays')
         app.config['PARSED_DIR'] = os.path.join(BASE_FOLDER, 'data', 'parsed')
         app.config['VERSION'] = CalculatedServer.get_version()

backend/tasks/celery_tasks.py

@@ -1,6 +1,7 @@
 # Celery workers
 import base64
 import json
+import time
 from enum import Enum, auto
 from typing import Dict
 
@@ -18,6 +19,17 @@
 from backend.tasks.add_replay import parse_replay
 from backend.tasks.middleware import DBTask
 from backend.tasks.periodic_stats import calculate_global_distributions
+try:
+    from backend.tasks.training_packs.task import TrainingPackCreation
+    from backend.utils.metrics import METRICS_TRAINING_PACK_CREATION_TIME
+except (ModuleNotFoundError, ImportError):
+    TrainingPackCreation = None
+    print("Missing config or AES Key and CRC, not creating training packs")
+
+try:
+    from backend.tasks.training_packs.training_packs import create_pack_from_replays
+except:
+    pass
 
 celery = Celery(__name__, broker=celeryconfig.broker_url)
 
@@ -32,7 +44,7 @@ def create_celery_config():
 
 @celery.on_after_configure.connect
 def setup_periodic_tasks(sender, **kwargs):
-    sender.add_periodic_task(60 * 60 * 3, calc_global_stats.s(), name='calculate global stats every 3 hrs')
+    sender.add_periodic_task(60 * 60 * 24 * 3, calc_global_stats.s(), name='calculate global stats every 3 days')
     sender.add_periodic_task(60 * 60 * 24, calc_global_dists.s(), name='calculate global dists every day')
     sender.add_periodic_task(60 * 60 * 24, calc_leaderboards.s(), name='calculate leaderboards every day')
     sender.add_periodic_task(60 * 60 * 24 * 3, calc_leaderboards.s(), name='calculate item stats every 3 days')
@@ -96,6 +108,21 @@ def calc_item_stats(self, session=None):
         lazy_get_redis().set('item_stats', json.dumps(results))
 
 
+@celery.task(base=DBTask, bind=True, priority=9)
+def create_training_pack(self, id_, n=10, date_start=None, date_end=None, session=None):
+    if session is None:
+        sess = self.session()
+    else:
+        sess = session
+    start = time.time()
+    url = TrainingPackCreation.create_from_player(id_, n, date_start, date_end, sess)
+    end = time.time()
+    METRICS_TRAINING_PACK_CREATION_TIME.observe(
+        start - end
+    )
+    return url
+
+
 class ResultState(Enum):
     PENDING = auto()
     STARTED = auto()