Tags on upload (#314)

* Bugfix: Fixed expansion panel opening on clicking visibility toggle.

* Remove tag query param (only allow private tag keys for now).

* optimised imports

* Warning fix: replaced Button variant=fab with Fab component.

* Moved theme context.

* Added Tags page with UploadTags selector in UploadForm

* Shortened import

* Added privateKey to returned type of Tags,
removed now-redundant type TagWithPrivateKey,
removed now-redundant getAllTagsWithPrivateKeys

* line wrapping.

* skipped tests

* Update key to camelCase

* Skip test for proto upload with tags using tag name

* Removed tag integration test

* Removed another tag test in integration tests.

* Added some tags to upload.

* Cleaned up private key/id mixup, formatting.

* Propagate private_key private_id fix.

* Propagate private_key private_id fix. 2

* Corrected test request url (missing `/`)

* Try creating all tags only after first replay is uploaded.

* Fixed url

* moved private key creation to only happen once

* added delay

* Arrays start at 0

Fixed bugs in tag creation

* stylin

* except when they start at 1

* Set logging to be more readable.

* fix logging level in celery

* Possibly fix decoding issue

* added logging

* fixed str logging error

* fixed logging

* url encode the keys.

This is important so that the server will accept them.

* Adding logging for all local dev requests

* fix issues with platformid for local players

* Make the first replay upload have user info

* fix imports

* fix typescript issue

* address comments

.travis.yml

@@ -83,7 +83,7 @@ jobs:
         - cd webapp
         - npm start &
         - cd $TRAVIS_BUILD_DIR
-        - celery -A backend.tasks.celery_worker.celery worker --pool=solo -l debug &
+        - celery -A backend.tasks.celery_worker.celery worker --pool=solo -l WARNING &
           # Wait for all other systems to be setup
         - sleep 30
         - pytest tests/integration_tests --cov=./

backend/blueprints/spa_api/errors/errors.py

@@ -91,6 +91,13 @@ class TagError(CalculatedError):
     message = "Exception with tags"
 
 
+class TagKeyError(TagError):
+    status_code = 400
+
+    def __init__(self, tag_key, exception):
+        self.message = f'Unable to decode tag_key; [{tag_key}] ' + str(exception)
+
+
 class TagNotFound(TagError):
     status_code = 404
     message = "Tag not found"

backend/blueprints/spa_api/service_layers/replay/tag.py

@@ -1,37 +1,41 @@
 import base64
+import urllib
 from typing import List, Dict
 
 from backend.blueprints.spa_api.service_layers.utils import with_session
-from backend.utils.logging import ErrorLogger
-from backend.blueprints.spa_api.errors.errors import TagNotFound, PlayerNotFound, TagError
-from backend.database.objects import Tag as DBTag, Player
+from backend.blueprints.spa_api.errors.errors import TagNotFound, TagError, TagKeyError
+from backend.database.objects import Tag as DBTag
 from backend.database.wrapper.tag_wrapper import TagWrapper, DBTagNotFound
 from backend.utils.safe_flask_globals import get_current_user_id
 
 
 class Tag:
-    def __init__(self, name: str, owner: str, db_tag: DBTag = None):
+    def __init__(self, name: str, owner: str, privateKey: str = None, db_tag: DBTag = None):
         super().__init__()
         self.name = name
-        self.owner_id = owner
+        self.ownerId = owner
+        self.privateKey = privateKey
         self.db_tag = db_tag
 
     def to_JSON(self, with_id=False):
         if with_id:
             return {
                 "name": self.name,
-                "owner_id": self.owner_id,
+                "ownerId": self.ownerId,
+                "privateKey": self.privateKey,
                 "tag_id": self.db_tag.id
             }
 
         return {
             "name": self.name,
-            "owner_id": self.owner_id
+            "ownerId": self.ownerId,
+            "privateKey": self.privateKey,
         }
 
     @staticmethod
     def create_from_dbtag(tag: DBTag):
-        return Tag(tag.name, tag.owner, db_tag=tag)
+        private_key = None if tag.private_id is None else Tag.encode_tag(tag.id, tag.private_id)
+        return Tag(tag.name, tag.owner, privateKey=private_key, db_tag=tag)
 
     @staticmethod
     @with_session
@@ -43,13 +47,13 @@ def add_private_key(name: str, private_key: str, session=None, player_id=None):
 
     @staticmethod
     @with_session
-    def create(name: str, session=None, player_id=None, private_key=None) -> 'Tag':
+    def create(name: str, session=None, player_id=None, private_id=None) -> 'Tag':
         """
         Creates a new instance of Tag, add one to the db if it does not exist.
         :param name: Tag name
         :param session: Database session
         :param player_id
-        :param private_key
+        :param private_id
         :return:
         """
         # Check if tag exists
@@ -59,7 +63,7 @@ def create(name: str, session=None, player_id=None, private_key=None) -> 'Tag':
             return tag
         except DBTagNotFound:
             pass
-        dbtag = TagWrapper.create_tag(session, get_current_user_id(player_id=player_id), name, private_key=private_key)
+        dbtag = TagWrapper.create_tag(session, get_current_user_id(player_id=player_id), name, private_id=private_id)
         tag = Tag.create_from_dbtag(dbtag)
         return tag
 
@@ -128,33 +132,30 @@ def get_encoded_private_key(name: str, session=None) -> str:
     @staticmethod
     def encode_tag(tag_id: int, private_id: str) -> str:
         merged = str(tag_id + 1000) + ":" + private_id
-        return base64.b85encode(merged.encode(encoding="utf-8")).decode('utf-8')
+        # b85 encodes it to make it smaller / unreadable followed by a url encode to make it friendly for urls.
+        return urllib.parse.quote(base64.b85encode(merged.encode(encoding="utf-8")).decode('utf-8'))
 
     @staticmethod
     def decode_tag(encoded_key: str):
-        decoded_key_bytes = base64.b85decode(encoded_key)
-        decoded_key = decoded_key_bytes.decode(encoding="utf-8")
+        decoded_key_bytes = base64.b85decode(urllib.parse.unquote(encoded_key).encode('utf-8'))
+
+        try:
+            decoded_key = decoded_key_bytes.decode(encoding="utf-8")
+        except UnicodeDecodeError as e:
+            raise TagKeyError(encoded_key, e)
+
         first_index = decoded_key.find(':')
         tag_id = int(decoded_key[0: first_index]) - 1000
         decoded_private_id = decoded_key[first_index + 1:]
         return tag_id, decoded_private_id
 
+
 @with_session
-def apply_tags_to_game(query_params: Dict[str, any]=None, game_id=None, session=None):
+def apply_tags_to_game(query_params: Dict[str, any] = None, game_id=None, session=None):
     if query_params is None:
         return None
-    if 'tags' not in query_params and 'private_tag_keys' not in query_params:
-        return None
-    tags = query_params['tags'] if 'tags' in query_params else []
+
     private_ids = query_params['private_tag_keys'] if 'private_tag_keys' in query_params else []
-    if len(tags) > 0:
-        player_id = query_params['player_id']
-        if session.query(Player).filter(Player.platformid == player_id).first() is None:
-            ErrorLogger.log_error(PlayerNotFound())
-        else:
-            for tag in tags:
-                created_tag = Tag.create(tag, session=session, player_id=player_id)
-                TagWrapper.add_tag_to_game(session, game_id, created_tag.db_tag)
 
     for private_id in private_ids:
         tag_id, private_key = Tag.decode_tag(private_id)

backend/blueprints/spa_api/spa_api.py

@@ -443,13 +443,13 @@ def api_upload_proto(query_params=None):
 @bp.route('/tag/<name>', methods=["PUT"])
 @require_user
 @with_query_params(accepted_query_params=[
-    QueryParam(name='private_key', type_=str, optional=True)
+    QueryParam(name='private_id', type_=str, optional=True)
 ])
 def api_create_tag(name: str, query_params=None):
-    private_key = None
-    if 'private_key' in query_params:
-        private_key = query_params['private_key']
-    tag = Tag.create(name, private_key=private_key)
+    private_id = None
+    if 'private_id' in query_params:
+        private_id = query_params['private_id']
+    tag = Tag.create(name, private_id=private_id)
     return better_jsonify(tag), 201
 
 

backend/blueprints/spa_api/utils/query_param_definitions.py

@@ -32,11 +32,9 @@ def convert_string_to_enum(string: str) -> T:
 ]
 
 tag_params = [
-    QueryParam(name="tags", optional=True,
-               type_=str, is_list=True,
-               required_siblings=['player_id']),
     QueryParam(name="private_tag_keys", optional=True,
-               tip='This is base 64 encoded it is not the private key directly.',
+               tip='This is only required only if you wish to apply a tag that is owned by another account. ' +
+                   'This is base 64 encoded.',
                type_=str, is_list=True)
 ]
 

backend/blueprints/spa_api/utils/query_params_handler.py

@@ -1,5 +1,4 @@
 from typing import List, Dict, Any, Callable, Optional, Type
-from urllib.parse import urlencode
 
 from flask import Request
 
@@ -151,7 +150,3 @@ def validate(created_query_params) -> Optional[CalculatedError]:
                                                      len(created_query_params[query]),
                                                      len(created_query_params[sibling_query.name]))
     return validate
-
-
-def create_query_string(query_params):
-    return urlencode(query_params)

backend/database/wrapper/player_wrapper.py

@@ -23,7 +23,7 @@ def create_default_player(session=None):
     if player is None:
         player = Player()
 
-    player.platformid = "LOCAL_PLATFORMID" if player is None else player.platformid
+    player.platformid = "LOCAL_PLATFORMID" if player.platformid is None else player.platformid
     player.platformname = 'test user with a really long name but even longer'
     if bool(random.getrandbits(1)):
         player.avatar = "https://media.istockphoto.com/photos/golden-retriever-puppy-looking-up-isolated-on-black-backround-picture-id466614709?k=6&m=466614709&s=612x612&w=0&h=AVW-4RuYXFPXxLBMHiqoAKnvLrMGT9g62SduH2eNHxA="

backend/database/wrapper/tag_wrapper.py

@@ -8,8 +8,8 @@
 
 class TagWrapper:
     @staticmethod
-    def create_tag(session, user_id: str, name: str, private_key: str = None) -> Tag:
-        tag = Tag(name=name, owner=user_id, private_id=private_key)
+    def create_tag(session, user_id: str, name: str, private_id: str = None) -> Tag:
+        tag = Tag(name=name, owner=user_id, private_id=private_id)
         session.add(tag)
         session.commit()
         return tag

backend/initial_setup.py

@@ -21,6 +21,7 @@
 from backend.utils.checks import is_local_dev
 from backend.utils.metrics import MetricsHandler
 from backend.utils.logging import ErrorLogger
+from backend.utils.safe_flask_globals import UserManager
 
 logger = logging.getLogger(__name__)
 logger.info("Setting up server.")

requirements.txt

@@ -22,7 +22,7 @@ kiwisolver==1.0.1
 kombu==4.4
 MarkupSafe==1.0
 matplotlib==2.2.3
-numpy==1.15.1
+numpy==1.17.0
 pandas==0.24.2
 proj==0.1.0
 protobuf==3.6.1

tests/integration_tests/no_react/heatmaps_test.py

@@ -1,4 +1,5 @@
 import json
+import logging
 import time
 
 import requests
@@ -15,6 +16,7 @@ class Test_Heatmaps:
 
     @classmethod
     def setup_class(cls):
+        logging.basicConfig(level=logging.ERROR)
         cls.thread = KillableThread(target=start_server)
         cls.thread.daemon = True
         cls.thread.start()

tests/integration_tests/no_react/test_upload.py

@@ -5,8 +5,7 @@
 import requests
 
 from RLBotServer import start_server
-from backend.database.objects import GameVisibilitySetting, User, Player
-from backend.blueprints.spa_api.errors.errors import PlayerNotFound
+from backend.database.objects import GameVisibilitySetting, Player
 from tests.utils.killable_thread import KillableThread
 from tests.utils.replay_utils import get_complex_replay_list, download_replay_discord
 
@@ -23,6 +22,8 @@ class Test_BasicServerCommands():
 
     @classmethod
     def setup_class(cls):
+        logging.basicConfig(level=logging.ERROR)
+        logger.setLevel(logging.DEBUG)
         cls.thread = KillableThread(target=start_server)
         cls.thread.daemon = True
         cls.thread.start()
@@ -37,21 +38,47 @@ def test_upload_files(self, mock_user, no_errors_are_logged):
 
         replay_list = get_complex_replay_list()[0:4]
 
-        tags = ['TAG1', 'TAG2', 'TAG3', ['TAG4', 'TAG2']]
+        tags = [['TAG1'], ['TAG2'], ['TAG3'], ['TAG4', 'TAG2']]
         privacy = [GameVisibilitySetting.DEFAULT.name,
                    GameVisibilitySetting.PUBLIC.name,
                    GameVisibilitySetting.PRIVATE.name,
                    GameVisibilitySetting.PRIVATE.name]
         users = [
-            'invalid',
             '76561198018756583',
+            'invalid',
             '76561198018756583',
             '76561198018756583'
         ]
 
+        tag_keys = ['invalid_key']
+
+        def create_all_tags():
+            created_tags = []
+            for _tags in tags:
+                keys = []
+                for _tag in _tags:
+                    if _tag not in created_tags:
+                        r = requests.put(LOCAL_URL + f'/api/tag/{_tag}')
+                        r.raise_for_status()
+                        created_tags.append(_tag)
+
+                        # create private id
+                        r = requests.put(LOCAL_URL + f'/api/tag/{_tag}/private_key/{_tag}')
+                        r.raise_for_status()
+                    json = requests.get(LOCAL_URL + f'/api/tag/{_tag}/private_key').json()
+                    keys.append(json)
+                tag_keys.append(keys)
+
         for index, replay_url in enumerate(replay_list):
-            params = {'tags': tags[index], 'visibility': privacy[index], 'player_id': users[index]}
-            logger.debug('Testing:', replay_url)
+            if index == 1:
+                logger.error("CREATING INITIAL TAG DATA")
+                time.sleep(20)
+                create_all_tags()
+                time.sleep(1)
+
+            params = {'visibility': privacy[index], 'player_id': users[index], 'private_tag_keys': tag_keys[index]}
+            logger.debug('TESTING URL:' + str(replay_url))
+            logger.debug('TESTING PARAMS:' + str(params))
             f = download_replay_discord(replay_url)
             r = requests.post(LOCAL_URL + '/api/upload', files={'replays': ('fake_file.replay', f)}, params=params)
             r.raise_for_status()
@@ -68,11 +95,10 @@ def test_upload_files(self, mock_user, no_errors_are_logged):
         assert(int(result) == len(replay_list))
 
         response = requests.get(LOCAL_URL + '/api/tag')
-
         result = json.loads(response.content)
-        assert result[0]['owner_id'] == "76561198018756583"
+        assert result[0]['ownerId'] == "76561198018756583"
         assert result[0]['name'].startswith('TAG')
-        assert len(result) == 3
+        assert len(result) == 4
 
         response = requests.get(LOCAL_URL + '/api/player/76561198018756583/match_history?page=0&limit=10')
         assert response.status_code == 200

tests/integration_tests/no_react/upload_proto_test.py

@@ -1,5 +1,6 @@
 import base64
 import json
+import logging
 import time
 import zlib
 
@@ -17,6 +18,7 @@ class Test_UploadingProtos():
 
     @classmethod
     def setup_class(cls):
+        logging.basicConfig(level=logging.ERROR)
         cls.thread = KillableThread(target=start_server)
         cls.thread.daemon = True
         cls.thread.start()
@@ -37,9 +39,9 @@ def test_upload_proto(self):
             'proto': encoded_proto,
             'pandas': encoded_pandas
         }
-        r = requests.post(LOCAL_URL + '/api/upload/proto', json=obj, params={'tags': ['TAG'],
-                                                                             'visibility': GameVisibilitySetting.PRIVATE.name,
-                                                                             'player_id': proto_game.players[0].id.id})
+        r = requests.post(LOCAL_URL + '/api/upload/proto', json=obj,
+                          params={'visibility': GameVisibilitySetting.PRIVATE.name,
+                                  'player_id': proto_game.players[0].id.id})
 
         r.raise_for_status()
         assert r.status_code == 200
@@ -51,9 +53,11 @@ def test_upload_proto(self):
         response = requests.get(LOCAL_URL + '/api/tag')
 
         result = json.loads(response.content)
-        assert result[0]['owner_id'] == "76561198018756583"
-        assert result[0]['name'].startswith('TAG')
-        assert len(result) == 1
+
+        # TODO: Readd test for tag using private key
+        # assert result[0]['ownerId'] == "76561198018756583"
+        # assert result[0]['name'].startswith('TAG')
+        # assert len(result) == 1
 
         response = requests.get(LOCAL_URL + '/api/player/76561198018756583/match_history?page=0&limit=10')
         assert response.status_code == 200