Added API resources and data models

mephisto/abstractions/providers/prolific/prolific_datastore.py

@@ -13,7 +13,6 @@
 
 from mephisto.abstractions.databases.local_database import is_unique_failure
 from mephisto.abstractions.providers.prolific.provider_type import PROVIDER_TYPE
-from mephisto.abstractions.providers.prolific.prolific_utils import get_prolific_api_key
 from mephisto.utils.logger_core import get_logger
 
 CREATE_REQUESTERS_TABLE = """
@@ -64,7 +63,7 @@
 class ProlificDatastore:
     def __init__(self, datastore_root: str):
         """Initialize local storage of active agents, connect to the database"""
-        self.session_storage: Dict[str, Any] = {}  # TODO (FB-3): Implement type
+        self.session_storage: Dict[str, Any] = {}  # TODO (#1008): Implement type
         self.agent_data: Dict[str, Dict[str, Any]] = {}
         self.table_access_condition = threading.Condition()
         self.conn: Dict[int, sqlite3.Connection] = {}
@@ -239,14 +238,13 @@ def get_unit_expired(self, unit_id: str) -> bool:
             results = c.fetchall()
             return bool(results[0]["is_expired"])
 
-    def get_session_for_requester(self, requester_name: str) -> Any:  # TODO (FB-3): Implement type
+    def get_session_for_requester(self, requester_name: str) -> Any:  # TODO (#1008): Implement type
         """
         Either create a new session for the given requester or return
         the existing one if it has already been created
         """
         if requester_name not in self.session_storage:
-            session = None  # TODO (FB-3): Implement client
-            session.api_key = os.environ.get('PROLIFIC_API_KEY', None) or get_prolific_api_key()
+            session = None  # TODO (#1008): Implement client
             self.session_storage[requester_name] = session
 
         return self.session_storage[requester_name]
@@ -261,7 +259,7 @@ def get_client_for_requester(self, requester_name: str) -> Any:
     def get_qualification_mapping(self, qualification_name: str) -> Optional[sqlite3.Row]:
         """
         Get the mapping between Mephisto qualifications and Prolific qualifications
-        (Surger Teams https://app.surgehq.ai/docs/api#surger-teams) # TODO (FB-3): Change doc
+        # TODO (#1008): Change doc
         """
         with self.table_access_condition:
             conn = self._get_connection()

mephisto/abstractions/providers/prolific/prolific_utils.py

@@ -16,39 +16,28 @@
 
 from mephisto.abstractions.crowd_provider import ProviderArgs
 from mephisto.utils.logger_core import get_logger
+from mephisto.utils.prolific_api.base_api_resource import CREDENTIALS_CONFIG_DIR
+from mephisto.utils.prolific_api.base_api_resource import CREDENTIALS_CONFIG_PATH
+from mephisto.utils import prolific_api
 
 PROLIFIC_BUDGET = 100000.0
-PROLIFIC_CONFIG_DIR = '~/.prolific/'
-PROLIFIC_CONFIG_PATH = os.path.join(PROLIFIC_CONFIG_DIR, 'credentials')
 
 logger = get_logger(name=__name__)
 
 
-def get_prolific_api_key() -> Union[str, None]:
-    credentials_path = os.path.expanduser(PROLIFIC_CONFIG_PATH)
-    if os.path.exists(credentials_path):
-        with open(credentials_path, 'r') as f:
-            api_key = f.read().strip()
-            return api_key
-    return None
-
-
-PROLIFIC_API_KEY = os.environ.get('PROLIFIC_API_KEY', None) or get_prolific_api_key()
-
-
 def check_credentials(*args, **kwargs) -> bool:
     """Check whether API KEY is correct"""
-    # TODO (FB-3): Implement
+    # TODO (#1008): Implement
     pass
 
 
 def setup_credentials(
     profile_name: str, register_args: Optional[ProviderArgs],
 ) -> bool:
-    if not os.path.exists(os.path.expanduser(PROLIFIC_CONFIG_DIR)):
-        os.mkdir(os.path.expanduser(PROLIFIC_CONFIG_DIR))
+    if not os.path.exists(os.path.expanduser(CREDENTIALS_CONFIG_DIR)):
+        os.mkdir(os.path.expanduser(CREDENTIALS_CONFIG_DIR))
 
-    with open(os.path.expanduser(PROLIFIC_CONFIG_PATH), 'w') as f:
+    with open(os.path.expanduser(CREDENTIALS_CONFIG_PATH), 'w') as f:
         f.write(register_args.api_key)
 
     return True
@@ -60,25 +49,25 @@ def check_balance(*args, **kwargs) -> Union[float, int]:
     requester account, returns True if the balance is greater than
     balance_needed
     """
-    # TODO (FB-3): Implement
+    # TODO (#1008): Implement
     return PROLIFIC_BUDGET
 
 
 def delete_qualification(
-    client: Any, # TODO (FB-3): Implement
+    client: prolific_api,  # TODO (#1008): Implement
     qualification_id: str,
 ) -> None:
     """Deletes a qualification by id"""
-    client.Team.delete(qualification_id)
+    client.ParticipantGroups.delete(qualification_id)
 
 
 def find_qualification(
-    client: Any,  # TODO (FB-3): Implement
+    client: prolific_api,  # TODO (#1008): Implement
     qualification_name: str,
 ) -> Tuple[bool, Optional[str]]:
     try:
-        qualifications = []  # TODO (FB-3): Implement
-    except Exception:  # TODO (FB-3): Implement
+        qualifications = []  # TODO (#1008): Implement
+    except Exception:  # TODO (#1008): Implement
         logger.exception(f'Could not receive a qualifications')
         raise
 
@@ -90,7 +79,7 @@ def find_qualification(
 
 
 def find_or_create_qualification(
-    client: Any,  # TODO (FB-3): Implement
+    client: prolific_api,  # TODO (#1008): Implement
     qualification_name: str,
     description: str,
 ) -> Optional[str]:
@@ -100,9 +89,9 @@ def find_or_create_qualification(
         return qualification_id
 
     try:
-        # TODO (FB-3): Implement
+        # TODO (#1008): Implement
         qualification = None
-    except Exception:  # TODO (FB-3): Implement
+    except Exception:  # TODO (#1008): Implement
         logger.exception(
             f'Could not create a qualification with name "{qualification_name}" '
             f'and description "{description}"'
@@ -113,7 +102,7 @@ def find_or_create_qualification(
 
 
 def create_study(
-    client: Any,  # TODO (FB-3): Implement
+    client: prolific_api,  # TODO (#1008): Implement
     task_args: "DictConfig",
     qualifications: List[Dict[str, Any]],
 ) -> str:
@@ -122,9 +111,9 @@ def create_study(
     instructions = task_args.task_description
 
     try:
-        # TODO (FB-3): Implement
+        # TODO (#1008): Implement
         pass
-    except Exception:  # TODO (FB-3): Implement
+    except Exception:  # TODO (#1008): Implement
         logger.exception(
             f'Could not create a Study with name "{name}" '
             f'and instructions "{instructions}"'

mephisto/utils/prolific_api/__init__.py

@@ -3,3 +3,9 @@
 # Copyright (c) Facebook, Inc. and its affiliates.
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
+
+from .participant_groups import ParticipantGroups
+from .projects import Projects
+from .studies import Studies
+from .users import Users
+from .workspaces import Workspaces

mephisto/utils/prolific_api/base_api_resource.py

@@ -6,6 +6,7 @@
 
 import os
 from typing import Optional
+from typing import Union
 from urllib.parse import urljoin
 
 import requests
@@ -17,12 +18,25 @@
 from .exceptions import ProlificException
 from .exceptions import ProlificRequestError
 
-API_KEY = os.environ.get('PROLIFIC_API_KEY', '')
 BASE_URL = os.environ.get('PROLIFIC_BASE_URL', 'https://api.prolific.co/api/v1/')
+CREDENTIALS_CONFIG_DIR = '~/.surge_ai/'
+CREDENTIALS_CONFIG_PATH = os.path.join(CREDENTIALS_CONFIG_DIR, 'credentials')
 
 logger = get_logger(name=__name__)
 
 
+def get_surge_ai_api_key() -> Union[str, None]:
+    credentials_path = os.path.expanduser(CREDENTIALS_CONFIG_PATH)
+    if os.path.exists(credentials_path):
+        with open(credentials_path, 'r') as f:
+            api_key = f.read().strip()
+            return api_key
+    return None
+
+
+API_KEY = os.environ.get('PROLIFIC_API_KEY', '') or get_surge_ai_api_key()
+
+
 class HTTPMethod:
     GET = 'get'
     POST = 'post'
@@ -69,7 +83,9 @@ def _base_request(
                 raise ProlificException('Invalid HTTP method.')
 
             response.raise_for_status()
-            return response.json()
+            json = response.json()
+            logger.debug(f'Response: {json}')
+            return json
 
         except requests.exceptions.HTTPError as err:
             logger.error(f'Request error: {str(err)}')

mephisto/utils/prolific_api/constants.py

@@ -0,0 +1 @@
+EMAIL_FORMAT = '^\\S+@\\S+\\.\\S+$'  # Simple email format checking

mephisto/utils/prolific_api/data_models/__init__.py

@@ -4,6 +4,9 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 
+from .participant import Participant
 from .participant_group import ParticipantGroup
+from .project import Project
 from .study import Study
 from .user import User
+from .workspace import Workspace

mephisto/utils/prolific_api/data_models/base_model.py

@@ -10,7 +10,7 @@
 class BaseModel:
     id: str
 
-    _schema = {
+    schema = {
         'type' : 'object',
         'properties' : {
             'id' : {
@@ -19,9 +19,24 @@ class BaseModel:
         },
     }
 
+    required_schema_fields = []
+    id_field_name = 'id'
+
     def __init__(self, **data):
-        validate(instance=data, schema=self._schema)
+        schema = dict(self.schema)
+
+        # Validate on required fields if object is intended to be created
+        if self.id_field_name not in data:
+            schema['required'] = self.required_schema_fields
+
+        validate(instance=data, schema=schema)
         self.__dict__ = data
 
+    def __str__(self) -> str:
+        return f'{self.__class__.__name__} {getattr(self, "id", "")}'
+
+    def __repr__(self) -> str:
+        return f'<{self.__str__()}>'
+
     def to_dict(self) -> dict:
         return self.__dict__

mephisto/utils/prolific_api/data_models/participant.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+
+# Copyright (c) Facebook, Inc. and its affiliates.
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from .base_model import BaseModel
+
+
+class Participant(BaseModel):
+    participant_id: str
+    datetime_created: str
+
+    schema = {
+        'type' : 'object',
+        'properties' : {
+            'participant_id' : {
+                'type' : 'string',
+            },
+            'datetime_created' : {
+                'type' : 'string',
+            },
+        },
+    }
+
+    id_field_name = 'participant_id'
+
+    def __str__(self) -> str:
+        return f'{self.__class__.__name__} {self.participant_id}'

mephisto/utils/prolific_api/data_models/participant_group.py

@@ -4,17 +4,64 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 
+from typing import Dict
+from typing import List
+
 from .base_model import BaseModel
 
 
 class ParticipantGroup(BaseModel):
     id: str
+    name: str
+    project_id: str
+    participant_count: int
+    feeder_studies: List[Dict]
 
-    _schema = {
+    schema = {
         'type' : 'object',
         'properties' : {
             'id' : {
                 'type' : 'string',
             },
+            'project_id' : {
+                'type' : 'string',
+            },
+            'name' : {
+                'type' : 'string',
+            },
+            'participant_count' : {
+                'type' : 'number',
+            },
+            'feeder_studies' : {
+                'type' : 'array',
+                'items': {
+                    'type': 'object',
+                    'properties': {
+                        'id': {
+                            'type': 'string',
+                        },
+                        'name': {
+                            'type': 'string',
+                        },
+                        'internal_name': {
+                            'type': 'string',
+                        },
+                        'status': {
+                            'type': 'string',
+                        },
+                        'completion_codes' : {
+                            'type' : 'array',
+                        },
+                    },
+                },
+            },
         },
     }
+
+    required_schema_fields = [
+        'project_id',
+        'name',
+    ]
+
+    def __str__(self) -> str:
+        return f'{self.__class__.__name__} {self.id} {self.name}'