Merge branch 'master' into attachment-support

README.md

@@ -9,7 +9,7 @@
 **aiosnow** is a Python [asyncio](https://docs.python.org/3/library/asyncio.html) library for interacting with ServiceNow programmatically. It hopes to be:
 
 - Convenient: A good deal of work is put into making **aiosnow** flexible and easy to use.
-- Performant: Remote API calls uses non-blocking sockets tracked by an event loop, allowing large amounts of lightweight request tasks to run concurrently.
+- Performant: Uses non-blocking I/O to allow large amounts of API request tasks to run concurrently while being friendly on system resources.
 - Modular: Core functionality is componentized into modules that are built with composability and extensibility in mind.
 
 *Example code*

aiosnow/models/common/base.py

@@ -17,7 +17,7 @@
     methods,
 )
 
-from .schema import ModelSchema, Nested
+from .schema import ModelSchema, ModelSchemaMeta, Nested
 from .schema.fields import BaseField
 
 req_cls_map = {
@@ -30,36 +30,33 @@
 
 class BaseModelMeta(type):
     def __new__(mcs, name: str, bases: tuple, attrs: dict) -> Any:
-        fields = {}
+        attrs["fields"] = fields = {}
+        base_members = {}
 
         for base in bases:
-            fields.update(base.schema_cls._declared_fields)
-
-        for key, value in attrs.copy().items():
-            if isinstance(value, BaseField):
-                fields[key] = value
-                fields[key].name = key
-            elif isinstance(value, marshmallow.schema.SchemaMeta):
-                fields[key] = Nested(key, value, allow_none=True, required=False)
-            else:
-                continue
-
-            # Do not allow override of base members with schema Field attributes.
-            for base in bases:
-                existing_member = getattr(base, key, None)
-                if existing_member is not None and not issubclass(
-                    existing_member.__class__,
-                    (BaseField, marshmallow.schema.SchemaMeta),
-                ):
+            base_members.update(
+                {
+                    k: v
+                    for k, v in base.__dict__.items()
+                    if not isinstance(v, (BaseField, Nested, ModelSchemaMeta))
+                }
+            )
+            inherited_fields = getattr(base.schema_cls, "_declared_fields")
+            fields.update(inherited_fields)
+
+        for k, v in attrs.items():
+            if isinstance(v, (BaseField, Nested, ModelSchemaMeta)):
+                if k in base_members.keys():
                     raise InvalidFieldName(
-                        f"Field :{name}.{key}: conflicts with a base member, name it something else. "
+                        f"Field :{name}.{k}: conflicts with a base member, name it something else. "
                         f"The Field :attribute: parameter can be used to give a field an alias."
                     )
 
-        attrs["schema_cls"] = type(name + "Schema", (ModelSchema,), fields)
-        cls = super().__new__(mcs, name, bases, attrs)
+                fields[k] = v
 
-        return cls
+        # Create the Model Schema
+        attrs["schema_cls"] = type(name + "Schema", (ModelSchema,), attrs["fields"])
+        return super().__new__(mcs, name, bases, attrs)
 
 
 class BaseModel(metaclass=BaseModelMeta):
@@ -74,10 +71,8 @@ class BaseModel(metaclass=BaseModelMeta):
     def __init__(self, client: Client):
         self._client = client
         self.fields = dict(self.schema_cls.fields)
-        self.nested_fields = {
-            n: f for n, f in self.fields.items() if isinstance(f, Nested)
-        }
         self.schema = self.schema_cls(unknown=marshmallow.EXCLUDE)
+        self.nested_fields = getattr(self.schema, "nested_fields")
         self._primary_key = getattr(self.schema, "_primary_key")
 
     @property

aiosnow/models/common/schema/base.py

@@ -1,4 +1,3 @@
-import warnings
 from typing import Any, Iterable, Tuple, Union
 
 import marshmallow
@@ -19,39 +18,42 @@
 
 class ModelSchemaMeta(marshmallow.schema.SchemaMeta):
     def __new__(mcs, name: str, bases: tuple, attrs: dict) -> Any:
-        attrs["fields"] = fields = {
-            k: v for k, v in attrs.items() if isinstance(v, (BaseField, Nested))
-        }
-        cls = super().__new__(mcs, name, bases, attrs)
-        for k, v in fields.items():
-            setattr(cls, k, v)
-
-        return cls
-
-
-class ModelSchema(marshmallow.Schema, metaclass=ModelSchemaMeta):
-    @property
-    def _primary_key(self) -> Union[str, None]:
-        pks = self._pk_candidates
+        fields = attrs["fields"] = {}
+        nested_fields = attrs["nested_fields"] = {}
+        pks = []
+
+        for k, v in attrs.items():
+            if isinstance(v, BaseField):
+                if v.is_primary:
+                    pks.append(k)
+
+                fields[k] = v
+                fields[k].name = k
+            elif isinstance(v, ModelSchemaMeta):
+                fields[k] = Nested(k, v, allow_none=True, required=False)
+                nested_fields.update({k: fields[k]})
+            else:
+                continue
 
-        if len(pks) > 1:
+        if len(pks) == 1:
+            attrs["_primary_key"] = pks[0]
+        elif len(pks) == 0:
+            attrs["_primary_key"] = None
+        elif len(pks) > 1:
             raise SchemaError(
                 f"Multiple primary keys (is_primary) supplied "
-                f"in {self.__class__.__name__}. Maximum allowed is 1."
+                f"in {name}. Maximum allowed is 1."
             )
-        elif len(pks) == 0:
-            return None
 
-        return pks[0]
+        cls = super().__new__(mcs, name, bases, {**attrs, **fields})
+
+        for k, v in fields.items():
+            setattr(cls, k, v)
+
+        return cls
 
-    @property
-    def _pk_candidates(self) -> list:
-        return [
-            n
-            for n, f in self.fields.items()
-            if isinstance(f, BaseField) and f.is_primary is True
-        ]
 
+class ModelSchema(marshmallow.Schema, metaclass=ModelSchemaMeta):
     @marshmallow.pre_load
     def _load_response(self, data: Union[list, dict], **_: Any) -> Union[list, dict]:
         """Load response content
@@ -85,12 +87,6 @@ def __load_response(self, content: dict) -> Iterable[Tuple[str, str]]:
         for key, value in content.items():
             field = self._declared_fields.get(key, None)
 
-            if not field:
-                warnings.warn(
-                    f"Unexpected field in response content: {key}, skipping..."
-                )
-                continue
-
             if isinstance(field, BaseField):
                 if isinstance(value, dict) and {"value", "display_value"} <= set(
                     value.keys()

aiosnow/request/base.py

@@ -21,11 +21,11 @@ class BaseRequest(ABC):
     log = logging.getLogger("aiosnow.request")
 
     def __init__(
-        self, api_url: str, session: Session, fields: list = None,
+        self, api_url: str, session: Session, fields: dict = None,
     ):
         self.api_url = api_url
         self.session = session
-        self.fields = [str(f) for f in fields or []]
+        self.fields = fields or {}
         self.url_segments: List[str] = []
         self.headers_default = {"Content-type": CONTENT_TYPE}
         self._req_id = f"REQ_{hex(int(round(time.time() * 1000)))}"

aiosnow/request/get.py

@@ -1,16 +1,14 @@
-from __future__ import annotations
-
-from typing import Any, Dict, Iterable, Union
+from time import time
+from typing import Any, Generator, Union
 from urllib.parse import urlparse
 
 from . import methods
 from .base import BaseRequest
 
-_cache: dict = {}
-
 
 class GetRequest(BaseRequest):
     _method = methods.GET
+    _cache: dict = {}
 
     def __init__(
         self,
@@ -19,10 +17,11 @@ def __init__(
         limit: int = 10000,
         offset: int = 0,
         query: str = None,
+        cache_secs: int = 20,
         **kwargs: Any,
     ):
-        self.nested_fields = nested_fields or {}
-        self.nested_attrs = list(self._nested_attrs)
+        self.nested_fields = list(self._nested_with_path(nested_fields or {}, []))
+        self._cache_secs = cache_secs
         self._limit = offset + limit
         self._offset = offset
         self.query = query
@@ -42,56 +41,76 @@ def offset(self) -> int:
     def limit(self) -> int:
         return self._limit
 
-    @property
-    def _nested_attrs(self) -> Iterable:
-        for field in self.nested_fields.values():
-            yield from field.nested.fields.keys()
+    def _nested_with_path(self, fields: dict, path_base: list) -> Generator:
+        path = path_base or []
+
+        for k, v in fields.items():
+            if not hasattr(v, "nested"):
+                continue
+
+            yield path + [k], k, v.schema
+            yield from list(
+                self._nested_with_path(v.schema.fields, path_base=path + [k])
+            )
+
+    async def __expand_document(self, document: dict) -> dict:
+        for path, field_name, schema in self.nested_fields:
+            if not path or not isinstance(path, list):
+                continue
+
+            target_field = path[-1]
+            sub_document = document.copy()
+
+            for name in path[:-1]:
+                sub_document = sub_document[name]
 
-    async def _expand_nested(
+            if not sub_document.get(target_field):
+                continue
+            elif "link" not in sub_document[target_field]:
+                continue
+
+            nested_data = await self.get_cached(
+                sub_document[target_field]["link"], fields=schema.fields.keys()
+            )
+            sub_document[field_name] = nested_data
+            document.update(sub_document)
+
+        return document
+
+    async def _expand_document(
         self, content: Union[dict, list, None]
     ) -> Union[dict, list, None]:
         if not self.nested_fields:
             pass
         elif isinstance(content, dict):
-            nested = await self._resolve_nested(content)
-            content.update(nested)
+            content = await self.__expand_document(content)
         elif isinstance(content, list):
             for idx, record in enumerate(content):
-                nested = await self._resolve_nested(record)
-                content[idx].update(nested)
+                content[idx] = await self._expand_document(record)
 
         return content
 
-    async def _resolve_nested(self, content: dict) -> dict:
-        nested: Dict[Any, Any] = {}
-        nested_attrs = list(self.nested_attrs)
-
-        for field_name in self.nested_fields.keys():
-            item = content[field_name]
-            if not item or not item["display_value"]:
-                continue
-            elif "link" not in item:
-                nested[field_name] = item
-                continue
-
-            nested[field_name] = await self.get_cached(item["link"], nested_attrs)
-
-        return nested
+    async def get_cached(self, url: str, fields: list = None) -> dict:
+        cache_key = hash(url + "".join(fields or []))
+        record_id = urlparse(url).path.split("/")[-1]
 
-    async def get_cached(self, url: str, fields: list) -> dict:
-        if url not in _cache:
-            record_id = urlparse(url).path.split("/")[-1]
+        if (
+            cache_key in self._cache
+            and self._cache[cache_key][1] > time() - self._cache_secs
+        ):
+            self.log.debug(f"Feching {record_id} from cache")
+        else:
             request = GetRequest(url, self.session, fields=fields)
             response = await request._send(method=methods.GET)
             self.log.debug(f"Caching response for: {record_id}")
-            _cache[url] = response.data
+            self._cache[cache_key] = response.data, time()
 
-        return _cache[url]
+        return self._cache[cache_key][0]
 
     async def send(self, *args: Any, resolve: bool = True, **kwargs: Any) -> Any:
         response = await self._send(**kwargs)
         if resolve:
-            response.data = await self._expand_nested(response.data)
+            response.data = await self._expand_document(response.data)
 
         return response