✨ Pagination with ordering (#1)

* ✨ add `order_by` and `descending` options to query / scan and fetch_all methods Signed-off-by: ff137 <[email protected]> * ✨ add `order_by` and `descending` options to PaginatedQuerySchema Signed-off-by: ff137 <[email protected]> * ✨ modify `get_limit_offset` to `get_paginated_query_params` Signed-off-by: ff137 <[email protected]> * ✨ add ordering to InMemoryStorage scan and fetch_all methods Signed-off-by: ff137 <[email protected]> * 🚧 test in-progress aries-askar PR: openwallet-foundation/askar#291 Signed-off-by: ff137 <[email protected]> * ⬆️ Update lock file Signed-off-by: ff137 <[email protected]> * 🎨 fix ruff warning Signed-off-by: ff137 <[email protected]> * ✅ fix assertions Signed-off-by: ff137 <[email protected]> * 🚧 test aries-askar with TestPyPI package Signed-off-by: ff137 <[email protected]> * 🚧 test latest askar testpypi package Signed-off-by: ff137 <[email protected]> * 🎨 Update order_by description and default value. Include in schema Signed-off-by: ff137 <[email protected]> * ⬆️ Update aries-askar test pypi package to pre-orjson feat release Signed-off-by: ff137 <[email protected]> --------- Signed-off-by: ff137 <[email protected]>
didx-xyz · Nov 14, 2024 · 374df29 · 374df29
1 parent b7d27fa
commit 374df29
Show file tree

Hide file tree

Showing 14 changed files with 506 additions and 423 deletions.
diff --git a/acapy_agent/messaging/models/base_record.py b/acapy_agent/messaging/models/base_record.py
@@ -293,6 +293,8 @@ async def query(
         *,
         limit: Optional[int] = None,
         offset: Optional[int] = None,
+        order_by: Optional[str] = None,
+        descending: bool = False,
         post_filter_positive: Optional[dict] = None,
         post_filter_negative: Optional[dict] = None,
         alt: bool = False,
@@ -304,6 +306,8 @@ async def query(
             tag_filter: An optional dictionary of tag filter clauses
             limit: The maximum number of records to retrieve
             offset: The offset to start retrieving records from
+            order_by: An optional field by which to order the records.
+            descending: Whether to order the records in descending order.
             post_filter_positive: Additional value filters to apply matching positively
             post_filter_negative: Additional value filters to apply matching negatively
             alt: set to match any (positive=True) value or miss all (positive=False)
@@ -327,11 +331,15 @@ async def query(
                 tag_query=tag_query,
                 limit=limit,
                 offset=offset,
+                order_by=order_by,
+                descending=descending,
             )
         else:
             rows = await storage.find_all_records(
                 type_filter=cls.RECORD_TYPE,
                 tag_query=tag_query,
+                order_by=order_by,
+                descending=descending,
             )
 
         num_results_post_filter = 0  # used if applying pagination post-filter

diff --git a/acapy_agent/messaging/models/paginated_query.py b/acapy_agent/messaging/models/paginated_query.py
@@ -4,6 +4,7 @@
 
 from aiohttp.web import BaseRequest
 from marshmallow import fields
+from marshmallow.validate import OneOf
 
 from ...messaging.models.openapi import OpenAPISchema
 from ...storage.base import DEFAULT_PAGE_SIZE, MAXIMUM_PAGE_SIZE
@@ -31,18 +32,40 @@ class PaginatedQuerySchema(OpenAPISchema):
         metadata={"description": "Offset for pagination", "example": 0},
         error_messages={"validator_failed": "Value must be 0 or greater"},
     )
+    order_by = fields.Str(
+        required=False,
+        load_default="id",
+        validate=OneOf(["id"]),  # only one possible column supported in askar
+        metadata={
+            "description": (
+                'The column to order results by. Only "id" is currently supported.'
+            )
+        },
+        error_messages={"validator_failed": '`order_by` only supports column "id"'},
+    )
+    descending = fields.Bool(
+        required=False,
+        load_default=False,
+        metadata={"description": "Order results in descending order if true"},
+    )
 
 
-def get_limit_offset(request: BaseRequest) -> Tuple[int, int]:
-    """Read the limit and offset query parameters from a request as ints, with defaults.
+def get_paginated_query_params(request: BaseRequest) -> Tuple[int, int, str, bool]:
+    """Read the limit, offset, order_by, and descending query parameters from a request.
 
     Args:
-        request: aiohttp request object
+        request: aiohttp request object.
 
     Returns:
-        A tuple of the limit and offset values
+        A tuple containing:
+        - limit (int): The number of results to return, defaulting to DEFAULT_PAGE_SIZE.
+        - offset (int): The offset for pagination, defaulting to 0.
+        - order_by (str): The field by which to order results, defaulting to "id".
+        - descending (bool): Order results in descending order; defaults to False.
     """
 
     limit = int(request.query.get("limit", DEFAULT_PAGE_SIZE))
     offset = int(request.query.get("offset", 0))
-    return limit, offset
+    order_by = request.query.get("order_by", "id")
+    descending = bool(request.query.get("descending", False))
+    return limit, offset, order_by, descending
diff --git a/acapy_agent/multitenant/admin/routes.py b/acapy_agent/multitenant/admin/routes.py
@@ -16,7 +16,10 @@
 from ...core.profile import ProfileManagerProvider
 from ...messaging.models.base import BaseModelError
 from ...messaging.models.openapi import OpenAPISchema
-from ...messaging.models.paginated_query import PaginatedQuerySchema, get_limit_offset
+from ...messaging.models.paginated_query import (
+    PaginatedQuerySchema,
+    get_paginated_query_params,
+)
 from ...messaging.valid import UUID4_EXAMPLE, JSONWebToken
 from ...multitenant.base import BaseMultitenantManager
 from ...storage.error import StorageError, StorageNotFoundError
@@ -383,7 +386,7 @@ async def wallets_list(request: web.BaseRequest):
     if wallet_name:
         query["wallet_name"] = wallet_name
 
-    limit, offset = get_limit_offset(request)
+    limit, offset, order_by, descending = get_paginated_query_params(request)
 
     try:
         async with profile.session() as session:
@@ -392,6 +395,8 @@ async def wallets_list(request: web.BaseRequest):
                 tag_filter=query,
                 limit=limit,
                 offset=offset,
+                order_by=order_by,
+                descending=descending,
             )
         results = [format_wallet_record(record) for record in records]
         results.sort(key=lambda w: w["created_at"])

diff --git a/acapy_agent/protocols/connections/v1_0/routes.py b/acapy_agent/protocols/connections/v1_0/routes.py
@@ -19,7 +19,10 @@
 from ....connections.models.conn_record import ConnRecord, ConnRecordSchema
 from ....messaging.models.base import BaseModelError
 from ....messaging.models.openapi import OpenAPISchema
-from ....messaging.models.paginated_query import PaginatedQuerySchema, get_limit_offset
+from ....messaging.models.paginated_query import (
+    PaginatedQuerySchema,
+    get_paginated_query_params,
+)
 from ....messaging.valid import (
     ENDPOINT_EXAMPLE,
     ENDPOINT_VALIDATE,
@@ -469,7 +472,7 @@ async def connections_list(request: web.BaseRequest):
     if request.query.get("connection_protocol"):
         post_filter["connection_protocol"] = request.query["connection_protocol"]
 
-    limit, offset = get_limit_offset(request)
+    limit, offset, order_by, descending = get_paginated_query_params(request)
 
     profile = context.profile
     try:
@@ -479,6 +482,8 @@ async def connections_list(request: web.BaseRequest):
                 tag_filter,
                 limit=limit,
                 offset=offset,
+                order_by=order_by,
+                descending=descending,
                 post_filter_positive=post_filter,
                 alt=True,
             )

diff --git a/acapy_agent/protocols/connections/v1_0/tests/test_routes.py b/acapy_agent/protocols/connections/v1_0/tests/test_routes.py
@@ -101,6 +101,8 @@ async def test_connections_list(self):
                     },
                     limit=100,
                     offset=0,
+                    order_by="id",
+                    descending=False,
                     post_filter_positive={
                         "their_role": list(ConnRecord.Role.REQUESTER.value),
                         "connection_protocol": "connections/1.0",

diff --git a/acapy_agent/protocols/issue_credential/v1_0/routes.py b/acapy_agent/protocols/issue_credential/v1_0/routes.py
@@ -23,7 +23,10 @@
 from ....messaging.credential_definitions.util import CRED_DEF_TAGS
 from ....messaging.models.base import BaseModelError
 from ....messaging.models.openapi import OpenAPISchema
-from ....messaging.models.paginated_query import PaginatedQuerySchema, get_limit_offset
+from ....messaging.models.paginated_query import (
+    PaginatedQuerySchema,
+    get_paginated_query_params,
+)
 from ....messaging.valid import (
     INDY_CRED_DEF_ID_EXAMPLE,
     INDY_CRED_DEF_ID_VALIDATE,
@@ -405,7 +408,7 @@ async def credential_exchange_list(request: web.BaseRequest):
         if request.query.get(k, "") != ""
     }
 
-    limit, offset = get_limit_offset(request)
+    limit, offset, order_by, descending = get_paginated_query_params(request)
 
     try:
         async with context.profile.session() as session:
@@ -414,6 +417,8 @@ async def credential_exchange_list(request: web.BaseRequest):
                 tag_filter=tag_filter,
                 limit=limit,
                 offset=offset,
+                order_by=order_by,
+                descending=descending,
                 post_filter_positive=post_filter,
             )
         results = [record.serialize() for record in records]

diff --git a/acapy_agent/protocols/issue_credential/v2_0/routes.py b/acapy_agent/protocols/issue_credential/v2_0/routes.py
@@ -26,7 +26,10 @@
 from ....messaging.decorators.attach_decorator import AttachDecorator
 from ....messaging.models.base import BaseModelError
 from ....messaging.models.openapi import OpenAPISchema
-from ....messaging.models.paginated_query import PaginatedQuerySchema, get_limit_offset
+from ....messaging.models.paginated_query import (
+    PaginatedQuerySchema,
+    get_paginated_query_params,
+)
 from ....messaging.valid import (
     INDY_CRED_DEF_ID_EXAMPLE,
     INDY_CRED_DEF_ID_VALIDATE,
@@ -568,7 +571,7 @@ async def credential_exchange_list(request: web.BaseRequest):
         if request.query.get(k, "") != ""
     }
 
-    limit, offset = get_limit_offset(request)
+    limit, offset, order_by, descending = get_paginated_query_params(request)
 
     try:
         async with profile.session() as session:
@@ -577,6 +580,8 @@ async def credential_exchange_list(request: web.BaseRequest):
                 tag_filter=tag_filter,
                 limit=limit,
                 offset=offset,
+                order_by=order_by,
+                descending=descending,
                 post_filter_positive=post_filter,
             )
 

diff --git a/acapy_agent/protocols/present_proof/v1_0/routes.py b/acapy_agent/protocols/present_proof/v1_0/routes.py
@@ -25,7 +25,10 @@
 from ....messaging.decorators.attach_decorator import AttachDecorator
 from ....messaging.models.base import BaseModelError
 from ....messaging.models.openapi import OpenAPISchema
-from ....messaging.models.paginated_query import PaginatedQuerySchema, get_limit_offset
+from ....messaging.models.paginated_query import (
+    PaginatedQuerySchema,
+    get_paginated_query_params,
+)
 from ....messaging.valid import (
     INDY_EXTRA_WQL_EXAMPLE,
     INDY_EXTRA_WQL_VALIDATE,
@@ -309,7 +312,7 @@ async def presentation_exchange_list(request: web.BaseRequest):
         if request.query.get(k, "") != ""
     }
 
-    limit, offset = get_limit_offset(request)
+    limit, offset, order_by, descending = get_paginated_query_params(request)
 
     try:
         async with context.profile.session() as session:
@@ -318,6 +321,8 @@ async def presentation_exchange_list(request: web.BaseRequest):
                 tag_filter=tag_filter,
                 limit=limit,
                 offset=offset,
+                order_by=order_by,
+                descending=descending,
                 post_filter_positive=post_filter,
             )
         results = [record.serialize() for record in records]

diff --git a/acapy_agent/protocols/present_proof/v2_0/routes.py b/acapy_agent/protocols/present_proof/v2_0/routes.py
@@ -26,7 +26,10 @@
 from ....messaging.decorators.attach_decorator import AttachDecorator
 from ....messaging.models.base import BaseModelError
 from ....messaging.models.openapi import OpenAPISchema
-from ....messaging.models.paginated_query import PaginatedQuerySchema, get_limit_offset
+from ....messaging.models.paginated_query import (
+    PaginatedQuerySchema,
+    get_paginated_query_params,
+)
 from ....messaging.valid import (
     INDY_EXTRA_WQL_EXAMPLE,
     INDY_EXTRA_WQL_VALIDATE,
@@ -448,7 +451,7 @@ async def present_proof_list(request: web.BaseRequest):
         if request.query.get(k, "") != ""
     }
 
-    limit, offset = get_limit_offset(request)
+    limit, offset, order_by, descending = get_paginated_query_params(request)
 
     try:
         async with profile.session() as session:
@@ -457,6 +460,8 @@ async def present_proof_list(request: web.BaseRequest):
                 tag_filter=tag_filter,
                 limit=limit,
                 offset=offset,
+                order_by=order_by,
+                descending=descending,
                 post_filter_positive=post_filter,
             )
         results = [record.serialize() for record in records]

diff --git a/acapy_agent/storage/askar.py b/acapy_agent/storage/askar.py
@@ -174,6 +174,8 @@ async def find_paginated_records(
         tag_query: Optional[Mapping] = None,
         limit: int = DEFAULT_PAGE_SIZE,
         offset: int = 0,
+        order_by: Optional[str] = None,
+        descending: bool = False,
     ) -> Sequence[StorageRecord]:
         """Retrieve a page of records matching a particular type filter and tag query.
 
@@ -182,6 +184,11 @@ async def find_paginated_records(
             tag_query: An optional dictionary of tag filter clauses
             limit: The maximum number of records to retrieve
             offset: The offset to start retrieving records from
+            order_by: An optional field by which to order the records.
+            descending: Whether to order the records in descending order.
+
+        Returns:
+            A sequence of StorageRecord matching the filter and query parameters.
         """
         results = []
 
@@ -190,6 +197,8 @@ async def find_paginated_records(
             tag_filter=tag_query,
             limit=limit,
             offset=offset,
+            order_by=order_by,
+            descending=descending,
             profile=self._session.profile.settings.get("wallet.askar_profile"),
         ):
             results += (
@@ -206,13 +215,19 @@ async def find_all_records(
         self,
         type_filter: str,
         tag_query: Optional[Mapping] = None,
+        order_by: Optional[str] = None,
+        descending: bool = False,
         options: Optional[Mapping] = None,
     ):
         """Retrieve all records matching a particular type filter and tag query."""
         for_update = bool(options and options.get("forUpdate"))
         results = []
         for row in await self._session.handle.fetch_all(
-            type_filter, tag_query, for_update=for_update
+            category=type_filter,
+            tag_filter=tag_query,
+            order_by=order_by,
+            descending=descending,
+            for_update=for_update,
         ):
             results.append(
                 StorageRecord(

diff --git a/acapy_agent/storage/base.py b/acapy_agent/storage/base.py
@@ -99,6 +99,8 @@ async def find_paginated_records(
         tag_query: Optional[Mapping] = None,
         limit: int = DEFAULT_PAGE_SIZE,
         offset: int = 0,
+        order_by: Optional[str] = None,
+        descending: bool = False,
     ) -> Sequence[StorageRecord]:
         """Retrieve a page of records matching a particular type filter and tag query.
 
@@ -107,20 +109,29 @@ async def find_paginated_records(
             tag_query: An optional dictionary of tag filter clauses
             limit: The maximum number of records to retrieve
             offset: The offset to start retrieving records from
+            order_by: An optional field by which to order the records.
+            descending: Whether to order the records in descending order.
+
+        Returns:
+            A sequence of StorageRecord matching the filter and query parameters.
         """
 
     @abstractmethod
     async def find_all_records(
         self,
         type_filter: str,
         tag_query: Optional[Mapping] = None,
+        order_by: Optional[str] = None,
+        descending: bool = False,
         options: Optional[Mapping] = None,
     ) -> Sequence[StorageRecord]:
         """Retrieve all records matching a particular type filter and tag query.
 
         Args:
             type_filter: The type of records to filter by.
             tag_query: An optional dictionary of tag filter clauses.
+            order_by: An optional field by which to order the records.
+            descending: Whether to order the records in descending order.
             options: Additional options for the query.
         """
 

diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -16,8 +16,8 @@ ARG acapy_name="aries-cloudagent"
 ARG acapy_version
 ARG acapy_reqs=[askar,didcommv2]
 
-ENV HOME="/home/$user" \
-    APP_ROOT="$HOME" \
+ENV HOME="/home/$user"
+ENV APP_ROOT="$HOME" \
     LC_ALL=C.UTF-8 \
     LANG=C.UTF-8 \
     PIP_NO_CACHE_DIR=off \
@@ -96,7 +96,9 @@ USER $user
 COPY --from=build /src/dist/acapy_agent*.whl .
 RUN acapy_agent_package=$(find ./ -name "acapy_agent*.whl" | head -n 1) && \
     echo "Installing ${acapy_agent_package} ..." && \
-    pip install --no-cache-dir --find-links=. ${acapy_agent_package}${acapy_reqs} && \
+    pip install --no-cache-dir --find-links=. \
+    --extra-index-url https://test.pypi.org/simple/ \
+    ${acapy_agent_package}${acapy_reqs} && \
     rm acapy_agent*.whl && \
     chmod +rx $(python -m site --user-site) $HOME/.local