Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

PP-129 OPDS 2 with acquisition links #1340

Merged
Merged
Show file tree
Hide file tree
Changes from 25 commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
cc45a75
Added acquisition link serialization for OPDS2
RishiDiwanTT Aug 21, 2023
6c78807
OPDS2 serialization with acuqisition links
RishiDiwanTT Aug 23, 2023
3abfc09
OPDS2 serializer contributors
RishiDiwanTT Aug 23, 2023
4e617f0
Availabilty state revert
RishiDiwanTT Aug 23, 2023
8a7801a
Loan and hold specific statuses for acquisition links
RishiDiwanTT Aug 23, 2023
6aef252
Mypy fixes
RishiDiwanTT Aug 23, 2023
ea99429
Content types come from the serializer now
RishiDiwanTT Aug 23, 2023
c8a7d7c
Fix minor error
RishiDiwanTT Aug 23, 2023
009a6ab
Test fixes
RishiDiwanTT Aug 24, 2023
bad77e6
Additional test cases
RishiDiwanTT Aug 24, 2023
cf6bf9d
Mypy fixes
RishiDiwanTT Aug 24, 2023
49af1b0
Test Fix
RishiDiwanTT Aug 29, 2023
a9137d1
Typing fixes
RishiDiwanTT Aug 29, 2023
03715d3
rebase fixes
RishiDiwanTT Aug 30, 2023
90defb3
Ensure test_url_for is only used during testing
RishiDiwanTT Aug 30, 2023
f77e2dd
Rebase fixes
RishiDiwanTT Sep 1, 2023
498c879
OPDS2 schema validation fixes
RishiDiwanTT Sep 1, 2023
8c4941b
Comment update
RishiDiwanTT Sep 4, 2023
508b81a
Mimetypes are iterated in priority order before serialization
RishiDiwanTT Sep 6, 2023
fbb7223
Fixed up the get_serializer method and wrote tests
RishiDiwanTT Sep 6, 2023
8e0bbe7
Fixed the opds2 mimetype
RishiDiwanTT Sep 6, 2023
ec2620e
Sort MIMEAccept type values in get_serializer
RishiDiwanTT Sep 7, 2023
fcad7b7
Fix for none types
RishiDiwanTT Sep 7, 2023
79fcb0d
Mypy fix
RishiDiwanTT Sep 7, 2023
0cbc81b
Only allow MIMEAccept information when getting the serializer
RishiDiwanTT Sep 7, 2023
9772860
Reverted OPDSMessage responses to type specific responses
RishiDiwanTT Sep 11, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 26 additions & 12 deletions api/controller.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,12 @@
from sqlalchemy import select
from sqlalchemy.orm import eagerload
from sqlalchemy.orm.exc import NoResultFound
from werkzeug.datastructures import MIMEAccept

from api.authentication.access_token import AccessTokenProvider
from api.model.patron_auth import PatronAuthAccessToken
from api.model.time_tracking import PlaytimeEntriesPost, PlaytimeEntriesPostResponse
from api.opds2 import OPDS2NavigationsAnnotator, OPDS2PublicationsAnnotator
from api.opds2 import OPDS2NavigationsAnnotator
from api.saml.controller import SAMLController
from core.analytics import Analytics
from core.app_server import ApplicationVersionController
Expand Down Expand Up @@ -961,6 +962,7 @@ def feed(self, lane_identifier, feed_class=OPDSAcquisitionFeed):
)
return feed.as_response(
max_age=int(max_age) if max_age else None,
mime_types=flask.request.accept_mimetypes,
)

def navigation(self, lane_identifier):
Expand Down Expand Up @@ -1161,7 +1163,7 @@ def search(self, lane_identifier, feed_class=OPDSAcquisitionFeed):
# Run a search.
annotator = self.manager.annotator(lane, facets)
info = OpenSearchDocument.search_info(lane)
return feed_class.search(
response = feed_class.search(
_db=self._db,
title=info["name"],
url=make_url(),
Expand All @@ -1172,6 +1174,9 @@ def search(self, lane_identifier, feed_class=OPDSAcquisitionFeed):
pagination=pagination,
facets=facets,
)
if isinstance(response, ProblemDetail):
return response
return response.as_response(mime_types=flask.request.accept_mimetypes)

def _qa_feed(
self, feed_factory, feed_title, controller_name, facet_class, worklist_factory
Expand Down Expand Up @@ -1305,23 +1310,22 @@ def publications(self):
params: FeedRequestParameters = self._parse_feed_request()
if params.problem:
return params.problem
annotator = OPDS2PublicationsAnnotator(
flask.request.url, params.facets, params.pagination, params.library
)
lane = self.load_lane(None)
annotator = self.manager.annotator(lane, params.facets)
max_age = flask.request.args.get("max_age")
feed = AcquisitonFeedOPDS2.publications(
feed = OPDSAcquisitionFeed.page(
self._db,
lane.display_name,
flask.request.url,
lane,
annotator,
params.facets,
params.pagination,
self.search_engine,
annotator,
max_age=int(max_age) if max_age is not None else None,
)

return Response(
str(feed), status=200, headers={"Content-Type": annotator.OPDS2_TYPE}
return feed.as_response(
mime_types=MIMEAccept([("application/opds+json", 1)]), # Force the type
max_age=int(max_age) if max_age is not None else None,
)

def navigation(self):
Expand Down Expand Up @@ -1464,7 +1468,17 @@ def sync(self):
)

# Then make the feed.
return OPDSAcquisitionFeed.active_loans_for(self.circulation, patron)
feed = OPDSAcquisitionFeed.active_loans_for(self.circulation, patron)
response = feed.as_response(
max_age=0,
private=True,
mime_types=flask.request.accept_mimetypes,
)

last_modified = patron.last_loan_activity_sync
if last_modified:
response.last_modified = last_modified
return response

def borrow(self, identifier_type, identifier, mechanism_id=None):
"""Create a new loan or hold for a book.
Expand Down
19 changes: 8 additions & 11 deletions core/feed_protocol/acquisition.py
Original file line number Diff line number Diff line change
Expand Up @@ -475,7 +475,7 @@ def active_loans_for(
patron: Patron,
annotator: Optional[LibraryAnnotator] = None,
**response_kwargs: Any,
) -> OPDSFeedResponse:
) -> OPDSAcquisitionFeed:
"""A patron specific feed that only contains the loans and holds of a patron"""
db = Session.object_session(patron)
active_loans_by_work = {}
Expand Down Expand Up @@ -516,12 +516,7 @@ def active_loans_for(

feed = OPDSAcquisitionFeed("Active loans and holds", url, works, annotator)
feed.generate_feed()
response = feed.as_response(max_age=0, private=True)

last_modified = patron.last_loan_activity_sync
if last_modified:
response.last_modified = last_modified
return response
return feed

@classmethod
def single_entry_loans_feed(
Expand Down Expand Up @@ -592,9 +587,12 @@ def single_entry_loans_feed(

entry = cls.single_entry(work, annotator, even_if_no_license_pool=True)

# TODO: max_age and private response kwargs
if isinstance(entry, WorkEntry) and entry.computed:
return cls.entry_as_response(entry, **response_kwargs)
elif isinstance(entry, OPDSMessage):
return ProblemDetail(
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It looks like this new case isn't covered by a test, and it seems like the original code doesn't return a problem detail in this case, but converts the OPDSMessage into an error response?

Can you write up a test for this and verify that we aren't changing the behavior of the single item loans feed here?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I realised when I changed the order of the return statements, that the single_entry method does not return a ProblemDetail at all, it only returns an OPDSMessage with the problem.
This does change the behaviour from one of returning an XML document (from OPDSMessage.tag ) to that of returning a ProblemDetail when a problem occurs in the single_entry method. This change has been made since the response types are not bound to always being XML.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think either:

  • we need to handle this at the serialization step, so that we get the same XML response we used to get in the case of an OPDS 1 feed
  • Or check with the mobile apps and see how returning a problem detail here instead of a feed will be handled

Since our mobile apps are the primary consumer of this feed still, we need to make sure they are in sync with any changes that get made to the responses.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can return to the XML format easily, will that be an acceptable behaviour when the client is expecting JSON though?
Re: ProblemDetail: The same method also returns problem details in different circumstances so a problem detail should be already handled by the client side.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The behavior of the mobile apps in the case still worries me. It seems like the thing to do here is to modify OPDSMessage to be non-XML specific, so we are able to serialize it in both OPDS1 and OPDS2 instead of changing the feed behavior.

If that is too much work here, or if you don't want to do that as part of this PR. Can we modify the responses here so that the OPDS1 response (the one that is currently used by the mobile apps) remains the same, rendering the OPDSMessage in the feed as XML and put a follow up ticket in JIRA to handle this case in OPDS2.

@RishiDiwanTT what do you think of those ideas or do you have another idea how we could handle this. The main thing I am concerned about is how this will impact the mobile apps using the OPDS1 feed. The OPDS2 is more "beta" and we can continue to iterate there. I'd like to impact our existing users as little as possible while doing so.

Copy link
Contributor Author

@RishiDiwanTT RishiDiwanTT Sep 11, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, I reverted to protocol specific data, OPDS1 will be the same XML and OPDS2 will be JSON.

entry.urn, status_code=entry.status_code, detail=entry.message
)

return None

Expand Down Expand Up @@ -769,7 +767,7 @@ def search(
pagination: Optional[Pagination] = None,
facets: Optional[FacetsWithEntryPoint] = None,
**response_kwargs: Any,
) -> OPDSFeedResponse | ProblemDetail:
) -> OPDSAcquisitionFeed | ProblemDetail:
"""Run a search against the given search engine and return
the results as a Flask Response.

Expand Down Expand Up @@ -833,8 +831,7 @@ def make_link(ep: Type[EntryPoint]) -> str:
# technically searching the this lane; you are searching the
# library's entire collection, using _some_ of the constraints
# imposed by this lane (notably language and audience).

return OPDSFeedResponse(response=feed.serialize(), **response_kwargs)
return feed

@classmethod
def from_query(
Expand Down
4 changes: 3 additions & 1 deletion core/feed_protocol/annotator/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,7 @@ def categories(cls, work: Work) -> Dict[str, List[Dict[str, str]]]:
# Add the audience as a category of schema
# http://schema.org/audience
if work.audience:
audience_uri = "audience"
audience_uri = "http://schema.org/audience"
categories[audience_uri] = [dict(term=work.audience, label=work.audience)]

# Any book can have a target age, but the target age
Expand Down Expand Up @@ -294,6 +294,8 @@ def annotate_work_entry(

if edition.subtitle:
computed.subtitle = FeedEntryType(text=edition.subtitle)
if edition.sort_title:
computed.sort_title = FeedEntryType(text=edition.sort_title)

author_entries = self.authors(edition)
computed.contributors = author_entries.get("contributors", [])
Expand Down
14 changes: 12 additions & 2 deletions core/feed_protocol/annotator/circulation.py
Original file line number Diff line number Diff line change
Expand Up @@ -652,7 +652,12 @@ def acquisition_link(
else:
initial_type = None
indirect_types = []
link = Acquisition(href=href, rel=rel, type=initial_type)
link = Acquisition(
href=href,
rel=rel,
type=initial_type,
is_loan=True if active_loan else False,
)
indirect = cls.indirect_acquisition(indirect_types)

if indirect is not None:
Expand Down Expand Up @@ -1342,7 +1347,12 @@ def borrow_link(
_external=True,
)
rel = OPDSFeed.BORROW_REL
borrow_link = Acquisition(rel=rel, href=borrow_url, type=OPDSFeed.ENTRY_TYPE)
borrow_link = Acquisition(
rel=rel,
href=borrow_url,
type=OPDSFeed.ENTRY_TYPE,
is_hold=True if active_hold else False,
)

indirect_acquisitions: List[IndirectAcquisition] = []
for lpdm in fulfillment_mechanisms:
Expand Down
11 changes: 9 additions & 2 deletions core/feed_protocol/navigation.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
from __future__ import annotations

from typing import Any, Optional

from sqlalchemy.orm import Session
from typing_extensions import Self
from werkzeug.datastructures import MIMEAccept

from core.feed_protocol.annotator.circulation import CirculationManagerAnnotator
from core.feed_protocol.opds import BaseOPDSFeed
Expand Down Expand Up @@ -79,7 +82,11 @@ def add_entry(
entry.links.append(Link(rel="subsection", href=url, type=type))
self._feed.data_entries.append(entry)

def as_response(self, **kwargs: Any) -> OPDSFeedResponse:
response = super().as_response(**kwargs)
def as_response(
self,
mime_types: Optional[MIMEAccept] = None,
**kwargs: Any,
) -> OPDSFeedResponse:
response = super().as_response(mime_types=mime_types, **kwargs)
response.content_type = OPDSFeed.NAVIGATION_FEED_TYPE
return response
61 changes: 50 additions & 11 deletions core/feed_protocol/opds.py
Original file line number Diff line number Diff line change
@@ -1,48 +1,87 @@
from __future__ import annotations

import logging
from typing import Any, List, Optional
from typing import Any, Dict, List, Optional, Type

from werkzeug.datastructures import MIMEAccept

from core.feed_protocol.base import FeedInterface
from core.feed_protocol.serializer.opds import OPDS1Serializer
from core.feed_protocol.serializer.opds2 import OPDS2Serializer
from core.feed_protocol.types import FeedData, WorkEntry
from core.util.flask_util import OPDSEntryResponse, OPDSFeedResponse
from core.util.opds_writer import OPDSMessage


def get_serializer(
jonathangreen marked this conversation as resolved.
Show resolved Hide resolved
mime_types: Optional[MIMEAccept],
) -> OPDS1Serializer | OPDS2Serializer:
# Loop through and return whichever mimetype is encountered first
# Sort values by q-value first
serializers: Dict[str, Type[Any]] = {
"application/opds+json": OPDS2Serializer,
"application/atom+xml": OPDS1Serializer,
}
if mime_types:
match = mime_types.best_match(
serializers.keys(), default="application/atom+xml"
)
return serializers[match]() # type: ignore[no-any-return]
jonathangreen marked this conversation as resolved.
Show resolved Hide resolved
# Default
return OPDS1Serializer()


class BaseOPDSFeed(FeedInterface):
def __init__(
self, title: str, url: str, precomposed_entries: Optional[List[Any]] = None
self,
title: str,
url: str,
precomposed_entries: Optional[List[OPDSMessage]] = None,
) -> None:
self.url = url
self.title = title
self._precomposed_entries = precomposed_entries or []
self._feed = FeedData()
self._serializer = OPDS1Serializer()
self.log = logging.getLogger(self.__class__.__name__)

def serialize(self) -> bytes:
return self._serializer.serialize_feed(self._feed)
def serialize(self, mime_types: Optional[MIMEAccept] = None) -> bytes:
serializer = get_serializer(mime_types)
return serializer.serialize_feed(self._feed)

def add_link(self, href: str, rel: Optional[str] = None, **kwargs: Any) -> None:
self._feed.add_link(href, rel=rel, **kwargs)

def as_response(self, **kwargs: Any) -> OPDSFeedResponse:
def as_response(
self,
mime_types: Optional[MIMEAccept] = None,
**kwargs: Any,
) -> OPDSFeedResponse:
"""Serialize the feed using the serializer protocol"""
serializer = get_serializer(mime_types)
return OPDSFeedResponse(
self._serializer.serialize_feed(
serializer.serialize_feed(
self._feed, precomposed_entries=self._precomposed_entries
),
content_type=serializer.content_type(),
**kwargs,
)

@classmethod
def entry_as_response(
cls, entry: WorkEntry, **response_kwargs: Any
cls,
entry: WorkEntry,
mime_types: Optional[MIMEAccept] = None,
**response_kwargs: Any,
) -> OPDSEntryResponse:
if not entry.computed:
logging.getLogger().error(f"Entry data has not been generated for {entry}")
raise ValueError(f"Entry data has not been generated")
serializer = OPDS1Serializer()
return OPDSEntryResponse(
response=serializer.serialize_work_entry(entry.computed), **response_kwargs
serializer = get_serializer(mime_types)
response = OPDSEntryResponse(
response=serializer.serialize_work_entry(entry.computed),
**response_kwargs,
)
if isinstance(serializer, OPDS2Serializer):
# Only OPDS2 has the same content type for feed and entry
response.content_type = serializer.content_type()
return response
3 changes: 3 additions & 0 deletions core/feed_protocol/serializer/opds.py
Original file line number Diff line number Diff line change
Expand Up @@ -354,3 +354,6 @@ def _serialize_data_entry(self, entry: DataEntry) -> etree._Element:
@classmethod
def to_string(cls, element: etree._Element) -> bytes:
return cast(bytes, etree.tostring(element))

def content_type(self) -> str:
return OPDSFeed.ACQUISITION_FEED_TYPE
Loading